mirror of
https://github.com/deuxfleurs-org/garage.git
synced 2026-08-14 08:03:11 +00:00
Compare commits
61 Commits
v0.9.2-rc1
...
main-v0.9
| Author | SHA1 | Date | |
|---|---|---|---|
 Alex
|
738bb2f09c | ||
|
Alex Auvolat
|
7dd7cb5759 | ||
|
Alex
|
8b663d8c5b | ||
|
Alex Auvolat
|
c051db8204 | ||
|
Alex
|
e5838b4837 | ||
|
Alex Auvolat
|
87dfaf2eb9 | ||
|
Alex
|
ecf641d88c | ||
|
Alex
|
75cd14926d | ||
|
Alex Auvolat
|
e1dc84e123 | ||
|
Alex Auvolat
|
85f580cbde | ||
|
Alex Auvolat
|
0d3e285d13 | ||
|
Alex
|
95eb8808e8 | ||
|
Alex Auvolat
|
e0a4fc097a | ||
|
Alex Auvolat
|
73551e9a2d | ||
|
Alex
|
80f81fa6f3 | ||
|
Alex Auvolat
|
f267609343 | ||
|
Alex
|
e835196940 | ||
|
Alex Auvolat
|
ba33bb31f1 | ||
|
Alex
|
30abf7e086 | ||
|
Alex Auvolat
|
84018be862 | ||
|
Alex Auvolat
|
091e693670 | ||
|
Alex Auvolat
|
fe8a7819fa | ||
|
Alex
|
26310f3242 | ||
|
Alex Auvolat
|
783b586de9 | ||
|
Alex
|
693b89b94b | ||
|
stefano
|
cf344d73d5 | ||
|
Alex
|
1a0bffae34 | ||
|
Alex Auvolat
|
b55f52a9b7 | ||
|
Alex Auvolat
|
e8f9718ccd | ||
|
Alex
|
fd2e19bf1b | ||
|
Alex Auvolat
|
8cf3d24875 | ||
|
Alex Auvolat
|
a68c37555d | ||
|
Alex Auvolat
|
1e42808a59 | ||
|
Alex Auvolat
|
8dff278b72 | ||
|
Alex
|
a80ce6ab5a | ||
|
Alex Auvolat
|
990205dc3b | ||
|
Alex Auvolat
|
7c86ff6c37 | ||
|
Alex
|
62b01d8705 | ||
|
Alex Auvolat
|
422d45b659 | ||
|
Alex Auvolat
|
a7dddebedd | ||
|
Alex
|
238545e564 | ||
|
Alex Auvolat
|
ec34728b27 | ||
|
Alex
|
afee8c2207 | ||
|
Alex Auvolat
|
eab2b81be2 | ||
|
Alex
|
3168bb34a0 | ||
|
Alex
|
512933a036 | ||
|
Alex Auvolat
|
8670140358 | ||
|
Alex
|
5bb69a1257 | ||
|
Alex Auvolat
|
c8e416aaa5 | ||
|
Yureka
|
fb55682c66 | ||
|
asonix
|
c94bf45cba | ||
|
asonix
|
7c4f3473af | ||
|
Alex Auvolat
|
b6a91e549b | ||
|
Alex
|
32d6b4def8 | ||
|
Alex
|
c4de471de1 | ||
|
Armaël Guéneau
|
16e17375c5 | ||
|
Alex
|
95ab36aae7 | ||
|
Alex Auvolat
|
6a7623e90d | ||
|
Alex
|
70b9904e91 | ||
|
Alex Auvolat
|
a36248a169 | ||
|
Alex Auvolat
|
ee2b0c8dda |
@@ -33,7 +33,11 @@ steps:
|
||||
- ./result/bin/garage_util-*
|
||||
- ./result/bin/garage_web-*
|
||||
- ./result/bin/garage-*
|
||||
- ./result/bin/integration-* || (cat tmp-garage-integration/stderr.log; false)
|
||||
- GARAGE_TEST_INTEGRATION_DB_ENGINE=sled ./result/bin/integration-* || (cat tmp-garage-integration/stderr.log; false)
|
||||
- nix-shell --attr ci --run "killall -9 garage" || true
|
||||
- GARAGE_TEST_INTEGRATION_DB_ENGINE=lmdb ./result/bin/integration-* || (cat tmp-garage-integration/stderr.log; false)
|
||||
- nix-shell --attr ci --run "killall -9 garage" || true
|
||||
- GARAGE_TEST_INTEGRATION_DB_ENGINE=sqlite ./result/bin/integration-* || (cat tmp-garage-integration/stderr.log; false)
|
||||
- rm result
|
||||
- rm -rv tmp-garage-integration
|
||||
|
||||
|
||||
Generated
+63
-16
@@ -1277,7 +1277,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "garage"
|
||||
version = "0.9.1"
|
||||
version = "0.9.4"
|
||||
dependencies = [
|
||||
"assert-json-diff",
|
||||
"async-trait",
|
||||
@@ -1322,6 +1322,7 @@ dependencies = [
|
||||
"sha2",
|
||||
"static_init",
|
||||
"structopt",
|
||||
"syslog-tracing",
|
||||
"timeago",
|
||||
"tokio",
|
||||
"toml",
|
||||
@@ -1331,7 +1332,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "garage_api"
|
||||
version = "0.9.1"
|
||||
version = "0.9.4"
|
||||
dependencies = [
|
||||
"argon2",
|
||||
"async-trait",
|
||||
@@ -1380,7 +1381,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "garage_block"
|
||||
version = "0.9.1"
|
||||
version = "0.9.4"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-compression",
|
||||
@@ -1407,12 +1408,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "garage_db"
|
||||
version = "0.9.1"
|
||||
version = "0.9.4"
|
||||
dependencies = [
|
||||
"err-derive",
|
||||
"heed",
|
||||
"hexdump",
|
||||
"mktemp",
|
||||
"r2d2",
|
||||
"r2d2_sqlite",
|
||||
"rusqlite",
|
||||
"sled",
|
||||
"tracing",
|
||||
@@ -1420,7 +1423,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "garage_model"
|
||||
version = "0.9.1"
|
||||
version = "0.9.4"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -1438,6 +1441,7 @@ dependencies = [
|
||||
"garage_util",
|
||||
"hex",
|
||||
"opentelemetry",
|
||||
"parse_duration",
|
||||
"rand",
|
||||
"serde",
|
||||
"serde_bytes",
|
||||
@@ -1448,7 +1452,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "garage_net"
|
||||
version = "0.9.1"
|
||||
version = "0.9.4"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -1474,7 +1478,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "garage_rpc"
|
||||
version = "0.9.1"
|
||||
version = "0.9.4"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -1509,7 +1513,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "garage_table"
|
||||
version = "0.9.1"
|
||||
version = "0.9.4"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -1531,7 +1535,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "garage_util"
|
||||
version = "0.9.1"
|
||||
version = "0.9.4"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -1565,7 +1569,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "garage_web"
|
||||
version = "0.9.1"
|
||||
version = "0.9.4"
|
||||
dependencies = [
|
||||
"err-derive",
|
||||
"futures",
|
||||
@@ -1707,9 +1711,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hashlink"
|
||||
version = "0.8.4"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7"
|
||||
checksum = "692eaaf7f7607518dd3cef090f1474b61edc5301d8012f09579920df68b725ee"
|
||||
dependencies = [
|
||||
"hashbrown 0.14.3",
|
||||
]
|
||||
@@ -2366,9 +2370,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "libsqlite3-sys"
|
||||
version = "0.27.0"
|
||||
version = "0.28.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716"
|
||||
checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"pkg-config",
|
||||
@@ -3128,6 +3132,28 @@ dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "r2d2"
|
||||
version = "0.8.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93"
|
||||
dependencies = [
|
||||
"log",
|
||||
"parking_lot 0.12.1",
|
||||
"scheduled-thread-pool",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "r2d2_sqlite"
|
||||
version = "0.24.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a982edf65c129796dba72f8775b292ef482b40d035e827a9825b3bc07ccc5f2"
|
||||
dependencies = [
|
||||
"r2d2",
|
||||
"rusqlite",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.8.5"
|
||||
@@ -3321,9 +3347,9 @@ checksum = "3cd14fd5e3b777a7422cca79358c57a8f6e3a703d9ac187448d0daf220c2407f"
|
||||
|
||||
[[package]]
|
||||
name = "rusqlite"
|
||||
version = "0.30.0"
|
||||
version = "0.31.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a78046161564f5e7cd9008aff3b2990b3850dc8e0349119b98e8f251e099f24d"
|
||||
checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae"
|
||||
dependencies = [
|
||||
"bitflags 2.4.2",
|
||||
"fallible-iterator",
|
||||
@@ -3488,6 +3514,15 @@ dependencies = [
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scheduled-thread-pool"
|
||||
version = "0.2.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19"
|
||||
dependencies = [
|
||||
"parking_lot 0.12.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "schemars"
|
||||
version = "0.8.16"
|
||||
@@ -3896,6 +3931,17 @@ dependencies = [
|
||||
"unicode-xid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syslog-tracing"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "340b1540dcdb6b066bc2966e7974f977ab1a38f21b2be189014ffb0cc2405768"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"tracing-core",
|
||||
"tracing-subscriber",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "system-configuration"
|
||||
version = "0.5.1"
|
||||
@@ -4441,6 +4487,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
"rand",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -34,7 +34,7 @@ args@{
|
||||
ignoreLockHash,
|
||||
}:
|
||||
let
|
||||
nixifiedLockHash = "69c86fff0acd6c7a9a19dc6966b4cbd48e8a50c5a9fb40b3090ad71aaa5b55d0";
|
||||
nixifiedLockHash = "9ea4045dd09421583b69811f95797af9c1f16239ecce89d8a6f5a9319d7d8526";
|
||||
workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc;
|
||||
currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock);
|
||||
lockHashIgnored = if ignoreLockHash
|
||||
@@ -58,17 +58,17 @@ in
|
||||
{
|
||||
cargo2nixVersion = "0.11.0";
|
||||
workspace = {
|
||||
garage_db = rustPackages.unknown.garage_db."0.9.1";
|
||||
garage_util = rustPackages.unknown.garage_util."0.9.1";
|
||||
garage_net = rustPackages.unknown.garage_net."0.9.1";
|
||||
garage_rpc = rustPackages.unknown.garage_rpc."0.9.1";
|
||||
garage_db = rustPackages.unknown.garage_db."0.9.4";
|
||||
garage_util = rustPackages.unknown.garage_util."0.9.4";
|
||||
garage_net = rustPackages.unknown.garage_net."0.9.4";
|
||||
garage_rpc = rustPackages.unknown.garage_rpc."0.9.4";
|
||||
format_table = rustPackages.unknown.format_table."0.1.1";
|
||||
garage_table = rustPackages.unknown.garage_table."0.9.1";
|
||||
garage_block = rustPackages.unknown.garage_block."0.9.1";
|
||||
garage_model = rustPackages.unknown.garage_model."0.9.1";
|
||||
garage_api = rustPackages.unknown.garage_api."0.9.1";
|
||||
garage_web = rustPackages.unknown.garage_web."0.9.1";
|
||||
garage = rustPackages.unknown.garage."0.9.1";
|
||||
garage_table = rustPackages.unknown.garage_table."0.9.4";
|
||||
garage_block = rustPackages.unknown.garage_block."0.9.4";
|
||||
garage_model = rustPackages.unknown.garage_model."0.9.4";
|
||||
garage_api = rustPackages.unknown.garage_api."0.9.4";
|
||||
garage_web = rustPackages.unknown.garage_web."0.9.4";
|
||||
garage = rustPackages.unknown.garage."0.9.4";
|
||||
k2v-client = rustPackages.unknown.k2v-client."0.0.4";
|
||||
};
|
||||
"registry+https://github.com/rust-lang/crates.io-index".addr2line."0.21.0" = overridableMkRustCrate (profileName: rec {
|
||||
@@ -100,13 +100,13 @@ in
|
||||
(lib.optional (rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "std")
|
||||
];
|
||||
dependencies = {
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "cfg_if" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "cfg_if" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "getrandom" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".getrandom."0.2.12" { inherit profileName; }).out;
|
||||
${ if (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") && !((hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l") && hostPlatform.parsed.kernel.name == "none") then "once_cell" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.19.0" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "zerocopy" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".zerocopy."0.7.32" { inherit profileName; }).out;
|
||||
${ if (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") && !((hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l") && hostPlatform.parsed.kernel.name == "none") then "once_cell" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.19.0" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "zerocopy" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".zerocopy."0.7.32" { inherit profileName; }).out;
|
||||
};
|
||||
buildDependencies = {
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "version_check" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".version_check."0.9.4" { profileName = "__noProfile"; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "version_check" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".version_check."0.9.4" { profileName = "__noProfile"; }).out;
|
||||
};
|
||||
});
|
||||
|
||||
@@ -130,7 +130,7 @@ in
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5"; };
|
||||
features = builtins.concatLists [
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "alloc")
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "alloc")
|
||||
];
|
||||
});
|
||||
|
||||
@@ -1624,8 +1624,8 @@ in
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"; };
|
||||
features = builtins.concatLists [
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "alloc")
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "default")
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "alloc")
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "default")
|
||||
];
|
||||
});
|
||||
|
||||
@@ -1871,9 +1871,9 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"unknown".garage."0.9.1" = overridableMkRustCrate (profileName: rec {
|
||||
"unknown".garage."0.9.4" = overridableMkRustCrate (profileName: rec {
|
||||
name = "garage";
|
||||
version = "0.9.1";
|
||||
version = "0.9.4";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/garage");
|
||||
features = builtins.concatLists [
|
||||
@@ -1889,6 +1889,8 @@ in
|
||||
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/prometheus") "prometheus")
|
||||
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled") "sled")
|
||||
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite") "sqlite")
|
||||
(lib.optional (rootFeatures' ? "garage/syslog") "syslog")
|
||||
(lib.optional (rootFeatures' ? "garage/syslog" || rootFeatures' ? "garage/syslog-tracing") "syslog-tracing")
|
||||
(lib.optional (rootFeatures' ? "garage/system-libs") "system-libs")
|
||||
(lib.optional (rootFeatures' ? "garage/telemetry-otlp") "telemetry-otlp")
|
||||
];
|
||||
@@ -1900,15 +1902,15 @@ in
|
||||
format_table = (rustPackages."unknown".format_table."0.1.1" { inherit profileName; }).out;
|
||||
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out;
|
||||
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.30" { inherit profileName; }).out;
|
||||
garage_api = (rustPackages."unknown".garage_api."0.9.1" { inherit profileName; }).out;
|
||||
garage_block = (rustPackages."unknown".garage_block."0.9.1" { inherit profileName; }).out;
|
||||
garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out;
|
||||
garage_model = (rustPackages."unknown".garage_model."0.9.1" { inherit profileName; }).out;
|
||||
garage_net = (rustPackages."unknown".garage_net."0.9.1" { inherit profileName; }).out;
|
||||
garage_rpc = (rustPackages."unknown".garage_rpc."0.9.1" { inherit profileName; }).out;
|
||||
garage_table = (rustPackages."unknown".garage_table."0.9.1" { inherit profileName; }).out;
|
||||
garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out;
|
||||
garage_web = (rustPackages."unknown".garage_web."0.9.1" { inherit profileName; }).out;
|
||||
garage_api = (rustPackages."unknown".garage_api."0.9.4" { inherit profileName; }).out;
|
||||
garage_block = (rustPackages."unknown".garage_block."0.9.4" { inherit profileName; }).out;
|
||||
garage_db = (rustPackages."unknown".garage_db."0.9.4" { inherit profileName; }).out;
|
||||
garage_model = (rustPackages."unknown".garage_model."0.9.4" { inherit profileName; }).out;
|
||||
garage_net = (rustPackages."unknown".garage_net."0.9.4" { inherit profileName; }).out;
|
||||
garage_rpc = (rustPackages."unknown".garage_rpc."0.9.4" { inherit profileName; }).out;
|
||||
garage_table = (rustPackages."unknown".garage_table."0.9.4" { inherit profileName; }).out;
|
||||
garage_util = (rustPackages."unknown".garage_util."0.9.4" { inherit profileName; }).out;
|
||||
garage_web = (rustPackages."unknown".garage_web."0.9.4" { inherit profileName; }).out;
|
||||
git_version = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.9" { inherit profileName; }).out;
|
||||
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
|
||||
sodiumoxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }).out;
|
||||
@@ -1921,6 +1923,7 @@ in
|
||||
serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.196" { inherit profileName; }).out;
|
||||
serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.14" { inherit profileName; }).out;
|
||||
structopt = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".structopt."0.3.26" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/syslog" || rootFeatures' ? "garage/syslog-tracing" then "syslog_tracing" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syslog-tracing."0.3.0" { inherit profileName; }).out;
|
||||
timeago = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".timeago."0.4.2" { inherit profileName; }).out;
|
||||
tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.36.0" { inherit profileName; }).out;
|
||||
toml = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".toml."0.8.10" { inherit profileName; }).out;
|
||||
@@ -1946,9 +1949,9 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"unknown".garage_api."0.9.1" = overridableMkRustCrate (profileName: rec {
|
||||
"unknown".garage_api."0.9.4" = overridableMkRustCrate (profileName: rec {
|
||||
name = "garage_api";
|
||||
version = "0.9.1";
|
||||
version = "0.9.4";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/api");
|
||||
features = builtins.concatLists [
|
||||
@@ -1968,12 +1971,12 @@ in
|
||||
form_urlencoded = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.2.1" { inherit profileName; }).out;
|
||||
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out;
|
||||
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.30" { inherit profileName; }).out;
|
||||
garage_block = (rustPackages."unknown".garage_block."0.9.1" { inherit profileName; }).out;
|
||||
garage_model = (rustPackages."unknown".garage_model."0.9.1" { inherit profileName; }).out;
|
||||
garage_net = (rustPackages."unknown".garage_net."0.9.1" { inherit profileName; }).out;
|
||||
garage_rpc = (rustPackages."unknown".garage_rpc."0.9.1" { inherit profileName; }).out;
|
||||
garage_table = (rustPackages."unknown".garage_table."0.9.1" { inherit profileName; }).out;
|
||||
garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out;
|
||||
garage_block = (rustPackages."unknown".garage_block."0.9.4" { inherit profileName; }).out;
|
||||
garage_model = (rustPackages."unknown".garage_model."0.9.4" { inherit profileName; }).out;
|
||||
garage_net = (rustPackages."unknown".garage_net."0.9.4" { inherit profileName; }).out;
|
||||
garage_rpc = (rustPackages."unknown".garage_rpc."0.9.4" { inherit profileName; }).out;
|
||||
garage_table = (rustPackages."unknown".garage_table."0.9.4" { inherit profileName; }).out;
|
||||
garage_util = (rustPackages."unknown".garage_util."0.9.4" { inherit profileName; }).out;
|
||||
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
|
||||
hmac = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.12.1" { inherit profileName; }).out;
|
||||
http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."1.0.0" { inherit profileName; }).out;
|
||||
@@ -2004,9 +2007,9 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"unknown".garage_block."0.9.1" = overridableMkRustCrate (profileName: rec {
|
||||
"unknown".garage_block."0.9.4" = overridableMkRustCrate (profileName: rec {
|
||||
name = "garage_block";
|
||||
version = "0.9.1";
|
||||
version = "0.9.4";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/block");
|
||||
features = builtins.concatLists [
|
||||
@@ -2020,11 +2023,11 @@ in
|
||||
bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.3.0" { inherit profileName; }).out;
|
||||
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out;
|
||||
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.30" { inherit profileName; }).out;
|
||||
garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out;
|
||||
garage_net = (rustPackages."unknown".garage_net."0.9.1" { inherit profileName; }).out;
|
||||
garage_rpc = (rustPackages."unknown".garage_rpc."0.9.1" { inherit profileName; }).out;
|
||||
garage_table = (rustPackages."unknown".garage_table."0.9.1" { inherit profileName; }).out;
|
||||
garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out;
|
||||
garage_db = (rustPackages."unknown".garage_db."0.9.4" { inherit profileName; }).out;
|
||||
garage_net = (rustPackages."unknown".garage_net."0.9.4" { inherit profileName; }).out;
|
||||
garage_rpc = (rustPackages."unknown".garage_rpc."0.9.4" { inherit profileName; }).out;
|
||||
garage_table = (rustPackages."unknown".garage_table."0.9.4" { inherit profileName; }).out;
|
||||
garage_util = (rustPackages."unknown".garage_util."0.9.4" { inherit profileName; }).out;
|
||||
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
|
||||
opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
|
||||
rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
|
||||
@@ -2037,9 +2040,9 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"unknown".garage_db."0.9.1" = overridableMkRustCrate (profileName: rec {
|
||||
"unknown".garage_db."0.9.4" = overridableMkRustCrate (profileName: rec {
|
||||
name = "garage_db";
|
||||
version = "0.9.1";
|
||||
version = "0.9.4";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/db");
|
||||
features = builtins.concatLists [
|
||||
@@ -2047,6 +2050,8 @@ in
|
||||
(lib.optional (rootFeatures' ? "garage_db/default") "default")
|
||||
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb") "heed")
|
||||
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb") "lmdb")
|
||||
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "r2d2")
|
||||
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "r2d2_sqlite")
|
||||
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "rusqlite")
|
||||
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled") "sled")
|
||||
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "sqlite")
|
||||
@@ -2055,7 +2060,9 @@ in
|
||||
err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
|
||||
${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/lmdb" then "heed" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".heed."0.11.0" { inherit profileName; }).out;
|
||||
hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "rusqlite" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusqlite."0.30.0" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "r2d2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".r2d2."0.8.10" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "r2d2_sqlite" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".r2d2_sqlite."0.24.0" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "rusqlite" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusqlite."0.31.0" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sled" then "sled" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sled."0.34.7" { inherit profileName; }).out;
|
||||
tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.40" { inherit profileName; }).out;
|
||||
};
|
||||
@@ -2064,9 +2071,9 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"unknown".garage_model."0.9.1" = overridableMkRustCrate (profileName: rec {
|
||||
"unknown".garage_model."0.9.4" = overridableMkRustCrate (profileName: rec {
|
||||
name = "garage_model";
|
||||
version = "0.9.1";
|
||||
version = "0.9.4";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/model");
|
||||
features = builtins.concatLists [
|
||||
@@ -2085,14 +2092,15 @@ in
|
||||
err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
|
||||
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out;
|
||||
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.30" { inherit profileName; }).out;
|
||||
garage_block = (rustPackages."unknown".garage_block."0.9.1" { inherit profileName; }).out;
|
||||
garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out;
|
||||
garage_net = (rustPackages."unknown".garage_net."0.9.1" { inherit profileName; }).out;
|
||||
garage_rpc = (rustPackages."unknown".garage_rpc."0.9.1" { inherit profileName; }).out;
|
||||
garage_table = (rustPackages."unknown".garage_table."0.9.1" { inherit profileName; }).out;
|
||||
garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out;
|
||||
garage_block = (rustPackages."unknown".garage_block."0.9.4" { inherit profileName; }).out;
|
||||
garage_db = (rustPackages."unknown".garage_db."0.9.4" { inherit profileName; }).out;
|
||||
garage_net = (rustPackages."unknown".garage_net."0.9.4" { inherit profileName; }).out;
|
||||
garage_rpc = (rustPackages."unknown".garage_rpc."0.9.4" { inherit profileName; }).out;
|
||||
garage_table = (rustPackages."unknown".garage_table."0.9.4" { inherit profileName; }).out;
|
||||
garage_util = (rustPackages."unknown".garage_util."0.9.4" { inherit profileName; }).out;
|
||||
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
|
||||
opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
|
||||
parse_duration = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parse_duration."2.1.1" { inherit profileName; }).out;
|
||||
rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
|
||||
serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.196" { inherit profileName; }).out;
|
||||
serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.14" { inherit profileName; }).out;
|
||||
@@ -2102,9 +2110,9 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"unknown".garage_net."0.9.1" = overridableMkRustCrate (profileName: rec {
|
||||
"unknown".garage_net."0.9.4" = overridableMkRustCrate (profileName: rec {
|
||||
name = "garage_net";
|
||||
version = "0.9.1";
|
||||
version = "0.9.4";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/net");
|
||||
features = builtins.concatLists [
|
||||
@@ -2139,9 +2147,9 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"unknown".garage_rpc."0.9.1" = overridableMkRustCrate (profileName: rec {
|
||||
"unknown".garage_rpc."0.9.4" = overridableMkRustCrate (profileName: rec {
|
||||
name = "garage_rpc";
|
||||
version = "0.9.1";
|
||||
version = "0.9.4";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/rpc");
|
||||
features = builtins.concatLists [
|
||||
@@ -2163,9 +2171,9 @@ in
|
||||
format_table = (rustPackages."unknown".format_table."0.1.1" { inherit profileName; }).out;
|
||||
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out;
|
||||
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.30" { inherit profileName; }).out;
|
||||
garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out;
|
||||
garage_net = (rustPackages."unknown".garage_net."0.9.1" { inherit profileName; }).out;
|
||||
garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out;
|
||||
garage_db = (rustPackages."unknown".garage_db."0.9.4" { inherit profileName; }).out;
|
||||
garage_net = (rustPackages."unknown".garage_net."0.9.4" { inherit profileName; }).out;
|
||||
garage_util = (rustPackages."unknown".garage_util."0.9.4" { inherit profileName; }).out;
|
||||
gethostname = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.4.3" { inherit profileName; }).out;
|
||||
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
|
||||
itertools = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itertools."0.12.1" { inherit profileName; }).out;
|
||||
@@ -2187,9 +2195,9 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"unknown".garage_table."0.9.1" = overridableMkRustCrate (profileName: rec {
|
||||
"unknown".garage_table."0.9.4" = overridableMkRustCrate (profileName: rec {
|
||||
name = "garage_table";
|
||||
version = "0.9.1";
|
||||
version = "0.9.4";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/table");
|
||||
dependencies = {
|
||||
@@ -2198,9 +2206,9 @@ in
|
||||
bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.5.0" { inherit profileName; }).out;
|
||||
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out;
|
||||
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.30" { inherit profileName; }).out;
|
||||
garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out;
|
||||
garage_rpc = (rustPackages."unknown".garage_rpc."0.9.1" { inherit profileName; }).out;
|
||||
garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out;
|
||||
garage_db = (rustPackages."unknown".garage_db."0.9.4" { inherit profileName; }).out;
|
||||
garage_rpc = (rustPackages."unknown".garage_rpc."0.9.4" { inherit profileName; }).out;
|
||||
garage_util = (rustPackages."unknown".garage_util."0.9.4" { inherit profileName; }).out;
|
||||
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
|
||||
hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out;
|
||||
opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
|
||||
@@ -2212,9 +2220,9 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"unknown".garage_util."0.9.1" = overridableMkRustCrate (profileName: rec {
|
||||
"unknown".garage_util."0.9.4" = overridableMkRustCrate (profileName: rec {
|
||||
name = "garage_util";
|
||||
version = "0.9.1";
|
||||
version = "0.9.4";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/util");
|
||||
features = builtins.concatLists [
|
||||
@@ -2230,8 +2238,8 @@ in
|
||||
digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.7" { inherit profileName; }).out;
|
||||
err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
|
||||
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out;
|
||||
garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out;
|
||||
garage_net = (rustPackages."unknown".garage_net."0.9.1" { inherit profileName; }).out;
|
||||
garage_db = (rustPackages."unknown".garage_db."0.9.4" { inherit profileName; }).out;
|
||||
garage_net = (rustPackages."unknown".garage_net."0.9.4" { inherit profileName; }).out;
|
||||
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
|
||||
hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out;
|
||||
http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."1.0.0" { inherit profileName; }).out;
|
||||
@@ -2256,18 +2264,18 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"unknown".garage_web."0.9.1" = overridableMkRustCrate (profileName: rec {
|
||||
"unknown".garage_web."0.9.4" = overridableMkRustCrate (profileName: rec {
|
||||
name = "garage_web";
|
||||
version = "0.9.1";
|
||||
version = "0.9.4";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/web");
|
||||
dependencies = {
|
||||
err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
|
||||
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out;
|
||||
garage_api = (rustPackages."unknown".garage_api."0.9.1" { inherit profileName; }).out;
|
||||
garage_model = (rustPackages."unknown".garage_model."0.9.1" { inherit profileName; }).out;
|
||||
garage_table = (rustPackages."unknown".garage_table."0.9.1" { inherit profileName; }).out;
|
||||
garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out;
|
||||
garage_api = (rustPackages."unknown".garage_api."0.9.4" { inherit profileName; }).out;
|
||||
garage_model = (rustPackages."unknown".garage_model."0.9.4" { inherit profileName; }).out;
|
||||
garage_table = (rustPackages."unknown".garage_table."0.9.4" { inherit profileName; }).out;
|
||||
garage_util = (rustPackages."unknown".garage_util."0.9.4" { inherit profileName; }).out;
|
||||
http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."1.0.0" { inherit profileName; }).out;
|
||||
http_body_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body-util."0.1.0" { inherit profileName; }).out;
|
||||
hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."1.1.0" { inherit profileName; }).out;
|
||||
@@ -2422,25 +2430,25 @@ in
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"; };
|
||||
features = builtins.concatLists [
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "ahash")
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "allocator-api2")
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "default")
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "inline-more")
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "ahash")
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "allocator-api2")
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "default")
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "inline-more")
|
||||
[ "raw" ]
|
||||
];
|
||||
dependencies = {
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "ahash" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ahash."0.8.7" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "allocator_api2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".allocator-api2."0.2.16" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "ahash" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ahash."0.8.7" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "allocator_api2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".allocator-api2."0.2.16" { inherit profileName; }).out;
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".hashlink."0.8.4" = overridableMkRustCrate (profileName: rec {
|
||||
"registry+https://github.com/rust-lang/crates.io-index".hashlink."0.9.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "hashlink";
|
||||
version = "0.8.4";
|
||||
version = "0.9.0";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7"; };
|
||||
src = fetchCratesIo { inherit name version; sha256 = "692eaaf7f7607518dd3cef090f1474b61edc5301d8012f09579920df68b725ee"; };
|
||||
dependencies = {
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "hashbrown" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hashbrown."0.14.3" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "hashbrown" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hashbrown."0.14.3" { inherit profileName; }).out;
|
||||
};
|
||||
});
|
||||
|
||||
@@ -3354,24 +3362,24 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".libsqlite3-sys."0.27.0" = overridableMkRustCrate (profileName: rec {
|
||||
"registry+https://github.com/rust-lang/crates.io-index".libsqlite3-sys."0.28.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "libsqlite3-sys";
|
||||
version = "0.27.0";
|
||||
version = "0.28.0";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716"; };
|
||||
src = fetchCratesIo { inherit name version; sha256 = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f"; };
|
||||
features = builtins.concatLists [
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "bundled")
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "bundled_bindings")
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "cc")
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "default")
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "min_sqlite_version_3_14_0")
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "pkg-config")
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "vcpkg")
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "default")
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "min_sqlite_version_3_14_0")
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "pkg-config")
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "vcpkg")
|
||||
];
|
||||
buildDependencies = {
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs" then "cc" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.0.83" { profileName = "__noProfile"; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "pkg_config" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".pkg-config."0.3.29" { profileName = "__noProfile"; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "vcpkg" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".vcpkg."0.2.15" { profileName = "__noProfile"; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "pkg_config" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".pkg-config."0.3.29" { profileName = "__noProfile"; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "vcpkg" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".vcpkg."0.2.15" { profileName = "__noProfile"; }).out;
|
||||
};
|
||||
});
|
||||
|
||||
@@ -3960,11 +3968,11 @@ in
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"; };
|
||||
features = builtins.concatLists [
|
||||
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "default")
|
||||
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "default")
|
||||
];
|
||||
dependencies = {
|
||||
${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "lock_api" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lock_api."0.4.11" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "parking_lot_core" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot_core."0.9.9" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "lock_api" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lock_api."0.4.11" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "parking_lot_core" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot_core."0.9.9" { inherit profileName; }).out;
|
||||
};
|
||||
});
|
||||
|
||||
@@ -3989,11 +3997,11 @@ in
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e"; };
|
||||
dependencies = {
|
||||
${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "cfg_if" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out;
|
||||
${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") && hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.153" { inherit profileName; }).out;
|
||||
${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") && hostPlatform.parsed.kernel.name == "redox" then "syscall" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".redox_syscall."0.4.1" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "smallvec" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".smallvec."1.13.1" { inherit profileName; }).out;
|
||||
${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") && hostPlatform.isWindows then "windows_targets" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-targets."0.48.5" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "cfg_if" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out;
|
||||
${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") && hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.153" { inherit profileName; }).out;
|
||||
${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") && hostPlatform.parsed.kernel.name == "redox" then "syscall" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".redox_syscall."0.4.1" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "smallvec" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".smallvec."1.13.1" { inherit profileName; }).out;
|
||||
${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") && hostPlatform.isWindows then "windows_targets" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-targets."0.48.5" { inherit profileName; }).out;
|
||||
};
|
||||
});
|
||||
|
||||
@@ -4435,6 +4443,30 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".r2d2."0.8.10" = overridableMkRustCrate (profileName: rec {
|
||||
name = "r2d2";
|
||||
version = "0.8.10";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93"; };
|
||||
dependencies = {
|
||||
${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "log" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.20" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "parking_lot" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.12.1" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "scheduled_thread_pool" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".scheduled-thread-pool."0.2.7" { inherit profileName; }).out;
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".r2d2_sqlite."0.24.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "r2d2_sqlite";
|
||||
version = "0.24.0";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "6a982edf65c129796dba72f8775b292ef482b40d035e827a9825b3bc07ccc5f2"; };
|
||||
dependencies = {
|
||||
${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "r2d2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".r2d2."0.8.10" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "rusqlite" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusqlite."0.31.0" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "uuid" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".uuid."1.4.1" { inherit profileName; }).out;
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" = overridableMkRustCrate (profileName: rec {
|
||||
name = "rand";
|
||||
version = "0.8.5";
|
||||
@@ -4502,7 +4534,7 @@ in
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa"; };
|
||||
dependencies = {
|
||||
${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "bitflags" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/opentelemetry-prometheus" || rootFeatures' ? "garage/prometheus" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage/telemetry-otlp" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus" || rootFeatures' ? "garage_api/prometheus" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "bitflags" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; }).out;
|
||||
};
|
||||
});
|
||||
|
||||
@@ -4763,22 +4795,23 @@ in
|
||||
];
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".rusqlite."0.30.0" = overridableMkRustCrate (profileName: rec {
|
||||
"registry+https://github.com/rust-lang/crates.io-index".rusqlite."0.31.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "rusqlite";
|
||||
version = "0.30.0";
|
||||
version = "0.31.0";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "a78046161564f5e7cd9008aff3b2990b3850dc8e0349119b98e8f251e099f24d"; };
|
||||
src = fetchCratesIo { inherit name version; sha256 = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae"; };
|
||||
features = builtins.concatLists [
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "backup")
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "bundled")
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "modern_sqlite")
|
||||
];
|
||||
dependencies = {
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "bitflags" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."2.4.2" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "fallible_iterator" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fallible-iterator."0.3.0" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "fallible_streaming_iterator" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fallible-streaming-iterator."0.1.9" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "hashlink" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hashlink."0.8.4" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "libsqlite3_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libsqlite3-sys."0.27.0" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "smallvec" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".smallvec."1.13.1" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "bitflags" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."2.4.2" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "fallible_iterator" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fallible-iterator."0.3.0" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "fallible_streaming_iterator" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fallible-streaming-iterator."0.1.9" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "hashlink" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hashlink."0.9.0" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "libsqlite3_sys" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libsqlite3-sys."0.28.0" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "smallvec" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".smallvec."1.13.1" { inherit profileName; }).out;
|
||||
};
|
||||
});
|
||||
|
||||
@@ -4992,6 +5025,16 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".scheduled-thread-pool."0.2.7" = overridableMkRustCrate (profileName: rec {
|
||||
name = "scheduled-thread-pool";
|
||||
version = "0.2.7";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19"; };
|
||||
dependencies = {
|
||||
${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "parking_lot" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.12.1" { inherit profileName; }).out;
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".schemars."0.8.16" = overridableMkRustCrate (profileName: rec {
|
||||
name = "schemars";
|
||||
version = "0.8.16";
|
||||
@@ -5572,6 +5615,18 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".syslog-tracing."0.3.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "syslog-tracing";
|
||||
version = "0.3.0";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "340b1540dcdb6b066bc2966e7974f977ab1a38f21b2be189014ffb0cc2405768"; };
|
||||
dependencies = {
|
||||
${ if rootFeatures' ? "garage/syslog" || rootFeatures' ? "garage/syslog-tracing" then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.153" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/syslog" || rootFeatures' ? "garage/syslog-tracing" then "tracing_core" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-core."0.1.32" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/syslog" || rootFeatures' ? "garage/syslog-tracing" then "tracing_subscriber" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-subscriber."0.3.18" { inherit profileName; }).out;
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".system-configuration."0.5.1" = overridableMkRustCrate (profileName: rec {
|
||||
name = "system-configuration";
|
||||
version = "0.5.1";
|
||||
@@ -6367,13 +6422,16 @@ in
|
||||
src = fetchCratesIo { inherit name version; sha256 = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d"; };
|
||||
features = builtins.concatLists [
|
||||
[ "default" ]
|
||||
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "fast-rng")
|
||||
[ "getrandom" ]
|
||||
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite") "rand")
|
||||
[ "rng" ]
|
||||
[ "std" ]
|
||||
[ "v4" ]
|
||||
];
|
||||
dependencies = {
|
||||
getrandom = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".getrandom."0.2.12" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" then "rand" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
|
||||
};
|
||||
});
|
||||
|
||||
@@ -6894,7 +6952,7 @@ in
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be"; };
|
||||
features = builtins.concatLists [
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "simd")
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery") "simd")
|
||||
];
|
||||
dependencies = {
|
||||
${ if false then "zerocopy_derive" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".zerocopy-derive."0.7.32" { profileName = "__noProfile"; }).out;
|
||||
@@ -6907,9 +6965,9 @@ in
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6"; };
|
||||
dependencies = {
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "proc_macro2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.78" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "quote" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.35" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "syn" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."2.0.48" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "proc_macro2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.78" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "quote" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.35" { inherit profileName; }).out;
|
||||
${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/default" || rootFeatures' ? "garage_db/r2d2_sqlite" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/default" || rootFeatures' ? "garage_model/sqlite" || rootFeatures' ? "garage_rpc/kube" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "syn" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."2.0.48" { inherit profileName; }).out;
|
||||
};
|
||||
});
|
||||
|
||||
|
||||
+13
-10
@@ -21,15 +21,15 @@ default-members = ["src/garage"]
|
||||
|
||||
# Internal Garage crates
|
||||
format_table = { version = "0.1.1", path = "src/format-table" }
|
||||
garage_api = { version = "0.9.1", path = "src/api" }
|
||||
garage_block = { version = "0.9.1", path = "src/block" }
|
||||
garage_db = { version = "0.9.1", path = "src/db", default-features = false }
|
||||
garage_model = { version = "0.9.1", path = "src/model", default-features = false }
|
||||
garage_net = { version = "0.9.1", path = "src/net" }
|
||||
garage_rpc = { version = "0.9.1", path = "src/rpc" }
|
||||
garage_table = { version = "0.9.1", path = "src/table" }
|
||||
garage_util = { version = "0.9.1", path = "src/util" }
|
||||
garage_web = { version = "0.9.1", path = "src/web" }
|
||||
garage_api = { version = "0.9.4", path = "src/api" }
|
||||
garage_block = { version = "0.9.4", path = "src/block" }
|
||||
garage_db = { version = "0.9.4", path = "src/db", default-features = false }
|
||||
garage_model = { version = "0.9.4", path = "src/model", default-features = false }
|
||||
garage_net = { version = "0.9.4", path = "src/net" }
|
||||
garage_rpc = { version = "0.9.4", path = "src/rpc" }
|
||||
garage_table = { version = "0.9.4", path = "src/table" }
|
||||
garage_util = { version = "0.9.4", path = "src/util" }
|
||||
garage_web = { version = "0.9.4", path = "src/web" }
|
||||
k2v-client = { version = "0.0.4", path = "src/k2v-client" }
|
||||
|
||||
# External crates from crates.io
|
||||
@@ -72,11 +72,14 @@ kuska-handshake = { version = "0.2.0", features = ["default", "async_std"] }
|
||||
clap = { version = "4.1", features = ["derive", "env"] }
|
||||
pretty_env_logger = "0.5"
|
||||
structopt = { version = "0.3", default-features = false }
|
||||
syslog-tracing = "0.3"
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
|
||||
heed = { version = "0.11", default-features = false, features = ["lmdb"] }
|
||||
rusqlite = "0.30.0"
|
||||
rusqlite = "0.31.0"
|
||||
r2d2 = "0.8"
|
||||
r2d2_sqlite = "0.24"
|
||||
sled = "0.34"
|
||||
|
||||
async-compression = { version = "0.4", features = ["tokio", "zstd"] }
|
||||
|
||||
@@ -55,8 +55,8 @@ Create your key and bucket:
|
||||
|
||||
```bash
|
||||
garage key create my-key
|
||||
garage bucket create backup
|
||||
garage bucket allow backup --read --write --key my-key
|
||||
garage bucket create backups
|
||||
garage bucket allow backups --read --write --key my-key
|
||||
```
|
||||
|
||||
Then register your Key ID and Secret key in your environment:
|
||||
|
||||
@@ -259,7 +259,7 @@ duck --delete garage:/my-files/an-object.txt
|
||||
|
||||
## WinSCP (libs3) {#winscp}
|
||||
|
||||
*You can find instructions on how to use the GUI in french [in our wiki](https://wiki.deuxfleurs.fr/fr/Guide/Garage/WinSCP).*
|
||||
*You can find instructions on how to use the GUI in french [in our wiki](https://guide.deuxfleurs.fr/prise_en_main/winscp/).*
|
||||
|
||||
How to use `winscp.com`, the CLI interface of WinSCP:
|
||||
|
||||
|
||||
@@ -90,6 +90,7 @@ The following feature flags are available in v0.8.0:
|
||||
| `kubernetes-discovery` | optional | Enable automatic registration and discovery<br>of cluster nodes through the Kubernetes API |
|
||||
| `metrics` | *by default* | Enable collection of metrics in Prometheus format on the admin API |
|
||||
| `telemetry-otlp` | optional | Enable collection of execution traces using OpenTelemetry |
|
||||
| `syslog` | optional | Enable logging to Syslog |
|
||||
| `sled` | *by default* | Enable using Sled to store Garage's metadata |
|
||||
| `lmdb` | optional | Enable using LMDB to store Garage's metadata |
|
||||
| `sqlite` | optional | Enable using Sqlite3 to store Garage's metadata |
|
||||
| `lmdb` | *by default* | Enable using LMDB to store Garage's metadata |
|
||||
| `sqlite` | *by default* | Enable using Sqlite3 to store Garage's metadata |
|
||||
|
||||
@@ -27,7 +27,7 @@ To run a real-world deployment, make sure the following conditions are met:
|
||||
[Yggdrasil](https://yggdrasil-network.github.io/) are approaches to consider
|
||||
in addition to building out your own VPN tunneling.
|
||||
|
||||
- This guide will assume you are using Docker containers to deploy Garage on each node.
|
||||
- This guide will assume you are using Docker containers to deploy Garage on each node.
|
||||
Garage can also be run independently, for instance as a [Systemd service](@/documentation/cookbook/systemd.md).
|
||||
You can also use an orchestrator such as Nomad or Kubernetes to automatically manage
|
||||
Docker containers on a fleet of nodes.
|
||||
@@ -53,9 +53,9 @@ to store 2 TB of data in total.
|
||||
|
||||
### Best practices
|
||||
|
||||
- If you have fast dedicated networking between all your nodes, and are planing to store
|
||||
very large files, bump the `block_size` configuration parameter to 10 MB
|
||||
(`block_size = 10485760`).
|
||||
- If you have reasonably fast networking between all your nodes, and are planing to store
|
||||
mostly large files, bump the `block_size` configuration parameter to 10 MB
|
||||
(`block_size = "10M"`).
|
||||
|
||||
- Garage stores its files in two locations: it uses a metadata directory to store frequently-accessed
|
||||
small metadata items, and a data directory to store data blocks of uploaded objects.
|
||||
@@ -68,31 +68,41 @@ to store 2 TB of data in total.
|
||||
EXT4 is not recommended as it has more strict limitations on the number of inodes,
|
||||
which might cause issues with Garage when large numbers of objects are stored.
|
||||
|
||||
- If you only have an HDD and no SSD, it's fine to put your metadata alongside the data
|
||||
on the same drive. Having lots of RAM for your kernel to cache the metadata will
|
||||
help a lot with performance. Make sure to use the LMDB database engine,
|
||||
instead of Sled, which suffers from quite bad performance degradation on HDDs.
|
||||
Sled is still the default for legacy reasons, but is not recommended anymore.
|
||||
|
||||
- For the metadata storage, Garage does not do checksumming and integrity
|
||||
verification on its own. If you are afraid of bitrot/data corruption,
|
||||
put your metadata directory on a ZFS or BTRFS partition. Otherwise, just use regular
|
||||
EXT4 or XFS.
|
||||
|
||||
- Servers with multiple HDDs are supported natively by Garage without resorting
|
||||
to RAID, see [our dedicated documentation page](@/documentation/operations/multi-hdd.md).
|
||||
|
||||
- For the metadata storage, Garage does not do checksumming and integrity
|
||||
verification on its own, so it is better to use a robust filesystem such as
|
||||
BTRFS or ZFS. Users have reported that when using the LMDB database engine
|
||||
(the default), database files have a tendency of becoming corrupted after an
|
||||
unclean shutdown (e.g. a power outage), so you should take regular snapshots
|
||||
to be able to recover from such a situation. This can be done using Garage's
|
||||
built-in automatic snapshotting (since v0.9.4), or by using filesystem level
|
||||
snapshots. If you cannot do so, you might want to switch to Sqlite which is
|
||||
more robust.
|
||||
|
||||
- LMDB is the fastest and most tested database engine, but it has the following
|
||||
weaknesses: 1/ data files are not architecture-independent, you cannot simply
|
||||
move a Garage metadata directory between nodes running different architectures,
|
||||
and 2/ LMDB is not suited for 32-bit platforms. Sqlite is a viable alternative
|
||||
if any of these are of concern.
|
||||
|
||||
- If you only have an HDD and no SSD, it's fine to put your metadata alongside
|
||||
the data on the same drive, but then consider your filesystem choice wisely
|
||||
(see above). Having lots of RAM for your kernel to cache the metadata will
|
||||
help a lot with performance.
|
||||
|
||||
## Get a Docker image
|
||||
|
||||
Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
|
||||
We encourage you to use a fixed tag (eg. `v0.9.1`) and not the `latest` tag.
|
||||
For this example, we will use the latest published version at the time of the writing which is `v0.9.1` but it's up to you
|
||||
We encourage you to use a fixed tag (eg. `v0.9.4`) and not the `latest` tag.
|
||||
For this example, we will use the latest published version at the time of the writing which is `v0.9.4` but it's up to you
|
||||
to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
|
||||
|
||||
For example:
|
||||
|
||||
```
|
||||
sudo docker pull dxflrs/garage:v0.9.1
|
||||
sudo docker pull dxflrs/garage:v0.9.4
|
||||
```
|
||||
|
||||
## Deploying and configuring Garage
|
||||
@@ -115,6 +125,7 @@ A valid `/etc/garage.toml` for our cluster would look as follows:
|
||||
metadata_dir = "/var/lib/garage/meta"
|
||||
data_dir = "/var/lib/garage/data"
|
||||
db_engine = "lmdb"
|
||||
metadata_auto_snapshot_interval = "6h"
|
||||
|
||||
replication_mode = "3"
|
||||
|
||||
@@ -157,7 +168,7 @@ docker run \
|
||||
-v /etc/garage.toml:/etc/garage.toml \
|
||||
-v /var/lib/garage/meta:/var/lib/garage/meta \
|
||||
-v /var/lib/garage/data:/var/lib/garage/data \
|
||||
dxflrs/garage:v0.9.1
|
||||
dxflrs/garage:v0.9.4
|
||||
```
|
||||
|
||||
With this command line, Garage should be started automatically at each boot.
|
||||
@@ -171,7 +182,7 @@ If you want to use `docker-compose`, you may use the following `docker-compose.y
|
||||
version: "3"
|
||||
services:
|
||||
garage:
|
||||
image: dxflrs/garage:v0.9.1
|
||||
image: dxflrs/garage:v0.9.4
|
||||
network_mode: "host"
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
@@ -187,7 +198,7 @@ upgrades. With the containerized setup proposed here, the upgrade process
|
||||
will require stopping and removing the existing container, and re-creating it
|
||||
with the upgraded version.
|
||||
|
||||
## Controling the daemon
|
||||
## Controlling the daemon
|
||||
|
||||
The `garage` binary has two purposes:
|
||||
- it acts as a daemon when launched with `garage server`
|
||||
@@ -245,7 +256,7 @@ You can then instruct nodes to connect to one another as follows:
|
||||
Venus$ garage node connect 563e1ac825ee3323aa441e72c26d1030d6d4414aeb3dd25287c531e7fc2bc95d@[fc00:1::1]:3901
|
||||
```
|
||||
|
||||
You don't nead to instruct all node to connect to all other nodes:
|
||||
You don't need to instruct all node to connect to all other nodes:
|
||||
nodes will discover one another transitively.
|
||||
|
||||
Now if your run `garage status` on any node, you should have an output that looks as follows:
|
||||
@@ -328,8 +339,8 @@ Given the information above, we will configure our cluster as follow:
|
||||
```bash
|
||||
garage layout assign 563e -z par1 -c 1T -t mercury
|
||||
garage layout assign 86f0 -z par1 -c 2T -t venus
|
||||
garage layout assign 6814 -z lon1 -c 2T -t earth
|
||||
garage layout assign 212f -z bru1 -c 1.5T -t mars
|
||||
garage layout assign 6814 -z lon1 -c 2T -t earth
|
||||
garage layout assign 212f -z bru1 -c 1.5T -t mars
|
||||
```
|
||||
|
||||
At this point, the changes in the cluster layout have not yet been applied.
|
||||
|
||||
@@ -19,7 +19,7 @@ connecting to. To run on all nodes, add the `-a` flag as follows:
|
||||
|
||||
# Data block operations
|
||||
|
||||
## Data store scrub
|
||||
## Data store scrub {#scrub}
|
||||
|
||||
Scrubbing the data store means examining each individual data block to check that
|
||||
their content is correct, by verifying their hash. Any block found to be corrupted
|
||||
@@ -104,6 +104,24 @@ operation will also move out all data from locations marked as read-only.
|
||||
|
||||
# Metadata operations
|
||||
|
||||
## Metadata snapshotting
|
||||
|
||||
It is good practice to setup automatic snapshotting of your metadata database
|
||||
file, to recover from situations where it becomes corrupted on disk. This can
|
||||
be done at the filesystem level if you are using ZFS or BTRFS.
|
||||
|
||||
Since Garage v0.9.4, Garage is able to take snapshots of the metadata database
|
||||
itself. This basically amounts to copying the database file, except that it can
|
||||
be run live while Garage is running without the risk of corruption or
|
||||
inconsistencies. This can be setup to run automatically on a schedule using
|
||||
[`metadata_auto_snapshot_interval`](@/documentation/reference-manual/configuration.md#metadata_auto_snapshot_interval).
|
||||
A snapshot can also be triggered manually using the `garage meta snapshot`
|
||||
command. Note that taking a snapshot using this method is very intensive as it
|
||||
requires making a full copy of the database file, so you might prefer using
|
||||
filesystem-level snapshots if possible. To recover a corrupted node from such a
|
||||
snapshot, read the instructions
|
||||
[here](@/documentation/operations/recovering.md#corrupted_meta).
|
||||
|
||||
## Metadata table resync
|
||||
|
||||
Garage automatically resyncs all entries stored in the metadata tables every hour,
|
||||
|
||||
@@ -108,3 +108,57 @@ garage layout apply # once satisfied, apply the changes
|
||||
|
||||
Garage will then start synchronizing all required data on the new node.
|
||||
This process can be monitored using the `garage stats -a` command.
|
||||
|
||||
## Replacement scenario 3: corrupted metadata {#corrupted_meta}
|
||||
|
||||
In some cases, your metadata DB file might become corrupted, for instance if
|
||||
your node suffered a power outage and did not shut down properly. In this case,
|
||||
you can recover without having to change the node ID and rebuilding a cluster
|
||||
layout. This means that data blocks will not need to be shuffled around, you
|
||||
must simply find a way to repair the metadata file. The best way is generally
|
||||
to discard the corrupted file and recover it from another source.
|
||||
|
||||
First of all, start by locating the database file in your metadata directory,
|
||||
which [depends on your `db_engine`
|
||||
choice](@/documentation/reference-manual/configuration.md#db_engine). Then,
|
||||
your recovery options are as follows:
|
||||
|
||||
- **Option 1: resyncing from other nodes.** In case your cluster is replicated
|
||||
with two or three copies, you can simply delete the database file, and Garage
|
||||
will resync from other nodes. To do so, stop Garage, delete the database file
|
||||
or directory, and restart Garage. Then, do a full table repair by calling
|
||||
`garage repair -a --yes tables`. This will take a bit of time to complete as
|
||||
the new node will need to receive copies of the metadata tables from the
|
||||
network.
|
||||
|
||||
- **Option 2: restoring a snapshot taken by Garage.** Since v0.9.4, Garage can
|
||||
[automatically take regular
|
||||
snapshots](@/documentation/reference-manual/configuration.md#metadata_auto_snapshot_interval)
|
||||
of your metadata DB file. This file or directory should be located under
|
||||
`<metadata_dir>/snapshots`, and is named according to the UTC time at which it
|
||||
was taken. Stop Garage, discard the database file/directory and replace it by the
|
||||
snapshot you want to use. For instance, in the case of LMDB:
|
||||
|
||||
```bash
|
||||
cd $METADATA_DIR
|
||||
mv db.lmdb db.lmdb.bak
|
||||
cp -r snapshots/2024-03-15T12:13:52Z db.lmdb
|
||||
```
|
||||
|
||||
And for Sqlite:
|
||||
|
||||
```bash
|
||||
cd $METADATA_DIR
|
||||
mv db.sqlite db.sqlite.bak
|
||||
cp snapshots/2024-03-15T12:13:52Z db.sqlite
|
||||
```
|
||||
|
||||
Then, restart Garage and run a full table repair by calling `garage repair -a
|
||||
--yes tables`. This should run relatively fast as only the changes that
|
||||
occurred since the snapshot was taken will need to be resynchronized. Of
|
||||
course, if your cluster is not replicated, you will lose all changes that
|
||||
occurred since the snapshot was taken.
|
||||
|
||||
- **Option 3: restoring a filesystem-level snapshot.** If you are using ZFS or
|
||||
BTRFS to snapshot your metadata partition, refer to their specific
|
||||
documentation on rolling back or copying files from an old snapshot.
|
||||
|
||||
@@ -73,6 +73,18 @@ The entire procedure would look something like this:
|
||||
You can do all of the nodes in a single zone at once as that won't impact global cluster availability.
|
||||
Do not try to make a backup of the metadata folder of a running node.
|
||||
|
||||
**Since Garage v0.9.4,** you can use the `garage meta snapshot --all` command
|
||||
to take a simultaneous snapshot of the metadata database files of all your
|
||||
nodes. This avoids the tedious process of having to take them down one by
|
||||
one before upgrading. Be careful that if automatic snapshotting is enabled,
|
||||
Garage only keeps the last two snapshots and deletes older ones, so you might
|
||||
want to disable automatic snapshotting in your upgraded configuration file
|
||||
until you have confirmed that the upgrade ran successfully. In addition to
|
||||
snapshotting the metadata databases of your nodes, you should back-up at
|
||||
least the `cluster_layout` file of one of your Garage instances (this file
|
||||
should be the same on all nodes and you can copy it safely while Garage is
|
||||
running).
|
||||
|
||||
3. Prepare your binaries and configuration files for the new Garage version
|
||||
|
||||
4. Restart all nodes simultaneously in the new version
|
||||
|
||||
@@ -57,7 +57,7 @@ to generate unique and private secrets for security reasons:
|
||||
cat > garage.toml <<EOF
|
||||
metadata_dir = "/tmp/meta"
|
||||
data_dir = "/tmp/data"
|
||||
db_engine = "lmdb"
|
||||
db_engine = "sqlite"
|
||||
|
||||
replication_mode = "none"
|
||||
|
||||
|
||||
@@ -14,10 +14,13 @@ metadata_dir = "/var/lib/garage/meta"
|
||||
data_dir = "/var/lib/garage/data"
|
||||
metadata_fsync = true
|
||||
data_fsync = false
|
||||
disable_scrub = false
|
||||
metadata_auto_snapshot_interval = "6h"
|
||||
|
||||
db_engine = "lmdb"
|
||||
|
||||
block_size = "1M"
|
||||
block_ram_buffer_max = "256MiB"
|
||||
|
||||
sled_cache_capacity = "128MiB"
|
||||
sled_flush_every_ms = 2000
|
||||
@@ -30,6 +33,8 @@ rpc_bind_addr = "[::]:3901"
|
||||
rpc_bind_outgoing = false
|
||||
rpc_public_addr = "[fc00:1::1]:3901"
|
||||
|
||||
allow_world_readable_secrets = false
|
||||
|
||||
bootstrap_peers = [
|
||||
"563e1ac825ee3323aa441e72c26d1030d6d4414aeb3dd25287c531e7fc2bc95d@[fc00:1::1]:3901",
|
||||
"86f0f26ae4afbd59aaf9cfb059eefac844951efd5b8caeec0d53f4ed6c85f332@[fc00:1::2]:3901",
|
||||
@@ -80,14 +85,20 @@ The following gives details about each available configuration option.
|
||||
|
||||
### Index
|
||||
|
||||
[Environment variables](#env_variables).
|
||||
|
||||
Top-level configuration options:
|
||||
[`allow_world_readable_secrets`](#allow_world_readable_secrets),
|
||||
[`block_ram_buffer_max`](#block_ram_buffer_max),
|
||||
[`block_size`](#block_size),
|
||||
[`bootstrap_peers`](#bootstrap_peers),
|
||||
[`compression_level`](#compression_level),
|
||||
[`data_dir`](#data_dir),
|
||||
[`data_fsync`](#data_fsync),
|
||||
[`db_engine`](#db_engine),
|
||||
[`disable_scrub`](#disable_scrub),
|
||||
[`lmdb_map_size`](#lmdb_map_size),
|
||||
[`metadata_auto_snapshot_interval`](#metadata_auto_snapshot_interval),
|
||||
[`metadata_dir`](#metadata_dir),
|
||||
[`metadata_fsync`](#metadata_fsync),
|
||||
[`replication_mode`](#replication_mode),
|
||||
@@ -130,6 +141,23 @@ The `[admin]` section:
|
||||
[`admin_token`/`admin_token_file`](#admin_token),
|
||||
[`trace_sink`](#admin_trace_sink),
|
||||
|
||||
### Environment variables {#env_variables}
|
||||
|
||||
The following configuration parameter must be specified as an environment
|
||||
variable, it does not exist in the configuration file:
|
||||
|
||||
- `GARAGE_LOG_TO_SYSLOG` (since v0.9.4): set this to `1` or `true` to make the
|
||||
Garage daemon send its logs to `syslog` (using the libc `syslog` function)
|
||||
instead of printing to stderr.
|
||||
|
||||
The following environment variables can be used to override the corresponding
|
||||
values in the configuration file:
|
||||
|
||||
- [`GARAGE_ALLOW_WORLD_READABLE_SECRETS`](#allow_world_readable_secrets)
|
||||
- [`GARAGE_RPC_SECRET` and `GARAGE_RPC_SECRET_FILE`](#rpc_secret)
|
||||
- [`GARAGE_ADMIN_TOKEN` and `GARAGE_ADMIN_TOKEN_FILE`](#admin_token)
|
||||
- [`GARAGE_METRICS_TOKEN` and `GARAGE_METRICS_TOKEN`](#admin_metrics_token)
|
||||
|
||||
|
||||
### Top-level configuration options
|
||||
|
||||
@@ -264,18 +292,31 @@ Performance characteristics of the different DB engines are as follows:
|
||||
- Sled: tends to produce large data files and also has performance issues,
|
||||
especially when the metadata folder is on a traditional HDD and not on SSD.
|
||||
|
||||
- LMDB: the recommended database engine on 64-bit systems, much more
|
||||
space-efficient and slightly faster. Note that the data format of LMDB is not
|
||||
portable between architectures, so for instance the Garage database of an
|
||||
x86-64 node cannot be moved to an ARM64 node. Also note that, while LMDB can
|
||||
technically be used on 32-bit systems, this will limit your node to very
|
||||
small database sizes due to how LMDB works; it is therefore not recommended.
|
||||
- LMDB: the recommended database engine for high-performance distributed
|
||||
clusters, much more space-efficient and significantly faster. LMDB works very
|
||||
well, but is known to have the following limitations:
|
||||
|
||||
- The data format of LMDB is not portable between architectures, so for
|
||||
instance the Garage database of an x86-64 node cannot be moved to an ARM64
|
||||
node.
|
||||
|
||||
- While LMDB can technically be used on 32-bit systems, this will limit your
|
||||
node to very small database sizes due to how LMDB works; it is therefore
|
||||
not recommended.
|
||||
|
||||
- Several users have reported corrupted LMDB database files after an unclean
|
||||
shutdown (e.g. a power outage). This situation can generally be recovered
|
||||
from if your cluster is geo-replicated (by rebuilding your metadata db from
|
||||
other nodes), or if you have saved regular snapshots at the filesystem
|
||||
level.
|
||||
|
||||
- Sqlite: Garage supports Sqlite as an alternative storage backend for
|
||||
metadata, and although it has not been tested as much, it is expected to work
|
||||
satisfactorily. Since Garage v0.9.0, performance issues have largely been
|
||||
fixed by allowing for a no-fsync mode (see `metadata_fsync`). Sqlite does not
|
||||
have the database size limitation of LMDB on 32-bit systems.
|
||||
metadata, which does not have the issues listed above for LMDB.
|
||||
On versions 0.8.x and earlier, Sqlite should be avoided due to abysmal
|
||||
performance, which was fixed with the addition of `metadata_fsync`.
|
||||
Sqlite is still probably slower than LMDB due to the way we use it,
|
||||
so it is not the best choice for high-performance storage clusters,
|
||||
but it should work fine in many cases.
|
||||
|
||||
It is possible to convert Garage's metadata directory from one format to another
|
||||
using the `garage convert-db` command, which should be used as follows:
|
||||
@@ -302,7 +343,7 @@ Using this option reduces the risk of simultaneous metadata corruption on severa
|
||||
cluster nodes, which could lead to data loss.
|
||||
|
||||
If multi-site replication is used, this option is most likely not necessary, as
|
||||
it is extremely unlikely that two nodes in different locations will have a
|
||||
it is extremely unlikely that two nodes in different locations will have a
|
||||
power failure at the exact same time.
|
||||
|
||||
(Metadata corruption on a single node is not an issue, the corrupted data file
|
||||
@@ -331,6 +372,43 @@ at the cost of a moderate drop in write performance.
|
||||
Similarly to `metatada_fsync`, this is likely not necessary
|
||||
if geographical replication is used.
|
||||
|
||||
#### `metadata_auto_snapshot_interval` (since Garage v0.9.4) {#metadata_auto_snapshot_interval}
|
||||
|
||||
If this value is set, Garage will automatically take a snapshot of the metadata
|
||||
DB file at a regular interval and save it in the metadata directory.
|
||||
This parameter can take any duration string that can be parsed by
|
||||
the [`parse_duration`](https://docs.rs/parse_duration/latest/parse_duration/#syntax) crate.
|
||||
|
||||
Snapshots can allow to recover from situations where the metadata DB file is
|
||||
corrupted, for instance after an unclean shutdown. See [this
|
||||
page](@/documentation/operations/recovering.md#corrupted_meta) for details.
|
||||
Garage keeps only the two most recent snapshots of the metadata DB and deletes
|
||||
older ones automatically.
|
||||
|
||||
Note that taking a metadata snapshot is a relatively intensive operation as the
|
||||
entire data file is copied. A snapshot being taken might have performance
|
||||
impacts on the Garage node while it is running. If the cluster is under heavy
|
||||
write load when a snapshot operation is running, this might also cause the
|
||||
database file to grow in size significantly as pages cannot be recycled easily.
|
||||
For this reason, it might be better to use filesystem-level snapshots instead
|
||||
if possible.
|
||||
|
||||
#### `disable_scrub` {#disable_scrub}
|
||||
|
||||
By default, Garage runs a scrub of the data directory approximately once per
|
||||
month, with a random delay to avoid all nodes running at the same time. When
|
||||
it scrubs the data directory, Garage will read all of the data files stored on
|
||||
disk to check their integrity, and will rebuild any data files that it finds
|
||||
corrupted, using the remaining valid copies stored on other nodes.
|
||||
See [this page](@/documentation/operations/durability-repair.md#scrub) for details.
|
||||
|
||||
Set the `disable_scrub` configuration value to `true` if you don't need Garage
|
||||
to scrub the data directory, for instance if you are already scrubbing at the
|
||||
filesystem level. Note that in this case, if you find a corrupted data file,
|
||||
you should delete it from the data directory and then call `garage repair
|
||||
blocks` on the node to ensure that it re-obtains a copy from another node on
|
||||
the network.
|
||||
|
||||
#### `block_size` {#block_size}
|
||||
|
||||
Garage splits stored objects in consecutive chunks of size `block_size`
|
||||
@@ -346,6 +424,37 @@ files will remain available. This however means that chunks from existing files
|
||||
will not be deduplicated with chunks from newly uploaded files, meaning you
|
||||
might use more storage space that is optimally possible.
|
||||
|
||||
#### `block_ram_buffer_max` (since v0.9.4) {#block_ram_buffer_max}
|
||||
|
||||
A limit on the total size of data blocks kept in RAM by S3 API nodes awaiting
|
||||
to be sent to storage nodes asynchronously.
|
||||
|
||||
Explanation: since Garage wants to tolerate node failures, it uses quorum
|
||||
writes to send data blocks to storage nodes: try to write the block to three
|
||||
nodes, and return ok as soon as two writes complete. So even if all three nodes
|
||||
are online, the third write always completes asynchronously. In general, there
|
||||
are not many writes to a cluster, and the third asynchronous write can
|
||||
terminate early enough so as to not cause unbounded RAM growth. However, if
|
||||
the S3 API node is continuously receiving large quantities of data and the
|
||||
third node is never able to catch up, many data blocks will be kept buffered in
|
||||
RAM as they are awaiting transfer to the third node.
|
||||
|
||||
The `block_ram_buffer_max` sets a limit to the size of buffers that can be kept
|
||||
in RAM in this process. When the limit is reached, backpressure is applied
|
||||
back to the S3 client.
|
||||
|
||||
Note that this only counts buffers that have arrived to a certain stage of
|
||||
processing (received from the client + encrypted and/or compressed as
|
||||
necessary) and are ready to send to the storage nodes. Many other buffers will
|
||||
not be counted and this is not a hard limit on RAM consumption. In particular,
|
||||
if many clients send requests simultaneously with large objects, the RAM
|
||||
consumption will always grow linearly with the number of concurrent requests,
|
||||
as each request will use a few buffers of size `block_size` for receiving and
|
||||
intermediate processing before even trying to send the data to the storage
|
||||
node.
|
||||
|
||||
The default value is 256MiB.
|
||||
|
||||
#### `sled_cache_capacity` {#sled_cache_capacity}
|
||||
|
||||
This parameter can be used to tune the capacity of the cache used by
|
||||
@@ -451,7 +560,7 @@ be obtained by running `garage node id` and then included directly in the
|
||||
key will be returned by `garage node id` and you will have to add the IP
|
||||
yourself.
|
||||
|
||||
### `allow_world_readable_secrets`
|
||||
### `allow_world_readable_secrets` or `GARAGE_ALLOW_WORLD_READABLE_SECRETS` (env) {#allow_world_readable_secrets}
|
||||
|
||||
Garage checks the permissions of your secret files to make sure they're not
|
||||
world-readable. In some cases, the check might fail and consider your files as
|
||||
|
||||
@@ -225,6 +225,17 @@ block_bytes_read 120586322022
|
||||
block_bytes_written 3386618077
|
||||
```
|
||||
|
||||
#### `block_ram_buffer_free_kb` (gauge)
|
||||
|
||||
Kibibytes available for buffering blocks that have to be sent to remote nodes.
|
||||
When clients send too much data to this node and a storage node is not receiving
|
||||
data fast enough due to slower network conditions, this will decrease down to
|
||||
zero and backpressure will be applied.
|
||||
|
||||
```
|
||||
block_ram_buffer_free_kb 219829
|
||||
```
|
||||
|
||||
#### `block_compression_level` (counter)
|
||||
|
||||
Exposes the block compression level configured for the Garage node.
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
*
|
||||
|
||||
!*.txt
|
||||
!*.md
|
||||
|
||||
!assets
|
||||
|
||||
!.gitignore
|
||||
!*.svg
|
||||
!*.png
|
||||
!*.jpg
|
||||
!*.tex
|
||||
!Makefile
|
||||
!.gitignore
|
||||
!assets/*.drawio.pdf
|
||||
|
||||
!talk.pdf
|
||||
@@ -0,0 +1,10 @@
|
||||
ASSETS=../assets/logos/deuxfleurs.pdf
|
||||
|
||||
talk.pdf: talk.tex $(ASSETS)
|
||||
pdflatex talk.tex
|
||||
|
||||
%.pdf: %.svg
|
||||
inkscape -D -z --file=$^ --export-pdf=$@
|
||||
|
||||
%.pdf_tex: %.svg
|
||||
inkscape -D -z --file=$^ --export-pdf=$@ --export-latex
|
||||
Binary file not shown.
@@ -0,0 +1,543 @@
|
||||
\nonstopmode
|
||||
\documentclass[aspectratio=169,xcolor={svgnames}]{beamer}
|
||||
\usepackage[utf8]{inputenc}
|
||||
% \usepackage[frenchb]{babel}
|
||||
\usepackage{amsmath}
|
||||
\usepackage{mathtools}
|
||||
\usepackage{breqn}
|
||||
\usepackage{multirow}
|
||||
\usetheme{boxes}
|
||||
\usepackage{graphicx}
|
||||
\usepackage{import}
|
||||
\usepackage{adjustbox}
|
||||
\usepackage[absolute,overlay]{textpos}
|
||||
%\useoutertheme[footline=authortitle,subsection=false]{miniframes}
|
||||
%\useoutertheme[footline=authorinstitute,subsection=false]{miniframes}
|
||||
\useoutertheme{infolines}
|
||||
\setbeamertemplate{headline}{}
|
||||
|
||||
\beamertemplatenavigationsymbolsempty
|
||||
|
||||
\definecolor{TitleOrange}{RGB}{255,137,0}
|
||||
\setbeamercolor{title}{fg=TitleOrange}
|
||||
\setbeamercolor{frametitle}{fg=TitleOrange}
|
||||
|
||||
\definecolor{ListOrange}{RGB}{255,145,5}
|
||||
\setbeamertemplate{itemize item}{\color{ListOrange}$\blacktriangleright$}
|
||||
|
||||
\definecolor{verygrey}{RGB}{70,70,70}
|
||||
\setbeamercolor{normal text}{fg=verygrey}
|
||||
|
||||
|
||||
\usepackage{tabu}
|
||||
\usepackage{multicol}
|
||||
\usepackage{vwcol}
|
||||
\usepackage{stmaryrd}
|
||||
\usepackage{graphicx}
|
||||
|
||||
\usepackage[normalem]{ulem}
|
||||
|
||||
\AtBeginSection[]{
|
||||
\begin{frame}
|
||||
\vfill
|
||||
\centering
|
||||
\begin{beamercolorbox}[sep=8pt,center,shadow=true,rounded=true]{title}
|
||||
\usebeamerfont{title}\insertsectionhead\par%
|
||||
\end{beamercolorbox}
|
||||
\vfill
|
||||
\end{frame}
|
||||
}
|
||||
|
||||
\title{Garage}
|
||||
\author{Alex Auvolat, Deuxfleurs}
|
||||
\date{Capitoul, 2024-02-29}
|
||||
|
||||
\begin{document}
|
||||
|
||||
\begin{frame}
|
||||
\centering
|
||||
\includegraphics[width=.3\linewidth]{../../sticker/Garage.png}
|
||||
\vspace{1em}
|
||||
|
||||
{\large\bf Alex Auvolat, Deuxfleurs Association}
|
||||
\vspace{1em}
|
||||
|
||||
\url{https://garagehq.deuxfleurs.fr/}
|
||||
|
||||
Matrix channel: \texttt{\#garage:deuxfleurs.fr}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Who I am}
|
||||
\begin{columns}[t]
|
||||
\begin{column}{.2\textwidth}
|
||||
\centering
|
||||
\adjincludegraphics[width=.4\linewidth, valign=t]{../assets/alex.jpg}
|
||||
\end{column}
|
||||
\begin{column}{.6\textwidth}
|
||||
\textbf{Alex Auvolat}\\
|
||||
PhD; co-founder of Deuxfleurs
|
||||
\end{column}
|
||||
\begin{column}{.2\textwidth}
|
||||
~
|
||||
\end{column}
|
||||
\end{columns}
|
||||
\vspace{2em}
|
||||
|
||||
\begin{columns}[t]
|
||||
\begin{column}{.2\textwidth}
|
||||
\centering
|
||||
\adjincludegraphics[width=.5\linewidth, valign=t]{../assets/logos/deuxfleurs.pdf}
|
||||
\end{column}
|
||||
\begin{column}{.6\textwidth}
|
||||
\textbf{Deuxfleurs}\\
|
||||
A non-profit self-hosting collective,\\
|
||||
member of the CHATONS network
|
||||
\end{column}
|
||||
\begin{column}{.2\textwidth}
|
||||
\centering
|
||||
\adjincludegraphics[width=.7\linewidth, valign=t]{../assets/logos/logo_chatons.png}
|
||||
\end{column}
|
||||
\end{columns}
|
||||
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Our objective at Deuxfleurs}
|
||||
|
||||
\begin{center}
|
||||
\textbf{Promote self-hosting and small-scale hosting\\
|
||||
as an alternative to large cloud providers}
|
||||
\end{center}
|
||||
\vspace{2em}
|
||||
\visible<2->{
|
||||
Why is it hard?
|
||||
\vspace{2em}
|
||||
\begin{center}
|
||||
\textbf{\underline{Resilience}}\\
|
||||
{\footnotesize we want good uptime/availability with low supervision}
|
||||
\end{center}
|
||||
}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Our very low-tech infrastructure}
|
||||
|
||||
\only<1,3-6>{
|
||||
\begin{itemize}
|
||||
\item \textcolor<4->{gray}{Commodity hardware (e.g. old desktop PCs)\\
|
||||
\vspace{.5em}
|
||||
\visible<3->{{\footnotesize (can die at any time)}}}
|
||||
\vspace{1.5em}
|
||||
\item<4-> \textcolor<6->{gray}{Regular Internet (e.g. FTTB, FTTH) and power grid connections\\
|
||||
\vspace{.5em}
|
||||
\visible<5->{{\footnotesize (can be unavailable randomly)}}}
|
||||
\vspace{1.5em}
|
||||
\item<6-> \textbf{Geographical redundancy} (multi-site replication)
|
||||
\end{itemize}
|
||||
}
|
||||
\only<2>{
|
||||
\begin{center}
|
||||
\includegraphics[width=.8\linewidth]{../assets/neptune.jpg}
|
||||
\end{center}
|
||||
}
|
||||
\only<7>{
|
||||
\begin{center}
|
||||
\includegraphics[width=.8\linewidth]{../assets/inframap_jdll2023.pdf}
|
||||
\end{center}
|
||||
}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{How to make this happen}
|
||||
\begin{center}
|
||||
\only<1>{\includegraphics[width=.8\linewidth]{../assets/intro/slide1.png}}%
|
||||
\only<2>{\includegraphics[width=.8\linewidth]{../assets/intro/slide2.png}}%
|
||||
\only<3>{\includegraphics[width=.8\linewidth]{../assets/intro/slide3.png}}%
|
||||
\end{center}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Distributed file systems are slow}
|
||||
File systems are complex, for example:
|
||||
\vspace{1em}
|
||||
\begin{itemize}
|
||||
\item Concurrent modification by several processes
|
||||
\vspace{1em}
|
||||
\item Folder hierarchies
|
||||
\vspace{1em}
|
||||
\item Other requirements of the POSIX spec (e.g.~locks)
|
||||
\end{itemize}
|
||||
\vspace{1em}
|
||||
Coordination in a distributed system is costly
|
||||
|
||||
\vspace{1em}
|
||||
Costs explode with commodity hardware / Internet connections\\
|
||||
{\small (we experienced this!)}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{A simpler solution: object storage}
|
||||
Only two operations:
|
||||
\vspace{1em}
|
||||
\begin{itemize}
|
||||
\item Put an object at a key
|
||||
\vspace{1em}
|
||||
\item Retrieve an object from its key
|
||||
\end{itemize}
|
||||
\vspace{1em}
|
||||
{\footnotesize (and a few others)}
|
||||
|
||||
\vspace{1em}
|
||||
Sufficient for many applications!
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{A simpler solution: object storage}
|
||||
\begin{center}
|
||||
\includegraphics[height=6em]{../assets/logos/Amazon-S3.jpg}
|
||||
\hspace{3em}
|
||||
\visible<2->{\includegraphics[height=5em]{../assets/logos/minio.png}}
|
||||
\hspace{3em}
|
||||
\visible<3>{\includegraphics[height=6em]{../../logo/garage_hires_crop.png}}
|
||||
\end{center}
|
||||
\vspace{1em}
|
||||
S3: a de-facto standard, many compatible applications
|
||||
|
||||
\vspace{1em}
|
||||
\visible<2->{MinIO is self-hostable but not suited for geo-distributed deployments}
|
||||
|
||||
\vspace{1em}
|
||||
\visible<3->{\textbf{Garage is a self-hosted drop-in replacement for the Amazon S3 object store}}
|
||||
\end{frame}
|
||||
|
||||
% --------- BASED ON CRDTS ----------
|
||||
|
||||
\section{Principle 1: based on CRDTs}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{CRDTs / weak consistency instead of consensus}
|
||||
|
||||
\underline{Internally, Garage uses only CRDTs} (conflict-free replicated data types)
|
||||
|
||||
\vspace{2em}
|
||||
Why not Raft, Paxos, ...? Issues of consensus algorithms:
|
||||
|
||||
\vspace{1em}
|
||||
\begin{itemize}
|
||||
\item<2-> \textbf{Software complexity}
|
||||
\vspace{1em}
|
||||
\item<3-> \textbf{Performance issues:}
|
||||
\vspace{.5em}
|
||||
\begin{itemize}
|
||||
\item<4-> The leader is a \textbf{bottleneck} for all requests\\
|
||||
\vspace{.5em}
|
||||
\item<5-> \textbf{Sensitive to higher latency} between nodes
|
||||
\vspace{.5em}
|
||||
\item<6-> \textbf{Takes time to reconverge} when disrupted (e.g. node going down)
|
||||
\end{itemize}
|
||||
\end{itemize}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{The data model of object storage}
|
||||
Object storage is basically a \textbf{key-value store}:
|
||||
\vspace{.5em}
|
||||
|
||||
{\scriptsize
|
||||
\begin{center}
|
||||
\begin{tabular}{|l|p{7cm}|}
|
||||
\hline
|
||||
\textbf{Key: file path + name} & \textbf{Value: file data + metadata} \\
|
||||
\hline
|
||||
\hline
|
||||
\texttt{index.html} &
|
||||
\texttt{Content-Type: text/html; charset=utf-8} \newline
|
||||
\texttt{Content-Length: 24929} \newline
|
||||
\texttt{<binary blob>} \\
|
||||
\hline
|
||||
\texttt{img/logo.svg} &
|
||||
\texttt{Content-Type: text/svg+xml} \newline
|
||||
\texttt{Content-Length: 13429} \newline
|
||||
\texttt{<binary blob>} \\
|
||||
\hline
|
||||
\texttt{download/index.html} &
|
||||
\texttt{Content-Type: text/html; charset=utf-8} \newline
|
||||
\texttt{Content-Length: 26563} \newline
|
||||
\texttt{<binary blob>} \\
|
||||
\hline
|
||||
\end{tabular}
|
||||
\end{center}
|
||||
}
|
||||
|
||||
\vspace{.5em}
|
||||
\begin{itemize}
|
||||
\item<2-> Maps well to CRDT data types
|
||||
\item<3> Read-after-write consistency with quorums
|
||||
\end{itemize}
|
||||
\end{frame}
|
||||
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Performance gains in practice}
|
||||
\begin{center}
|
||||
\includegraphics[width=.8\linewidth]{../assets/perf/endpoint_latency_0.7_0.8_minio.png}
|
||||
\end{center}
|
||||
\end{frame}
|
||||
|
||||
% --------- GEO-DISTRIBUTED MODEL ----------
|
||||
|
||||
\section{Principle 2: geo-distributed data model}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Key-value stores, upgraded: the Dynamo model}
|
||||
\textbf{Two keys:}
|
||||
\begin{itemize}
|
||||
\item Partition key: used to divide data into partitions {\small (a.k.a.~shards)}
|
||||
\item Sort key: used to identify items inside a partition
|
||||
\end{itemize}
|
||||
|
||||
\vspace{1em}
|
||||
|
||||
\begin{center}
|
||||
\begin{tabular}{|l|l|p{3cm}|}
|
||||
\hline
|
||||
\textbf{Partition key: bucket} & \textbf{Sort key: filename} & \textbf{Value} \\
|
||||
\hline
|
||||
\hline
|
||||
\texttt{website} & \texttt{index.html} & (file data) \\
|
||||
\hline
|
||||
\texttt{website} & \texttt{img/logo.svg} & (file data) \\
|
||||
\hline
|
||||
\texttt{website} & \texttt{download/index.html} & (file data) \\
|
||||
\hline
|
||||
\hline
|
||||
\texttt{backup} & \texttt{borg/index.2822} & (file data) \\
|
||||
\hline
|
||||
\texttt{backup} & \texttt{borg/data/2/2329} & (file data) \\
|
||||
\hline
|
||||
\texttt{backup} & \texttt{borg/data/2/2680} & (file data) \\
|
||||
\hline
|
||||
\hline
|
||||
\texttt{private} & \texttt{qq3a2nbe1qjq0ebbvo6ocsp6co} & (file data) \\
|
||||
\hline
|
||||
\end{tabular}
|
||||
\end{center}
|
||||
\end{frame}
|
||||
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Layout computation}
|
||||
\begin{overprint}
|
||||
\onslide<1>
|
||||
\begin{center}
|
||||
\includegraphics[width=\linewidth, trim=0 0 0 -4cm]{../assets/screenshots/garage_status_0.9_prod_zonehl.png}
|
||||
\end{center}
|
||||
\onslide<2>
|
||||
\begin{center}
|
||||
\includegraphics[width=.7\linewidth]{../assets/map.png}
|
||||
\end{center}
|
||||
\end{overprint}
|
||||
\vspace{1em}
|
||||
Garage stores replicas on different zones when possible
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{What a "layout" is}
|
||||
\textbf{A layout is a precomputed index table:}
|
||||
\vspace{1em}
|
||||
|
||||
{\footnotesize
|
||||
\begin{center}
|
||||
\begin{tabular}{|l|l|l|l|}
|
||||
\hline
|
||||
\textbf{Partition} & \textbf{Node 1} & \textbf{Node 2} & \textbf{Node 3} \\
|
||||
\hline
|
||||
\hline
|
||||
Partition 0 & df-ymk (bespin) & Abricot (scorpio) & Courgette (neptune) \\
|
||||
\hline
|
||||
Partition 1 & Ananas (scorpio) & Courgette (neptune) & df-ykl (bespin) \\
|
||||
\hline
|
||||
Partition 2 & df-ymf (bespin) & Celeri (neptune) & Abricot (scorpio) \\
|
||||
\hline
|
||||
\hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ \\
|
||||
\hline
|
||||
Partition 255 & Concombre (neptune) & df-ykl (bespin) & Abricot (scorpio) \\
|
||||
\hline
|
||||
\end{tabular}
|
||||
\end{center}
|
||||
}
|
||||
|
||||
\vspace{2em}
|
||||
\visible<2->{
|
||||
The index table is built centrally using an optimal algorithm,\\
|
||||
then propagated to all nodes
|
||||
}
|
||||
|
||||
\vspace{1em}
|
||||
\visible<3->{
|
||||
\footnotesize
|
||||
Oulamara, M., \& Auvolat, A. (2023). \emph{An algorithm for geo-distributed and redundant storage in Garage}.\\ arXiv preprint arXiv:2302.13798.
|
||||
}
|
||||
\end{frame}
|
||||
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{The relationship between \emph{partition} and \emph{partition key}}
|
||||
\begin{center}
|
||||
\begin{tabular}{|l|l|l|l|}
|
||||
\hline
|
||||
\textbf{Partition key} & \textbf{Partition} & \textbf{Sort key} & \textbf{Value} \\
|
||||
\hline
|
||||
\hline
|
||||
\texttt{website} & Partition 12 & \texttt{index.html} & (file data) \\
|
||||
\hline
|
||||
\texttt{website} & Partition 12 & \texttt{img/logo.svg} & (file data) \\
|
||||
\hline
|
||||
\texttt{website} & Partition 12 &\texttt{download/index.html} & (file data) \\
|
||||
\hline
|
||||
\hline
|
||||
\texttt{backup} & Partition 42 & \texttt{borg/index.2822} & (file data) \\
|
||||
\hline
|
||||
\texttt{backup} & Partition 42 & \texttt{borg/data/2/2329} & (file data) \\
|
||||
\hline
|
||||
\texttt{backup} & Partition 42 & \texttt{borg/data/2/2680} & (file data) \\
|
||||
\hline
|
||||
\hline
|
||||
\texttt{private} & Partition 42 & \texttt{qq3a2nbe1qjq0ebbvo6ocsp6co} & (file data) \\
|
||||
\hline
|
||||
\end{tabular}
|
||||
\end{center}
|
||||
\vspace{1em}
|
||||
\textbf{To read or write an item:} hash partition key
|
||||
\\ \hspace{5cm} $\to$ determine partition number (first 8 bits)
|
||||
\\ \hspace{5cm} $\to$ find associated nodes
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Garage's internal data structures}
|
||||
\centering
|
||||
\includegraphics[width=.75\columnwidth]{../assets/garage_tables.pdf}
|
||||
\end{frame}
|
||||
|
||||
% ---------- OPERATING GARAGE ---------
|
||||
|
||||
\section{Operating Garage clusters}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Operating Garage}
|
||||
\begin{center}
|
||||
\only<1-2>{
|
||||
\includegraphics[width=.9\linewidth]{../assets/screenshots/garage_status_0.10.png}
|
||||
\\\vspace{1em}
|
||||
\visible<2>{\includegraphics[width=.9\linewidth]{../assets/screenshots/garage_status_unhealthy_0.10.png}}
|
||||
}
|
||||
\end{center}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Background synchronization}
|
||||
\begin{center}
|
||||
\includegraphics[width=.6\linewidth]{../assets/garage_sync.drawio.pdf}
|
||||
\end{center}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Digging deeper}
|
||||
\begin{center}
|
||||
\only<1>{\includegraphics[width=.9\linewidth]{../assets/screenshots/garage_stats_0.10.png}}
|
||||
\only<2>{\includegraphics[width=.5\linewidth]{../assets/screenshots/garage_worker_list_0.10.png}}
|
||||
\only<3>{\includegraphics[width=.6\linewidth]{../assets/screenshots/garage_worker_param_0.10.png}}
|
||||
\end{center}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Monitoring with Prometheus + Grafana}
|
||||
\begin{center}
|
||||
\includegraphics[width=.9\linewidth]{../assets/screenshots/grafana_dashboard.png}
|
||||
\end{center}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Debugging with traces}
|
||||
\begin{center}
|
||||
\includegraphics[width=.8\linewidth]{../assets/screenshots/jaeger_listobjects.png}
|
||||
\end{center}
|
||||
\end{frame}
|
||||
|
||||
% ---------- SCALING GARAGE ---------
|
||||
|
||||
\section{Scaling Garage clusters}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Potential limitations and bottlenecks}
|
||||
\begin{itemize}
|
||||
\item Global:
|
||||
\begin{itemize}
|
||||
\item Max. $\sim$100 nodes per cluster (excluding gateways)
|
||||
\end{itemize}
|
||||
\vspace{1em}
|
||||
\item Metadata:
|
||||
\begin{itemize}
|
||||
\item One big bucket = bottleneck, object list on 3 nodes only
|
||||
\end{itemize}
|
||||
\vspace{1em}
|
||||
\item Block manager:
|
||||
\begin{itemize}
|
||||
\item Lots of small files on disk
|
||||
\item Processing the resync queue can be slow
|
||||
\end{itemize}
|
||||
\end{itemize}
|
||||
\end{frame}
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Deployment advice for very large clusters}
|
||||
\begin{itemize}
|
||||
\item Metadata storage:
|
||||
\begin{itemize}
|
||||
\item ZFS mirror (x2) on fast NVMe
|
||||
\item Use LMDB storage engine
|
||||
\end{itemize}
|
||||
\vspace{.5em}
|
||||
\item Data block storage:
|
||||
\begin{itemize}
|
||||
\item Use Garage's native multi-HDD support
|
||||
\item XFS on individual drives
|
||||
\item Increase block size (1MB $\to$ 10MB, requires more RAM and good networking)
|
||||
\item Tune \texttt{resync-tranquility} and \texttt{resync-worker-count} dynamically
|
||||
\end{itemize}
|
||||
\vspace{.5em}
|
||||
\item Other :
|
||||
\begin{itemize}
|
||||
\item Split data over several buckets
|
||||
\item Use less than 100 storage nodes
|
||||
\item Use gateway nodes
|
||||
\end{itemize}
|
||||
\vspace{.5em}
|
||||
\end{itemize}
|
||||
Our deployments: $< 10$ TB. Some people have done more!
|
||||
\end{frame}
|
||||
|
||||
|
||||
% ======================================== END
|
||||
% ======================================== END
|
||||
% ======================================== END
|
||||
|
||||
\begin{frame}
|
||||
\frametitle{Where to find us}
|
||||
\begin{center}
|
||||
\includegraphics[width=.25\linewidth]{../../logo/garage_hires.png}\\
|
||||
\vspace{-1em}
|
||||
\url{https://garagehq.deuxfleurs.fr/}\\
|
||||
\url{mailto:garagehq@deuxfleurs.fr}\\
|
||||
\texttt{\#garage:deuxfleurs.fr} on Matrix
|
||||
|
||||
\vspace{1.5em}
|
||||
\includegraphics[width=.06\linewidth]{../assets/logos/rust_logo.png}
|
||||
\includegraphics[width=.13\linewidth]{../assets/logos/AGPLv3_Logo.png}
|
||||
\end{center}
|
||||
\end{frame}
|
||||
|
||||
\end{document}
|
||||
|
||||
%% vim: set ts=4 sw=4 tw=0 noet spelllang=en :
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 87 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 81 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 124 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 84 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 81 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 81 KiB |
File diff suppressed because it is too large
Load Diff
|
After Width: | Height: | Size: 315 KiB |
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 286 KiB |
-158
@@ -1,158 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import os
|
||||
import requests
|
||||
from datetime import datetime
|
||||
|
||||
# let's talk to our AWS Elasticsearch cluster
|
||||
#from requests_aws4auth import AWS4Auth
|
||||
#auth = AWS4Auth('GK31c2f218a2e44f485b94239e',
|
||||
# 'b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835',
|
||||
# 'us-east-1',
|
||||
# 's3')
|
||||
|
||||
from aws_requests_auth.aws_auth import AWSRequestsAuth
|
||||
auth = AWSRequestsAuth(aws_access_key='GK31c2f218a2e44f485b94239e',
|
||||
aws_secret_access_key='b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835',
|
||||
aws_host='localhost:3812',
|
||||
aws_region='us-east-1',
|
||||
aws_service='k2v')
|
||||
|
||||
|
||||
print("-- ReadIndex")
|
||||
response = requests.get('http://localhost:3812/alex',
|
||||
auth=auth)
|
||||
print(response.headers)
|
||||
print(response.text)
|
||||
|
||||
|
||||
sort_keys = ["a", "b", "c", "d"]
|
||||
|
||||
for sk in sort_keys:
|
||||
print("-- (%s) Put initial (no CT)"%sk)
|
||||
response = requests.put('http://localhost:3812/alex/root?sort_key=%s'%sk,
|
||||
auth=auth,
|
||||
data='{}: Hello, world!'.format(datetime.timestamp(datetime.now())))
|
||||
print(response.headers)
|
||||
print(response.text)
|
||||
|
||||
print("-- Get")
|
||||
response = requests.get('http://localhost:3812/alex/root?sort_key=%s'%sk,
|
||||
auth=auth)
|
||||
print(response.headers)
|
||||
print(response.text)
|
||||
ct = response.headers["x-garage-causality-token"]
|
||||
|
||||
print("-- ReadIndex")
|
||||
response = requests.get('http://localhost:3812/alex',
|
||||
auth=auth)
|
||||
print(response.headers)
|
||||
print(response.text)
|
||||
|
||||
print("-- Put with CT")
|
||||
response = requests.put('http://localhost:3812/alex/root?sort_key=%s'%sk,
|
||||
auth=auth,
|
||||
headers={'x-garage-causality-token': ct},
|
||||
data='{}: Good bye, world!'.format(datetime.timestamp(datetime.now())))
|
||||
print(response.headers)
|
||||
print(response.text)
|
||||
|
||||
print("-- Get")
|
||||
response = requests.get('http://localhost:3812/alex/root?sort_key=%s'%sk,
|
||||
auth=auth)
|
||||
print(response.headers)
|
||||
print(response.text)
|
||||
|
||||
print("-- Put again with same CT (concurrent)")
|
||||
response = requests.put('http://localhost:3812/alex/root?sort_key=%s'%sk,
|
||||
auth=auth,
|
||||
headers={'x-garage-causality-token': ct},
|
||||
data='{}: Concurrent value, oops'.format(datetime.timestamp(datetime.now())))
|
||||
print(response.headers)
|
||||
print(response.text)
|
||||
|
||||
for sk in sort_keys:
|
||||
print("-- (%s) Get"%sk)
|
||||
response = requests.get('http://localhost:3812/alex/root?sort_key=%s'%sk,
|
||||
auth=auth)
|
||||
print(response.headers)
|
||||
print(response.text)
|
||||
ct = response.headers["x-garage-causality-token"]
|
||||
|
||||
print("-- Delete")
|
||||
response = requests.delete('http://localhost:3812/alex/root?sort_key=%s'%sk,
|
||||
headers={'x-garage-causality-token': ct},
|
||||
auth=auth)
|
||||
print(response.headers)
|
||||
print(response.text)
|
||||
|
||||
print("-- ReadIndex")
|
||||
response = requests.get('http://localhost:3812/alex',
|
||||
auth=auth)
|
||||
print(response.headers)
|
||||
print(response.text)
|
||||
|
||||
print("-- InsertBatch")
|
||||
response = requests.post('http://localhost:3812/alex',
|
||||
auth=auth,
|
||||
data='''
|
||||
[
|
||||
{"pk": "root", "sk": "a", "ct": null, "v": "aW5pdGlhbCB0ZXN0Cg=="},
|
||||
{"pk": "root", "sk": "b", "ct": null, "v": "aW5pdGlhbCB0ZXN1Cg=="},
|
||||
{"pk": "root", "sk": "c", "ct": null, "v": "aW5pdGlhbCB0ZXN2Cg=="}
|
||||
]
|
||||
''')
|
||||
print(response.headers)
|
||||
print(response.text)
|
||||
|
||||
print("-- ReadIndex")
|
||||
response = requests.get('http://localhost:3812/alex',
|
||||
auth=auth)
|
||||
print(response.headers)
|
||||
print(response.text)
|
||||
|
||||
for sk in sort_keys:
|
||||
print("-- (%s) Get"%sk)
|
||||
response = requests.get('http://localhost:3812/alex/root?sort_key=%s'%sk,
|
||||
auth=auth)
|
||||
print(response.headers)
|
||||
print(response.text)
|
||||
ct = response.headers["x-garage-causality-token"]
|
||||
|
||||
print("-- ReadBatch")
|
||||
response = requests.post('http://localhost:3812/alex?search',
|
||||
auth=auth,
|
||||
data='''
|
||||
[
|
||||
{"partitionKey": "root"},
|
||||
{"partitionKey": "root", "tombstones": true},
|
||||
{"partitionKey": "root", "tombstones": true, "limit": 2},
|
||||
{"partitionKey": "root", "start": "c", "singleItem": true},
|
||||
{"partitionKey": "root", "start": "b", "end": "d", "tombstones": true}
|
||||
]
|
||||
''')
|
||||
print(response.headers)
|
||||
print(response.text)
|
||||
|
||||
|
||||
print("-- DeleteBatch")
|
||||
response = requests.post('http://localhost:3812/alex?delete',
|
||||
auth=auth,
|
||||
data='''
|
||||
[
|
||||
{"partitionKey": "root", "start": "b", "end": "c"}
|
||||
]
|
||||
''')
|
||||
print(response.headers)
|
||||
print(response.text)
|
||||
|
||||
print("-- ReadBatch")
|
||||
response = requests.post('http://localhost:3812/alex?search',
|
||||
auth=auth,
|
||||
data='''
|
||||
[
|
||||
{"partitionKey": "root"}
|
||||
]
|
||||
''')
|
||||
print(response.headers)
|
||||
print(response.text)
|
||||
+2
-3
@@ -168,13 +168,12 @@ let
|
||||
rootFeatures = if features != null then
|
||||
features
|
||||
else
|
||||
([ "garage/bundled-libs" "garage/sled" "garage/lmdb" "garage/k2v" ] ++ (if release then [
|
||||
([ "garage/bundled-libs" "garage/sled" "garage/lmdb" "garage/sqlite" "garage/k2v" ] ++ (if release then [
|
||||
"garage/consul-discovery"
|
||||
"garage/kubernetes-discovery"
|
||||
"garage/metrics"
|
||||
"garage/telemetry-otlp"
|
||||
"garage/lmdb"
|
||||
"garage/sqlite"
|
||||
"garage/syslog"
|
||||
] else
|
||||
[ ]));
|
||||
|
||||
|
||||
@@ -15,10 +15,10 @@ type: application
|
||||
# This is the chart version. This version number should be incremented each time you make changes
|
||||
# to the chart and its templates, including the app version.
|
||||
# Versions are expected to follow Semantic Versioning (https://semver.org/)
|
||||
version: 0.4.1
|
||||
version: 0.4.2
|
||||
|
||||
# This is the version number of the application being deployed. This version number should be
|
||||
# incremented each time you make changes to the application. Versions are not expected to
|
||||
# follow Semantic Versioning. They should reflect the version the application is using.
|
||||
# It is recommended to use it with quotes.
|
||||
appVersion: "v0.9.1"
|
||||
appVersion: "v0.9.4"
|
||||
|
||||
Vendored
+7
-7
@@ -30,11 +30,11 @@ Vagrant.configure("2") do |config|
|
||||
config.vm.define "n6" do |config| vm(config, "n6", "192.168.56.26") end
|
||||
config.vm.define "n7" do |config| vm(config, "n7", "192.168.56.27") end
|
||||
|
||||
config.vm.define "n8" do |config| vm(config, "n8", "192.168.56.28") end
|
||||
config.vm.define "n9" do |config| vm(config, "n9", "192.168.56.29") end
|
||||
config.vm.define "n10" do |config| vm(config, "n10", "192.168.56.30") end
|
||||
config.vm.define "n11" do |config| vm(config, "n11", "192.168.56.31") end
|
||||
config.vm.define "n12" do |config| vm(config, "n12", "192.168.56.32") end
|
||||
config.vm.define "n13" do |config| vm(config, "n13", "192.168.56.33") end
|
||||
config.vm.define "n14" do |config| vm(config, "n14", "192.168.56.34") end
|
||||
#config.vm.define "n8" do |config| vm(config, "n8", "192.168.56.28") end
|
||||
#config.vm.define "n9" do |config| vm(config, "n9", "192.168.56.29") end
|
||||
#config.vm.define "n10" do |config| vm(config, "n10", "192.168.56.30") end
|
||||
#config.vm.define "n11" do |config| vm(config, "n11", "192.168.56.31") end
|
||||
#config.vm.define "n12" do |config| vm(config, "n12", "192.168.56.32") end
|
||||
#config.vm.define "n13" do |config| vm(config, "n13", "192.168.56.33") end
|
||||
#config.vm.define "n14" do |config| vm(config, "n14", "192.168.56.34") end
|
||||
end
|
||||
|
||||
@@ -3,11 +3,10 @@
|
||||
set -x
|
||||
|
||||
#for ppatch in task3c task3a tsfix2; do
|
||||
for ppatch in tsfix2; do
|
||||
for ppatch in v093 v1rc1; do
|
||||
#for psc in c cp cdp r pr cpr dpr; do
|
||||
for psc in cdp r pr cpr dpr; do
|
||||
#for ptsk in reg2 set1 set2; do
|
||||
for ptsk in set1; do
|
||||
for ptsk in reg2 set2; do
|
||||
for psc in c cp cdp r pr cpr dpr; do
|
||||
for irun in $(seq 10); do
|
||||
lein run test --nodes-file nodes.vagrant \
|
||||
--time-limit 60 --rate 100 --concurrency 100 --ops-per-key 100 \
|
||||
|
||||
@@ -38,7 +38,9 @@
|
||||
"tsfix2" "c82d91c6bccf307186332b6c5c6fc0b128b1b2b1"
|
||||
"task3a" "707442f5de416fdbed4681a33b739f0a787b7834"
|
||||
"task3b" "431b28e0cfdc9cac6c649193cf602108a8b02997"
|
||||
"task3c" "0041b013a473e3ae72f50209d8f79db75a72848b"})
|
||||
"task3c" "0041b013a473e3ae72f50209d8f79db75a72848b"
|
||||
"v093" "v0.9.3"
|
||||
"v1rc1" "v1.0.0-rc1"})
|
||||
|
||||
(def cli-opts
|
||||
"Additional command line options."
|
||||
|
||||
@@ -43,7 +43,7 @@
|
||||
"rpc_bind_addr = \"0.0.0.0:3901\"\n"
|
||||
"rpc_public_addr = \"" node ":3901\"\n"
|
||||
"db_engine = \"lmdb\"\n"
|
||||
"replication_mode = \"2\"\n"
|
||||
"replication_mode = \"3\"\n"
|
||||
"data_dir = \"" data-dir "\"\n"
|
||||
"metadata_dir = \"" meta-dir "\"\n"
|
||||
"[s3_api]\n"
|
||||
|
||||
@@ -81,11 +81,9 @@ if [ -z "$SKIP_AWS" ]; then
|
||||
echo "Invalid multipart upload"
|
||||
exit 1
|
||||
fi
|
||||
aws s3api delete-object --bucket eprouvette --key upload
|
||||
fi
|
||||
|
||||
echo "OK!!"
|
||||
exit 0
|
||||
|
||||
# S3CMD
|
||||
if [ -z "$SKIP_S3CMD" ]; then
|
||||
echo "🛠️ Testing with s3cmd"
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_api"
|
||||
version = "0.9.1"
|
||||
version = "0.9.4"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use std::convert::Infallible;
|
||||
use std::sync::Arc;
|
||||
|
||||
use futures::{Stream, StreamExt, TryStreamExt};
|
||||
|
||||
@@ -10,6 +11,10 @@ use hyper::{
|
||||
use idna::domain_to_unicode;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use garage_model::bucket_table::BucketParams;
|
||||
use garage_model::garage::Garage;
|
||||
use garage_model::key_table::Key;
|
||||
use garage_util::data::Uuid;
|
||||
use garage_util::error::Error as GarageError;
|
||||
|
||||
use crate::common_error::{CommonError as Error, *};
|
||||
@@ -27,6 +32,15 @@ pub enum Authorization {
|
||||
Owner,
|
||||
}
|
||||
|
||||
/// The values which are known for each request related to a bucket
|
||||
pub struct ReqCtx {
|
||||
pub garage: Arc<Garage>,
|
||||
pub bucket_id: Uuid,
|
||||
pub bucket_name: String,
|
||||
pub bucket_params: BucketParams,
|
||||
pub api_key: Key,
|
||||
}
|
||||
|
||||
/// Host to bucket
|
||||
///
|
||||
/// Convert a host, like "bucket.garage-site.tld" to the corresponding bucket "bucket",
|
||||
|
||||
+24
-21
@@ -95,6 +95,7 @@ impl ApiHandler for K2VApiServer {
|
||||
.bucket_helper()
|
||||
.get_existing_bucket(bucket_id)
|
||||
.await?;
|
||||
let bucket_params = bucket.state.into_option().unwrap();
|
||||
|
||||
let allowed = match endpoint.authorization_type() {
|
||||
Authorization::Read => api_key.allow_read(&bucket_id),
|
||||
@@ -112,40 +113,42 @@ impl ApiHandler for K2VApiServer {
|
||||
// are always preflighted, i.e. the browser should make
|
||||
// an OPTIONS call before to check it is allowed
|
||||
let matching_cors_rule = match *req.method() {
|
||||
Method::GET | Method::HEAD | Method::POST => find_matching_cors_rule(&bucket, &req)
|
||||
.ok_or_internal_error("Error looking up CORS rule")?,
|
||||
Method::GET | Method::HEAD | Method::POST => {
|
||||
find_matching_cors_rule(&bucket_params, &req)
|
||||
.ok_or_internal_error("Error looking up CORS rule")?
|
||||
.cloned()
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
|
||||
let ctx = ReqCtx {
|
||||
garage,
|
||||
bucket_id,
|
||||
bucket_name,
|
||||
bucket_params,
|
||||
api_key,
|
||||
};
|
||||
|
||||
let resp = match endpoint {
|
||||
Endpoint::DeleteItem {
|
||||
partition_key,
|
||||
sort_key,
|
||||
} => handle_delete_item(garage, req, bucket_id, &partition_key, &sort_key).await,
|
||||
} => handle_delete_item(ctx, req, &partition_key, &sort_key).await,
|
||||
Endpoint::InsertItem {
|
||||
partition_key,
|
||||
sort_key,
|
||||
} => handle_insert_item(garage, req, bucket_id, &partition_key, &sort_key).await,
|
||||
} => handle_insert_item(ctx, req, &partition_key, &sort_key).await,
|
||||
Endpoint::ReadItem {
|
||||
partition_key,
|
||||
sort_key,
|
||||
} => handle_read_item(garage, &req, bucket_id, &partition_key, &sort_key).await,
|
||||
} => handle_read_item(ctx, &req, &partition_key, &sort_key).await,
|
||||
Endpoint::PollItem {
|
||||
partition_key,
|
||||
sort_key,
|
||||
causality_token,
|
||||
timeout,
|
||||
} => {
|
||||
handle_poll_item(
|
||||
garage,
|
||||
&req,
|
||||
bucket_id,
|
||||
partition_key,
|
||||
sort_key,
|
||||
causality_token,
|
||||
timeout,
|
||||
)
|
||||
.await
|
||||
handle_poll_item(ctx, &req, partition_key, sort_key, causality_token, timeout).await
|
||||
}
|
||||
Endpoint::ReadIndex {
|
||||
prefix,
|
||||
@@ -153,12 +156,12 @@ impl ApiHandler for K2VApiServer {
|
||||
end,
|
||||
limit,
|
||||
reverse,
|
||||
} => handle_read_index(garage, bucket_id, prefix, start, end, limit, reverse).await,
|
||||
Endpoint::InsertBatch {} => handle_insert_batch(garage, bucket_id, req).await,
|
||||
Endpoint::ReadBatch {} => handle_read_batch(garage, bucket_id, req).await,
|
||||
Endpoint::DeleteBatch {} => handle_delete_batch(garage, bucket_id, req).await,
|
||||
} => handle_read_index(ctx, prefix, start, end, limit, reverse).await,
|
||||
Endpoint::InsertBatch {} => handle_insert_batch(ctx, req).await,
|
||||
Endpoint::ReadBatch {} => handle_read_batch(ctx, req).await,
|
||||
Endpoint::DeleteBatch {} => handle_delete_batch(ctx, req).await,
|
||||
Endpoint::PollRange { partition_key } => {
|
||||
handle_poll_range(garage, bucket_id, &partition_key, req).await
|
||||
handle_poll_range(ctx, &partition_key, req).await
|
||||
}
|
||||
Endpoint::Options => unreachable!(),
|
||||
};
|
||||
@@ -167,7 +170,7 @@ impl ApiHandler for K2VApiServer {
|
||||
// add the corresponding CORS headers to the response
|
||||
let mut resp_ok = resp?;
|
||||
if let Some(rule) = matching_cors_rule {
|
||||
add_cors_headers(&mut resp_ok, rule)
|
||||
add_cors_headers(&mut resp_ok, &rule)
|
||||
.ok_or_internal_error("Invalid bucket CORS configuration")?;
|
||||
}
|
||||
|
||||
|
||||
+27
-24
@@ -1,14 +1,9 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use base64::prelude::*;
|
||||
use hyper::{Request, Response, StatusCode};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use garage_util::data::*;
|
||||
|
||||
use garage_table::{EnumerationOrder, TableSchema};
|
||||
|
||||
use garage_model::garage::Garage;
|
||||
use garage_model::k2v::causality::*;
|
||||
use garage_model::k2v::item_table::*;
|
||||
|
||||
@@ -18,10 +13,12 @@ use crate::k2v::error::*;
|
||||
use crate::k2v::range::read_range;
|
||||
|
||||
pub async fn handle_insert_batch(
|
||||
garage: Arc<Garage>,
|
||||
bucket_id: Uuid,
|
||||
ctx: ReqCtx,
|
||||
req: Request<ReqBody>,
|
||||
) -> Result<Response<ResBody>, Error> {
|
||||
let ReqCtx {
|
||||
garage, bucket_id, ..
|
||||
} = &ctx;
|
||||
let items = parse_json_body::<Vec<InsertBatchItem>, _, Error>(req).await?;
|
||||
|
||||
let mut items2 = vec![];
|
||||
@@ -38,7 +35,7 @@ pub async fn handle_insert_batch(
|
||||
items2.push((it.pk, it.sk, ct, v));
|
||||
}
|
||||
|
||||
garage.k2v.rpc.insert_batch(bucket_id, items2).await?;
|
||||
garage.k2v.rpc.insert_batch(*bucket_id, items2).await?;
|
||||
|
||||
Ok(Response::builder()
|
||||
.status(StatusCode::NO_CONTENT)
|
||||
@@ -46,8 +43,7 @@ pub async fn handle_insert_batch(
|
||||
}
|
||||
|
||||
pub async fn handle_read_batch(
|
||||
garage: Arc<Garage>,
|
||||
bucket_id: Uuid,
|
||||
ctx: ReqCtx,
|
||||
req: Request<ReqBody>,
|
||||
) -> Result<Response<ResBody>, Error> {
|
||||
let queries = parse_json_body::<Vec<ReadBatchQuery>, _, Error>(req).await?;
|
||||
@@ -55,7 +51,7 @@ pub async fn handle_read_batch(
|
||||
let resp_results = futures::future::join_all(
|
||||
queries
|
||||
.into_iter()
|
||||
.map(|q| handle_read_batch_query(&garage, bucket_id, q)),
|
||||
.map(|q| handle_read_batch_query(&ctx, q)),
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -68,12 +64,15 @@ pub async fn handle_read_batch(
|
||||
}
|
||||
|
||||
async fn handle_read_batch_query(
|
||||
garage: &Arc<Garage>,
|
||||
bucket_id: Uuid,
|
||||
ctx: &ReqCtx,
|
||||
query: ReadBatchQuery,
|
||||
) -> Result<ReadBatchResponse, Error> {
|
||||
let ReqCtx {
|
||||
garage, bucket_id, ..
|
||||
} = ctx;
|
||||
|
||||
let partition = K2VItemPartition {
|
||||
bucket_id,
|
||||
bucket_id: *bucket_id,
|
||||
partition_key: query.partition_key.clone(),
|
||||
};
|
||||
|
||||
@@ -138,8 +137,7 @@ async fn handle_read_batch_query(
|
||||
}
|
||||
|
||||
pub async fn handle_delete_batch(
|
||||
garage: Arc<Garage>,
|
||||
bucket_id: Uuid,
|
||||
ctx: ReqCtx,
|
||||
req: Request<ReqBody>,
|
||||
) -> Result<Response<ResBody>, Error> {
|
||||
let queries = parse_json_body::<Vec<DeleteBatchQuery>, _, Error>(req).await?;
|
||||
@@ -147,7 +145,7 @@ pub async fn handle_delete_batch(
|
||||
let resp_results = futures::future::join_all(
|
||||
queries
|
||||
.into_iter()
|
||||
.map(|q| handle_delete_batch_query(&garage, bucket_id, q)),
|
||||
.map(|q| handle_delete_batch_query(&ctx, q)),
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -160,12 +158,15 @@ pub async fn handle_delete_batch(
|
||||
}
|
||||
|
||||
async fn handle_delete_batch_query(
|
||||
garage: &Arc<Garage>,
|
||||
bucket_id: Uuid,
|
||||
ctx: &ReqCtx,
|
||||
query: DeleteBatchQuery,
|
||||
) -> Result<DeleteBatchResponse, Error> {
|
||||
let ReqCtx {
|
||||
garage, bucket_id, ..
|
||||
} = &ctx;
|
||||
|
||||
let partition = K2VItemPartition {
|
||||
bucket_id,
|
||||
bucket_id: *bucket_id,
|
||||
partition_key: query.partition_key.clone(),
|
||||
};
|
||||
|
||||
@@ -195,7 +196,7 @@ async fn handle_delete_batch_query(
|
||||
.k2v
|
||||
.rpc
|
||||
.insert(
|
||||
bucket_id,
|
||||
*bucket_id,
|
||||
i.partition.partition_key,
|
||||
i.sort_key,
|
||||
Some(cc),
|
||||
@@ -235,7 +236,7 @@ async fn handle_delete_batch_query(
|
||||
.collect::<Vec<_>>();
|
||||
let n = items.len();
|
||||
|
||||
garage.k2v.rpc.insert_batch(bucket_id, items).await?;
|
||||
garage.k2v.rpc.insert_batch(*bucket_id, items).await?;
|
||||
|
||||
n
|
||||
};
|
||||
@@ -251,11 +252,13 @@ async fn handle_delete_batch_query(
|
||||
}
|
||||
|
||||
pub(crate) async fn handle_poll_range(
|
||||
garage: Arc<Garage>,
|
||||
bucket_id: Uuid,
|
||||
ctx: ReqCtx,
|
||||
partition_key: &str,
|
||||
req: Request<ReqBody>,
|
||||
) -> Result<Response<ResBody>, Error> {
|
||||
let ReqCtx {
|
||||
garage, bucket_id, ..
|
||||
} = ctx;
|
||||
use garage_model::k2v::sub::PollRange;
|
||||
|
||||
let query = parse_json_body::<PollRangeQuery, _, Error>(req).await?;
|
||||
|
||||
@@ -3,12 +3,9 @@ use std::sync::Arc;
|
||||
use hyper::Response;
|
||||
use serde::Serialize;
|
||||
|
||||
use garage_util::data::*;
|
||||
|
||||
use garage_rpc::ring::Ring;
|
||||
use garage_table::util::*;
|
||||
|
||||
use garage_model::garage::Garage;
|
||||
use garage_model::k2v::item_table::{BYTES, CONFLICTS, ENTRIES, VALUES};
|
||||
|
||||
use crate::helpers::*;
|
||||
@@ -17,14 +14,17 @@ use crate::k2v::error::*;
|
||||
use crate::k2v::range::read_range;
|
||||
|
||||
pub async fn handle_read_index(
|
||||
garage: Arc<Garage>,
|
||||
bucket_id: Uuid,
|
||||
ctx: ReqCtx,
|
||||
prefix: Option<String>,
|
||||
start: Option<String>,
|
||||
end: Option<String>,
|
||||
limit: Option<u64>,
|
||||
reverse: Option<bool>,
|
||||
) -> Result<Response<ResBody>, Error> {
|
||||
let ReqCtx {
|
||||
garage, bucket_id, ..
|
||||
} = &ctx;
|
||||
|
||||
let reverse = reverse.unwrap_or(false);
|
||||
|
||||
let ring: Arc<Ring> = garage.system.ring.borrow().clone();
|
||||
|
||||
+21
-17
@@ -1,13 +1,8 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use base64::prelude::*;
|
||||
use http::header;
|
||||
|
||||
use hyper::{Request, Response, StatusCode};
|
||||
|
||||
use garage_util::data::*;
|
||||
|
||||
use garage_model::garage::Garage;
|
||||
use garage_model::k2v::causality::*;
|
||||
use garage_model::k2v::item_table::*;
|
||||
|
||||
@@ -100,12 +95,15 @@ impl ReturnFormat {
|
||||
/// Handle ReadItem request
|
||||
#[allow(clippy::ptr_arg)]
|
||||
pub async fn handle_read_item(
|
||||
garage: Arc<Garage>,
|
||||
ctx: ReqCtx,
|
||||
req: &Request<ReqBody>,
|
||||
bucket_id: Uuid,
|
||||
partition_key: &str,
|
||||
sort_key: &String,
|
||||
) -> Result<Response<ResBody>, Error> {
|
||||
let ReqCtx {
|
||||
garage, bucket_id, ..
|
||||
} = &ctx;
|
||||
|
||||
let format = ReturnFormat::from(req)?;
|
||||
|
||||
let item = garage
|
||||
@@ -113,7 +111,7 @@ pub async fn handle_read_item(
|
||||
.item_table
|
||||
.get(
|
||||
&K2VItemPartition {
|
||||
bucket_id,
|
||||
bucket_id: *bucket_id,
|
||||
partition_key: partition_key.to_string(),
|
||||
},
|
||||
sort_key,
|
||||
@@ -125,12 +123,14 @@ pub async fn handle_read_item(
|
||||
}
|
||||
|
||||
pub async fn handle_insert_item(
|
||||
garage: Arc<Garage>,
|
||||
ctx: ReqCtx,
|
||||
req: Request<ReqBody>,
|
||||
bucket_id: Uuid,
|
||||
partition_key: &str,
|
||||
sort_key: &str,
|
||||
) -> Result<Response<ResBody>, Error> {
|
||||
let ReqCtx {
|
||||
garage, bucket_id, ..
|
||||
} = &ctx;
|
||||
let causal_context = req
|
||||
.headers()
|
||||
.get(X_GARAGE_CAUSALITY_TOKEN)
|
||||
@@ -149,7 +149,7 @@ pub async fn handle_insert_item(
|
||||
.k2v
|
||||
.rpc
|
||||
.insert(
|
||||
bucket_id,
|
||||
*bucket_id,
|
||||
partition_key.to_string(),
|
||||
sort_key.to_string(),
|
||||
causal_context,
|
||||
@@ -163,12 +163,14 @@ pub async fn handle_insert_item(
|
||||
}
|
||||
|
||||
pub async fn handle_delete_item(
|
||||
garage: Arc<Garage>,
|
||||
ctx: ReqCtx,
|
||||
req: Request<ReqBody>,
|
||||
bucket_id: Uuid,
|
||||
partition_key: &str,
|
||||
sort_key: &str,
|
||||
) -> Result<Response<ResBody>, Error> {
|
||||
let ReqCtx {
|
||||
garage, bucket_id, ..
|
||||
} = &ctx;
|
||||
let causal_context = req
|
||||
.headers()
|
||||
.get(X_GARAGE_CAUSALITY_TOKEN)
|
||||
@@ -183,7 +185,7 @@ pub async fn handle_delete_item(
|
||||
.k2v
|
||||
.rpc
|
||||
.insert(
|
||||
bucket_id,
|
||||
*bucket_id,
|
||||
partition_key.to_string(),
|
||||
sort_key.to_string(),
|
||||
causal_context,
|
||||
@@ -199,14 +201,16 @@ pub async fn handle_delete_item(
|
||||
/// Handle ReadItem request
|
||||
#[allow(clippy::ptr_arg)]
|
||||
pub async fn handle_poll_item(
|
||||
garage: Arc<Garage>,
|
||||
ctx: ReqCtx,
|
||||
req: &Request<ReqBody>,
|
||||
bucket_id: Uuid,
|
||||
partition_key: String,
|
||||
sort_key: String,
|
||||
causality_token: String,
|
||||
timeout_secs: Option<u64>,
|
||||
) -> Result<Response<ResBody>, Error> {
|
||||
let ReqCtx {
|
||||
garage, bucket_id, ..
|
||||
} = &ctx;
|
||||
let format = ReturnFormat::from(req)?;
|
||||
|
||||
let causal_context =
|
||||
@@ -218,7 +222,7 @@ pub async fn handle_poll_item(
|
||||
.k2v
|
||||
.rpc
|
||||
.poll_item(
|
||||
bucket_id,
|
||||
*bucket_id,
|
||||
partition_key,
|
||||
sort_key,
|
||||
causal_context,
|
||||
|
||||
+82
-130
@@ -155,6 +155,7 @@ impl ApiHandler for S3ApiServer {
|
||||
.bucket_helper()
|
||||
.get_existing_bucket(bucket_id)
|
||||
.await?;
|
||||
let bucket_params = bucket.state.into_option().unwrap();
|
||||
|
||||
let allowed = match endpoint.authorization_type() {
|
||||
Authorization::Read => api_key.allow_read(&bucket_id),
|
||||
@@ -167,12 +168,20 @@ impl ApiHandler for S3ApiServer {
|
||||
return Err(Error::forbidden("Operation is not allowed for this key."));
|
||||
}
|
||||
|
||||
let matching_cors_rule = find_matching_cors_rule(&bucket, &req)?;
|
||||
let matching_cors_rule = find_matching_cors_rule(&bucket_params, &req)?.cloned();
|
||||
|
||||
let ctx = ReqCtx {
|
||||
garage,
|
||||
bucket_id,
|
||||
bucket_name,
|
||||
bucket_params,
|
||||
api_key,
|
||||
};
|
||||
|
||||
let resp = match endpoint {
|
||||
Endpoint::HeadObject {
|
||||
key, part_number, ..
|
||||
} => handle_head(garage, &req, bucket_id, &key, part_number).await,
|
||||
} => handle_head(ctx, &req, &key, part_number).await,
|
||||
Endpoint::GetObject {
|
||||
key,
|
||||
part_number,
|
||||
@@ -192,74 +201,37 @@ impl ApiHandler for S3ApiServer {
|
||||
response_content_type,
|
||||
response_expires,
|
||||
};
|
||||
handle_get(garage, &req, bucket_id, &key, part_number, overrides).await
|
||||
handle_get(ctx, &req, &key, part_number, overrides).await
|
||||
}
|
||||
Endpoint::UploadPart {
|
||||
key,
|
||||
part_number,
|
||||
upload_id,
|
||||
} => {
|
||||
handle_put_part(
|
||||
garage,
|
||||
req,
|
||||
bucket_id,
|
||||
&key,
|
||||
part_number,
|
||||
&upload_id,
|
||||
content_sha256,
|
||||
)
|
||||
.await
|
||||
}
|
||||
Endpoint::CopyObject { key } => {
|
||||
handle_copy(garage, &api_key, &req, bucket_id, &key).await
|
||||
}
|
||||
} => handle_put_part(ctx, req, &key, part_number, &upload_id, content_sha256).await,
|
||||
Endpoint::CopyObject { key } => handle_copy(ctx, &req, &key).await,
|
||||
Endpoint::UploadPartCopy {
|
||||
key,
|
||||
part_number,
|
||||
upload_id,
|
||||
} => {
|
||||
handle_upload_part_copy(
|
||||
garage,
|
||||
&api_key,
|
||||
&req,
|
||||
bucket_id,
|
||||
&key,
|
||||
part_number,
|
||||
&upload_id,
|
||||
)
|
||||
.await
|
||||
}
|
||||
Endpoint::PutObject { key } => {
|
||||
handle_put(garage, req, &bucket, &key, content_sha256).await
|
||||
}
|
||||
} => handle_upload_part_copy(ctx, &req, &key, part_number, &upload_id).await,
|
||||
Endpoint::PutObject { key } => handle_put(ctx, req, &key, content_sha256).await,
|
||||
Endpoint::AbortMultipartUpload { key, upload_id } => {
|
||||
handle_abort_multipart_upload(garage, bucket_id, &key, &upload_id).await
|
||||
handle_abort_multipart_upload(ctx, &key, &upload_id).await
|
||||
}
|
||||
Endpoint::DeleteObject { key, .. } => handle_delete(garage, bucket_id, &key).await,
|
||||
Endpoint::DeleteObject { key, .. } => handle_delete(ctx, &key).await,
|
||||
Endpoint::CreateMultipartUpload { key } => {
|
||||
handle_create_multipart_upload(garage, &req, &bucket_name, bucket_id, &key).await
|
||||
handle_create_multipart_upload(ctx, &req, &key).await
|
||||
}
|
||||
Endpoint::CompleteMultipartUpload { key, upload_id } => {
|
||||
handle_complete_multipart_upload(
|
||||
garage,
|
||||
req,
|
||||
&bucket_name,
|
||||
&bucket,
|
||||
&key,
|
||||
&upload_id,
|
||||
content_sha256,
|
||||
)
|
||||
.await
|
||||
handle_complete_multipart_upload(ctx, req, &key, &upload_id, content_sha256).await
|
||||
}
|
||||
Endpoint::CreateBucket {} => unreachable!(),
|
||||
Endpoint::HeadBucket {} => {
|
||||
let response = Response::builder().body(empty_body()).unwrap();
|
||||
Ok(response)
|
||||
}
|
||||
Endpoint::DeleteBucket {} => {
|
||||
handle_delete_bucket(&garage, bucket_id, bucket_name, &api_key.key_id).await
|
||||
}
|
||||
Endpoint::GetBucketLocation {} => handle_get_bucket_location(garage),
|
||||
Endpoint::DeleteBucket {} => handle_delete_bucket(ctx).await,
|
||||
Endpoint::GetBucketLocation {} => handle_get_bucket_location(ctx),
|
||||
Endpoint::GetBucketVersioning {} => handle_get_bucket_versioning(),
|
||||
Endpoint::ListObjects {
|
||||
delimiter,
|
||||
@@ -268,24 +240,21 @@ impl ApiHandler for S3ApiServer {
|
||||
max_keys,
|
||||
prefix,
|
||||
} => {
|
||||
handle_list(
|
||||
garage,
|
||||
&ListObjectsQuery {
|
||||
common: ListQueryCommon {
|
||||
bucket_name,
|
||||
bucket_id,
|
||||
delimiter,
|
||||
page_size: max_keys.unwrap_or(1000).clamp(1, 1000),
|
||||
prefix: prefix.unwrap_or_default(),
|
||||
urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false),
|
||||
},
|
||||
is_v2: false,
|
||||
marker,
|
||||
continuation_token: None,
|
||||
start_after: None,
|
||||
let query = ListObjectsQuery {
|
||||
common: ListQueryCommon {
|
||||
bucket_name: ctx.bucket_name.clone(),
|
||||
bucket_id,
|
||||
delimiter,
|
||||
page_size: max_keys.unwrap_or(1000).clamp(1, 1000),
|
||||
prefix: prefix.unwrap_or_default(),
|
||||
urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false),
|
||||
},
|
||||
)
|
||||
.await
|
||||
is_v2: false,
|
||||
marker,
|
||||
continuation_token: None,
|
||||
start_after: None,
|
||||
};
|
||||
handle_list(ctx, &query).await
|
||||
}
|
||||
Endpoint::ListObjectsV2 {
|
||||
delimiter,
|
||||
@@ -298,24 +267,21 @@ impl ApiHandler for S3ApiServer {
|
||||
..
|
||||
} => {
|
||||
if list_type == "2" {
|
||||
handle_list(
|
||||
garage,
|
||||
&ListObjectsQuery {
|
||||
common: ListQueryCommon {
|
||||
bucket_name,
|
||||
bucket_id,
|
||||
delimiter,
|
||||
page_size: max_keys.unwrap_or(1000).clamp(1, 1000),
|
||||
urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false),
|
||||
prefix: prefix.unwrap_or_default(),
|
||||
},
|
||||
is_v2: true,
|
||||
marker: None,
|
||||
continuation_token,
|
||||
start_after,
|
||||
let query = ListObjectsQuery {
|
||||
common: ListQueryCommon {
|
||||
bucket_name: ctx.bucket_name.clone(),
|
||||
bucket_id,
|
||||
delimiter,
|
||||
page_size: max_keys.unwrap_or(1000).clamp(1, 1000),
|
||||
urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false),
|
||||
prefix: prefix.unwrap_or_default(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
is_v2: true,
|
||||
marker: None,
|
||||
continuation_token,
|
||||
start_after,
|
||||
};
|
||||
handle_list(ctx, &query).await
|
||||
} else {
|
||||
Err(Error::bad_request(format!(
|
||||
"Invalid endpoint: list-type={}",
|
||||
@@ -331,22 +297,19 @@ impl ApiHandler for S3ApiServer {
|
||||
prefix,
|
||||
upload_id_marker,
|
||||
} => {
|
||||
handle_list_multipart_upload(
|
||||
garage,
|
||||
&ListMultipartUploadsQuery {
|
||||
common: ListQueryCommon {
|
||||
bucket_name,
|
||||
bucket_id,
|
||||
delimiter,
|
||||
page_size: max_uploads.unwrap_or(1000).clamp(1, 1000),
|
||||
prefix: prefix.unwrap_or_default(),
|
||||
urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false),
|
||||
},
|
||||
key_marker,
|
||||
upload_id_marker,
|
||||
let query = ListMultipartUploadsQuery {
|
||||
common: ListQueryCommon {
|
||||
bucket_name: ctx.bucket_name.clone(),
|
||||
bucket_id,
|
||||
delimiter,
|
||||
page_size: max_uploads.unwrap_or(1000).clamp(1, 1000),
|
||||
prefix: prefix.unwrap_or_default(),
|
||||
urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false),
|
||||
},
|
||||
)
|
||||
.await
|
||||
key_marker,
|
||||
upload_id_marker,
|
||||
};
|
||||
handle_list_multipart_upload(ctx, &query).await
|
||||
}
|
||||
Endpoint::ListParts {
|
||||
key,
|
||||
@@ -354,39 +317,28 @@ impl ApiHandler for S3ApiServer {
|
||||
part_number_marker,
|
||||
upload_id,
|
||||
} => {
|
||||
handle_list_parts(
|
||||
garage,
|
||||
&ListPartsQuery {
|
||||
bucket_name,
|
||||
bucket_id,
|
||||
key,
|
||||
upload_id,
|
||||
part_number_marker: part_number_marker.map(|p| p.min(10000)),
|
||||
max_parts: max_parts.unwrap_or(1000).clamp(1, 1000),
|
||||
},
|
||||
)
|
||||
.await
|
||||
let query = ListPartsQuery {
|
||||
bucket_name: ctx.bucket_name.clone(),
|
||||
bucket_id,
|
||||
key,
|
||||
upload_id,
|
||||
part_number_marker: part_number_marker.map(|p| p.min(10000)),
|
||||
max_parts: max_parts.unwrap_or(1000).clamp(1, 1000),
|
||||
};
|
||||
handle_list_parts(ctx, &query).await
|
||||
}
|
||||
Endpoint::DeleteObjects {} => {
|
||||
handle_delete_objects(garage, bucket_id, req, content_sha256).await
|
||||
}
|
||||
Endpoint::GetBucketWebsite {} => handle_get_website(&bucket).await,
|
||||
Endpoint::PutBucketWebsite {} => {
|
||||
handle_put_website(garage, bucket.clone(), req, content_sha256).await
|
||||
}
|
||||
Endpoint::DeleteBucketWebsite {} => handle_delete_website(garage, bucket.clone()).await,
|
||||
Endpoint::GetBucketCors {} => handle_get_cors(&bucket).await,
|
||||
Endpoint::PutBucketCors {} => {
|
||||
handle_put_cors(garage, bucket.clone(), req, content_sha256).await
|
||||
}
|
||||
Endpoint::DeleteBucketCors {} => handle_delete_cors(garage, bucket.clone()).await,
|
||||
Endpoint::GetBucketLifecycleConfiguration {} => handle_get_lifecycle(&bucket).await,
|
||||
Endpoint::DeleteObjects {} => handle_delete_objects(ctx, req, content_sha256).await,
|
||||
Endpoint::GetBucketWebsite {} => handle_get_website(ctx).await,
|
||||
Endpoint::PutBucketWebsite {} => handle_put_website(ctx, req, content_sha256).await,
|
||||
Endpoint::DeleteBucketWebsite {} => handle_delete_website(ctx).await,
|
||||
Endpoint::GetBucketCors {} => handle_get_cors(ctx).await,
|
||||
Endpoint::PutBucketCors {} => handle_put_cors(ctx, req, content_sha256).await,
|
||||
Endpoint::DeleteBucketCors {} => handle_delete_cors(ctx).await,
|
||||
Endpoint::GetBucketLifecycleConfiguration {} => handle_get_lifecycle(ctx).await,
|
||||
Endpoint::PutBucketLifecycleConfiguration {} => {
|
||||
handle_put_lifecycle(garage, bucket.clone(), req, content_sha256).await
|
||||
}
|
||||
Endpoint::DeleteBucketLifecycle {} => {
|
||||
handle_delete_lifecycle(garage, bucket.clone()).await
|
||||
handle_put_lifecycle(ctx, req, content_sha256).await
|
||||
}
|
||||
Endpoint::DeleteBucketLifecycle {} => handle_delete_lifecycle(ctx).await,
|
||||
endpoint => Err(Error::NotImplemented(endpoint.name().to_owned())),
|
||||
};
|
||||
|
||||
@@ -394,7 +346,7 @@ impl ApiHandler for S3ApiServer {
|
||||
// add the corresponding CORS headers to the response
|
||||
let mut resp_ok = resp?;
|
||||
if let Some(rule) = matching_cors_rule {
|
||||
add_cors_headers(&mut resp_ok, rule)
|
||||
add_cors_headers(&mut resp_ok, &rule)
|
||||
.ok_or_internal_error("Invalid bucket CORS configuration")?;
|
||||
}
|
||||
|
||||
|
||||
+25
-23
@@ -1,5 +1,4 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use http_body_util::BodyExt;
|
||||
use hyper::{Request, Response, StatusCode};
|
||||
@@ -21,7 +20,8 @@ use crate::s3::error::*;
|
||||
use crate::s3::xml as s3_xml;
|
||||
use crate::signature::verify_signed_content;
|
||||
|
||||
pub fn handle_get_bucket_location(garage: Arc<Garage>) -> Result<Response<ResBody>, Error> {
|
||||
pub fn handle_get_bucket_location(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
|
||||
let ReqCtx { garage, .. } = ctx;
|
||||
let loc = s3_xml::LocationConstraint {
|
||||
xmlns: (),
|
||||
region: garage.config.s3_api.s3_region.to_string(),
|
||||
@@ -204,21 +204,20 @@ pub async fn handle_create_bucket(
|
||||
.unwrap())
|
||||
}
|
||||
|
||||
pub async fn handle_delete_bucket(
|
||||
garage: &Garage,
|
||||
bucket_id: Uuid,
|
||||
bucket_name: String,
|
||||
api_key_id: &String,
|
||||
) -> Result<Response<ResBody>, Error> {
|
||||
pub async fn handle_delete_bucket(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
|
||||
let ReqCtx {
|
||||
garage,
|
||||
bucket_id,
|
||||
bucket_name,
|
||||
bucket_params: bucket_state,
|
||||
api_key,
|
||||
..
|
||||
} = &ctx;
|
||||
let helper = garage.locked_helper().await;
|
||||
|
||||
let api_key = helper.key().get_existing_key(api_key_id).await?;
|
||||
let key_params = api_key.params().unwrap();
|
||||
|
||||
let is_local_alias = matches!(key_params.local_aliases.get(&bucket_name), Some(Some(_)));
|
||||
|
||||
let mut bucket = helper.bucket().get_existing_bucket(bucket_id).await?;
|
||||
let bucket_state = bucket.state.as_option().unwrap();
|
||||
let is_local_alias = matches!(key_params.local_aliases.get(bucket_name), Some(Some(_)));
|
||||
|
||||
// If the bucket has no other aliases, this is a true deletion.
|
||||
// Otherwise, it is just an alias removal.
|
||||
@@ -228,20 +227,20 @@ pub async fn handle_delete_bucket(
|
||||
.items()
|
||||
.iter()
|
||||
.filter(|(_, _, active)| *active)
|
||||
.any(|(n, _, _)| is_local_alias || (*n != bucket_name));
|
||||
.any(|(n, _, _)| is_local_alias || (*n != *bucket_name));
|
||||
|
||||
let has_other_local_aliases = bucket_state
|
||||
.local_aliases
|
||||
.items()
|
||||
.iter()
|
||||
.filter(|(_, _, active)| *active)
|
||||
.any(|((k, n), _, _)| !is_local_alias || *n != bucket_name || *k != api_key.key_id);
|
||||
.any(|((k, n), _, _)| !is_local_alias || *n != *bucket_name || *k != api_key.key_id);
|
||||
|
||||
if !has_other_global_aliases && !has_other_local_aliases {
|
||||
// Delete bucket
|
||||
|
||||
// Check bucket is empty
|
||||
if !helper.bucket().is_bucket_empty(bucket_id).await? {
|
||||
if !helper.bucket().is_bucket_empty(*bucket_id).await? {
|
||||
return Err(CommonError::BucketNotEmpty.into());
|
||||
}
|
||||
|
||||
@@ -249,33 +248,36 @@ pub async fn handle_delete_bucket(
|
||||
// 1. delete bucket alias
|
||||
if is_local_alias {
|
||||
helper
|
||||
.unset_local_bucket_alias(bucket_id, &api_key.key_id, &bucket_name)
|
||||
.unset_local_bucket_alias(*bucket_id, &api_key.key_id, bucket_name)
|
||||
.await?;
|
||||
} else {
|
||||
helper
|
||||
.unset_global_bucket_alias(bucket_id, &bucket_name)
|
||||
.unset_global_bucket_alias(*bucket_id, bucket_name)
|
||||
.await?;
|
||||
}
|
||||
|
||||
// 2. delete authorization from keys that had access
|
||||
for (key_id, _) in bucket.authorized_keys() {
|
||||
for (key_id, _) in bucket_state.authorized_keys.items() {
|
||||
helper
|
||||
.set_bucket_key_permissions(bucket.id, key_id, BucketKeyPerm::NO_PERMISSIONS)
|
||||
.set_bucket_key_permissions(*bucket_id, key_id, BucketKeyPerm::NO_PERMISSIONS)
|
||||
.await?;
|
||||
}
|
||||
|
||||
let bucket = Bucket {
|
||||
id: *bucket_id,
|
||||
state: Deletable::delete(),
|
||||
};
|
||||
// 3. delete bucket
|
||||
bucket.state = Deletable::delete();
|
||||
garage.bucket_table.insert(&bucket).await?;
|
||||
} else if is_local_alias {
|
||||
// Just unalias
|
||||
helper
|
||||
.unset_local_bucket_alias(bucket_id, &api_key.key_id, &bucket_name)
|
||||
.unset_local_bucket_alias(*bucket_id, &api_key.key_id, bucket_name)
|
||||
.await?;
|
||||
} else {
|
||||
// Just unalias (but from global namespace)
|
||||
helper
|
||||
.unset_global_bucket_alias(bucket_id, &bucket_name)
|
||||
.unset_global_bucket_alias(*bucket_id, bucket_name)
|
||||
.await?;
|
||||
}
|
||||
|
||||
|
||||
+18
-17
@@ -1,5 +1,4 @@
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use futures::{stream, stream::Stream, StreamExt};
|
||||
@@ -15,8 +14,6 @@ use garage_table::*;
|
||||
use garage_util::data::*;
|
||||
use garage_util::time::*;
|
||||
|
||||
use garage_model::garage::Garage;
|
||||
use garage_model::key_table::Key;
|
||||
use garage_model::s3::block_ref_table::*;
|
||||
use garage_model::s3::mpu_table::*;
|
||||
use garage_model::s3::object_table::*;
|
||||
@@ -30,15 +27,19 @@ use crate::s3::put::get_headers;
|
||||
use crate::s3::xml::{self as s3_xml, xmlns_tag};
|
||||
|
||||
pub async fn handle_copy(
|
||||
garage: Arc<Garage>,
|
||||
api_key: &Key,
|
||||
ctx: ReqCtx,
|
||||
req: &Request<ReqBody>,
|
||||
dest_bucket_id: Uuid,
|
||||
dest_key: &str,
|
||||
) -> Result<Response<ResBody>, Error> {
|
||||
let copy_precondition = CopyPreconditionHeaders::parse(req)?;
|
||||
|
||||
let source_object = get_copy_source(&garage, api_key, req).await?;
|
||||
let source_object = get_copy_source(&ctx, req).await?;
|
||||
|
||||
let ReqCtx {
|
||||
garage,
|
||||
bucket_id: dest_bucket_id,
|
||||
..
|
||||
} = ctx;
|
||||
|
||||
let (source_version, source_version_data, source_version_meta) =
|
||||
extract_source_info(&source_object)?;
|
||||
@@ -181,10 +182,8 @@ pub async fn handle_copy(
|
||||
}
|
||||
|
||||
pub async fn handle_upload_part_copy(
|
||||
garage: Arc<Garage>,
|
||||
api_key: &Key,
|
||||
ctx: ReqCtx,
|
||||
req: &Request<ReqBody>,
|
||||
dest_bucket_id: Uuid,
|
||||
dest_key: &str,
|
||||
part_number: u64,
|
||||
upload_id: &str,
|
||||
@@ -195,10 +194,12 @@ pub async fn handle_upload_part_copy(
|
||||
|
||||
let dest_key = dest_key.to_string();
|
||||
let (source_object, (_, _, mut dest_mpu)) = futures::try_join!(
|
||||
get_copy_source(&garage, api_key, req),
|
||||
multipart::get_upload(&garage, &dest_bucket_id, &dest_key, &dest_upload_id)
|
||||
get_copy_source(&ctx, req),
|
||||
multipart::get_upload(&ctx, &dest_key, &dest_upload_id)
|
||||
)?;
|
||||
|
||||
let ReqCtx { garage, .. } = ctx;
|
||||
|
||||
let (source_object_version, source_version_data, source_version_meta) =
|
||||
extract_source_info(&source_object)?;
|
||||
|
||||
@@ -439,11 +440,11 @@ pub async fn handle_upload_part_copy(
|
||||
.body(string_body(resp_xml))?)
|
||||
}
|
||||
|
||||
async fn get_copy_source(
|
||||
garage: &Garage,
|
||||
api_key: &Key,
|
||||
req: &Request<ReqBody>,
|
||||
) -> Result<Object, Error> {
|
||||
async fn get_copy_source(ctx: &ReqCtx, req: &Request<ReqBody>) -> Result<Object, Error> {
|
||||
let ReqCtx {
|
||||
garage, api_key, ..
|
||||
} = ctx;
|
||||
|
||||
let copy_source = req.headers().get("x-amz-copy-source").unwrap().to_str()?;
|
||||
let copy_source = percent_encoding::percent_decode_str(copy_source).decode_utf8()?;
|
||||
|
||||
|
||||
+35
-30
@@ -21,16 +21,13 @@ use crate::s3::error::*;
|
||||
use crate::s3::xml::{to_xml_with_header, xmlns_tag, IntValue, Value};
|
||||
use crate::signature::verify_signed_content;
|
||||
|
||||
use garage_model::bucket_table::{Bucket, CorsRule as GarageCorsRule};
|
||||
use garage_model::bucket_table::{Bucket, BucketParams, CorsRule as GarageCorsRule};
|
||||
use garage_model::garage::Garage;
|
||||
use garage_util::data::*;
|
||||
|
||||
pub async fn handle_get_cors(bucket: &Bucket) -> Result<Response<ResBody>, Error> {
|
||||
let param = bucket
|
||||
.params()
|
||||
.ok_or_internal_error("Bucket should not be deleted at this point")?;
|
||||
|
||||
if let Some(cors) = param.cors_config.get() {
|
||||
pub async fn handle_get_cors(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
|
||||
let ReqCtx { bucket_params, .. } = ctx;
|
||||
if let Some(cors) = bucket_params.cors_config.get() {
|
||||
let wc = CorsConfiguration {
|
||||
xmlns: (),
|
||||
cors_rules: cors
|
||||
@@ -50,16 +47,18 @@ pub async fn handle_get_cors(bucket: &Bucket) -> Result<Response<ResBody>, Error
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle_delete_cors(
|
||||
garage: Arc<Garage>,
|
||||
mut bucket: Bucket,
|
||||
) -> Result<Response<ResBody>, Error> {
|
||||
let param = bucket
|
||||
.params_mut()
|
||||
.ok_or_internal_error("Bucket should not be deleted at this point")?;
|
||||
|
||||
param.cors_config.update(None);
|
||||
garage.bucket_table.insert(&bucket).await?;
|
||||
pub async fn handle_delete_cors(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
|
||||
let ReqCtx {
|
||||
garage,
|
||||
bucket_id,
|
||||
mut bucket_params,
|
||||
..
|
||||
} = ctx;
|
||||
bucket_params.cors_config.update(None);
|
||||
garage
|
||||
.bucket_table
|
||||
.insert(&Bucket::present(bucket_id, bucket_params))
|
||||
.await?;
|
||||
|
||||
Ok(Response::builder()
|
||||
.status(StatusCode::NO_CONTENT)
|
||||
@@ -67,28 +66,33 @@ pub async fn handle_delete_cors(
|
||||
}
|
||||
|
||||
pub async fn handle_put_cors(
|
||||
garage: Arc<Garage>,
|
||||
mut bucket: Bucket,
|
||||
ctx: ReqCtx,
|
||||
req: Request<ReqBody>,
|
||||
content_sha256: Option<Hash>,
|
||||
) -> Result<Response<ResBody>, Error> {
|
||||
let ReqCtx {
|
||||
garage,
|
||||
bucket_id,
|
||||
mut bucket_params,
|
||||
..
|
||||
} = ctx;
|
||||
|
||||
let body = BodyExt::collect(req.into_body()).await?.to_bytes();
|
||||
|
||||
if let Some(content_sha256) = content_sha256 {
|
||||
verify_signed_content(content_sha256, &body[..])?;
|
||||
}
|
||||
|
||||
let param = bucket
|
||||
.params_mut()
|
||||
.ok_or_internal_error("Bucket should not be deleted at this point")?;
|
||||
|
||||
let conf: CorsConfiguration = from_reader(&body as &[u8])?;
|
||||
conf.validate()?;
|
||||
|
||||
param
|
||||
bucket_params
|
||||
.cors_config
|
||||
.update(Some(conf.into_garage_cors_config()?));
|
||||
garage.bucket_table.insert(&bucket).await?;
|
||||
garage
|
||||
.bucket_table
|
||||
.insert(&Bucket::present(bucket_id, bucket_params))
|
||||
.await?;
|
||||
|
||||
Ok(Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
@@ -115,7 +119,8 @@ pub async fn handle_options_api(
|
||||
let bucket_id = helper.resolve_global_bucket_name(&bn).await?;
|
||||
if let Some(id) = bucket_id {
|
||||
let bucket = garage.bucket_helper().get_existing_bucket(id).await?;
|
||||
handle_options_for_bucket(req, &bucket)
|
||||
let bucket_params = bucket.state.into_option().unwrap();
|
||||
handle_options_for_bucket(req, &bucket_params)
|
||||
} else {
|
||||
// If there is a bucket name in the request, but that name
|
||||
// does not correspond to a global alias for a bucket,
|
||||
@@ -145,7 +150,7 @@ pub async fn handle_options_api(
|
||||
|
||||
pub fn handle_options_for_bucket(
|
||||
req: &Request<IncomingBody>,
|
||||
bucket: &Bucket,
|
||||
bucket_params: &BucketParams,
|
||||
) -> Result<Response<EmptyBody>, CommonError> {
|
||||
let origin = req
|
||||
.headers()
|
||||
@@ -162,7 +167,7 @@ pub fn handle_options_for_bucket(
|
||||
None => vec![],
|
||||
};
|
||||
|
||||
if let Some(cors_config) = bucket.params().unwrap().cors_config.get() {
|
||||
if let Some(cors_config) = bucket_params.cors_config.get() {
|
||||
let matching_rule = cors_config
|
||||
.iter()
|
||||
.find(|rule| cors_rule_matches(rule, origin, request_method, request_headers.iter()));
|
||||
@@ -181,10 +186,10 @@ pub fn handle_options_for_bucket(
|
||||
}
|
||||
|
||||
pub fn find_matching_cors_rule<'a>(
|
||||
bucket: &'a Bucket,
|
||||
bucket_params: &'a BucketParams,
|
||||
req: &Request<impl Body>,
|
||||
) -> Result<Option<&'a GarageCorsRule>, Error> {
|
||||
if let Some(cors_config) = bucket.params().unwrap().cors_config.get() {
|
||||
if let Some(cors_config) = bucket_params.cors_config.get() {
|
||||
if let Some(origin) = req.headers().get("Origin") {
|
||||
let origin = origin.to_str()?;
|
||||
let request_headers = match req.headers().get(ACCESS_CONTROL_REQUEST_HEADERS) {
|
||||
|
||||
+10
-19
@@ -1,11 +1,8 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use http_body_util::BodyExt;
|
||||
use hyper::{Request, Response, StatusCode};
|
||||
|
||||
use garage_util::data::*;
|
||||
|
||||
use garage_model::garage::Garage;
|
||||
use garage_model::s3::object_table::*;
|
||||
|
||||
use crate::helpers::*;
|
||||
@@ -15,14 +12,13 @@ use crate::s3::put::next_timestamp;
|
||||
use crate::s3::xml as s3_xml;
|
||||
use crate::signature::verify_signed_content;
|
||||
|
||||
async fn handle_delete_internal(
|
||||
garage: &Garage,
|
||||
bucket_id: Uuid,
|
||||
key: &str,
|
||||
) -> Result<(Uuid, Uuid), Error> {
|
||||
async fn handle_delete_internal(ctx: &ReqCtx, key: &str) -> Result<(Uuid, Uuid), Error> {
|
||||
let ReqCtx {
|
||||
garage, bucket_id, ..
|
||||
} = ctx;
|
||||
let object = garage
|
||||
.object_table
|
||||
.get(&bucket_id, &key.to_string())
|
||||
.get(bucket_id, &key.to_string())
|
||||
.await?
|
||||
.ok_or(Error::NoSuchKey)?; // No need to delete
|
||||
|
||||
@@ -44,7 +40,7 @@ async fn handle_delete_internal(
|
||||
};
|
||||
|
||||
let object = Object::new(
|
||||
bucket_id,
|
||||
*bucket_id,
|
||||
key.into(),
|
||||
vec![ObjectVersion {
|
||||
uuid: del_uuid,
|
||||
@@ -58,12 +54,8 @@ async fn handle_delete_internal(
|
||||
Ok((deleted_version, del_uuid))
|
||||
}
|
||||
|
||||
pub async fn handle_delete(
|
||||
garage: Arc<Garage>,
|
||||
bucket_id: Uuid,
|
||||
key: &str,
|
||||
) -> Result<Response<ResBody>, Error> {
|
||||
match handle_delete_internal(&garage, bucket_id, key).await {
|
||||
pub async fn handle_delete(ctx: ReqCtx, key: &str) -> Result<Response<ResBody>, Error> {
|
||||
match handle_delete_internal(&ctx, key).await {
|
||||
Ok(_) | Err(Error::NoSuchKey) => Ok(Response::builder()
|
||||
.status(StatusCode::NO_CONTENT)
|
||||
.body(empty_body())
|
||||
@@ -73,8 +65,7 @@ pub async fn handle_delete(
|
||||
}
|
||||
|
||||
pub async fn handle_delete_objects(
|
||||
garage: Arc<Garage>,
|
||||
bucket_id: Uuid,
|
||||
ctx: ReqCtx,
|
||||
req: Request<ReqBody>,
|
||||
content_sha256: Option<Hash>,
|
||||
) -> Result<Response<ResBody>, Error> {
|
||||
@@ -91,7 +82,7 @@ pub async fn handle_delete_objects(
|
||||
let mut ret_errors = Vec::new();
|
||||
|
||||
for obj in cmd.objects.iter() {
|
||||
match handle_delete_internal(&garage, bucket_id, &obj.key).await {
|
||||
match handle_delete_internal(&ctx, &obj.key).await {
|
||||
Ok((deleted_version, delete_marker_version)) => {
|
||||
if cmd.quiet {
|
||||
continue;
|
||||
|
||||
@@ -131,6 +131,16 @@ fn try_answer_cached(
|
||||
|
||||
/// Handle HEAD request
|
||||
pub async fn handle_head(
|
||||
ctx: ReqCtx,
|
||||
req: &Request<impl Body>,
|
||||
key: &str,
|
||||
part_number: Option<u64>,
|
||||
) -> Result<Response<ResBody>, Error> {
|
||||
handle_head_without_ctx(ctx.garage, req, ctx.bucket_id, key, part_number).await
|
||||
}
|
||||
|
||||
/// Handle HEAD request for website
|
||||
pub async fn handle_head_without_ctx(
|
||||
garage: Arc<Garage>,
|
||||
req: &Request<impl Body>,
|
||||
bucket_id: Uuid,
|
||||
@@ -218,6 +228,17 @@ pub async fn handle_head(
|
||||
|
||||
/// Handle GET request
|
||||
pub async fn handle_get(
|
||||
ctx: ReqCtx,
|
||||
req: &Request<impl Body>,
|
||||
key: &str,
|
||||
part_number: Option<u64>,
|
||||
overrides: GetObjectOverrides,
|
||||
) -> Result<Response<ResBody>, Error> {
|
||||
handle_get_without_ctx(ctx.garage, req, ctx.bucket_id, key, part_number, overrides).await
|
||||
}
|
||||
|
||||
/// Handle GET request
|
||||
pub async fn handle_get_without_ctx(
|
||||
garage: Arc<Garage>,
|
||||
req: &Request<impl Body>,
|
||||
bucket_id: Uuid,
|
||||
|
||||
+28
-25
@@ -1,5 +1,4 @@
|
||||
use quick_xml::de::from_reader;
|
||||
use std::sync::Arc;
|
||||
|
||||
use http_body_util::BodyExt;
|
||||
use hyper::{Request, Response, StatusCode};
|
||||
@@ -16,15 +15,12 @@ use garage_model::bucket_table::{
|
||||
parse_lifecycle_date, Bucket, LifecycleExpiration as GarageLifecycleExpiration,
|
||||
LifecycleFilter as GarageLifecycleFilter, LifecycleRule as GarageLifecycleRule,
|
||||
};
|
||||
use garage_model::garage::Garage;
|
||||
use garage_util::data::*;
|
||||
|
||||
pub async fn handle_get_lifecycle(bucket: &Bucket) -> Result<Response<ResBody>, Error> {
|
||||
let param = bucket
|
||||
.params()
|
||||
.ok_or_internal_error("Bucket should not be deleted at this point")?;
|
||||
pub async fn handle_get_lifecycle(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
|
||||
let ReqCtx { bucket_params, .. } = ctx;
|
||||
|
||||
if let Some(lifecycle) = param.lifecycle_config.get() {
|
||||
if let Some(lifecycle) = bucket_params.lifecycle_config.get() {
|
||||
let wc = LifecycleConfiguration::from_garage_lifecycle_config(lifecycle);
|
||||
let xml = to_xml_with_header(&wc)?;
|
||||
Ok(Response::builder()
|
||||
@@ -38,16 +34,18 @@ pub async fn handle_get_lifecycle(bucket: &Bucket) -> Result<Response<ResBody>,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle_delete_lifecycle(
|
||||
garage: Arc<Garage>,
|
||||
mut bucket: Bucket,
|
||||
) -> Result<Response<ResBody>, Error> {
|
||||
let param = bucket
|
||||
.params_mut()
|
||||
.ok_or_internal_error("Bucket should not be deleted at this point")?;
|
||||
|
||||
param.lifecycle_config.update(None);
|
||||
garage.bucket_table.insert(&bucket).await?;
|
||||
pub async fn handle_delete_lifecycle(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
|
||||
let ReqCtx {
|
||||
garage,
|
||||
bucket_id,
|
||||
mut bucket_params,
|
||||
..
|
||||
} = ctx;
|
||||
bucket_params.lifecycle_config.update(None);
|
||||
garage
|
||||
.bucket_table
|
||||
.insert(&Bucket::present(bucket_id, bucket_params))
|
||||
.await?;
|
||||
|
||||
Ok(Response::builder()
|
||||
.status(StatusCode::NO_CONTENT)
|
||||
@@ -55,28 +53,33 @@ pub async fn handle_delete_lifecycle(
|
||||
}
|
||||
|
||||
pub async fn handle_put_lifecycle(
|
||||
garage: Arc<Garage>,
|
||||
mut bucket: Bucket,
|
||||
ctx: ReqCtx,
|
||||
req: Request<ReqBody>,
|
||||
content_sha256: Option<Hash>,
|
||||
) -> Result<Response<ResBody>, Error> {
|
||||
let ReqCtx {
|
||||
garage,
|
||||
bucket_id,
|
||||
mut bucket_params,
|
||||
..
|
||||
} = ctx;
|
||||
|
||||
let body = BodyExt::collect(req.into_body()).await?.to_bytes();
|
||||
|
||||
if let Some(content_sha256) = content_sha256 {
|
||||
verify_signed_content(content_sha256, &body[..])?;
|
||||
}
|
||||
|
||||
let param = bucket
|
||||
.params_mut()
|
||||
.ok_or_internal_error("Bucket should not be deleted at this point")?;
|
||||
|
||||
let conf: LifecycleConfiguration = from_reader(&body as &[u8])?;
|
||||
let config = conf
|
||||
.validate_into_garage_lifecycle_config()
|
||||
.ok_or_bad_request("Invalid lifecycle configuration")?;
|
||||
|
||||
param.lifecycle_config.update(Some(config));
|
||||
garage.bucket_table.insert(&bucket).await?;
|
||||
bucket_params.lifecycle_config.update(Some(config));
|
||||
garage
|
||||
.bucket_table
|
||||
.insert(&Bucket::present(bucket_id, bucket_params))
|
||||
.await?;
|
||||
|
||||
Ok(Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
|
||||
+7
-7
@@ -1,6 +1,5 @@
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::iter::{Iterator, Peekable};
|
||||
use std::sync::Arc;
|
||||
|
||||
use base64::prelude::*;
|
||||
use hyper::Response;
|
||||
@@ -9,7 +8,6 @@ use garage_util::data::*;
|
||||
use garage_util::error::Error as GarageError;
|
||||
use garage_util::time::*;
|
||||
|
||||
use garage_model::garage::Garage;
|
||||
use garage_model::s3::mpu_table::*;
|
||||
use garage_model::s3::object_table::*;
|
||||
|
||||
@@ -62,9 +60,10 @@ pub struct ListPartsQuery {
|
||||
}
|
||||
|
||||
pub async fn handle_list(
|
||||
garage: Arc<Garage>,
|
||||
ctx: ReqCtx,
|
||||
query: &ListObjectsQuery,
|
||||
) -> Result<Response<ResBody>, Error> {
|
||||
let ReqCtx { garage, .. } = &ctx;
|
||||
let io = |bucket, key, count| {
|
||||
let t = &garage.object_table;
|
||||
async move {
|
||||
@@ -167,9 +166,11 @@ pub async fn handle_list(
|
||||
}
|
||||
|
||||
pub async fn handle_list_multipart_upload(
|
||||
garage: Arc<Garage>,
|
||||
ctx: ReqCtx,
|
||||
query: &ListMultipartUploadsQuery,
|
||||
) -> Result<Response<ResBody>, Error> {
|
||||
let ReqCtx { garage, .. } = &ctx;
|
||||
|
||||
let io = |bucket, key, count| {
|
||||
let t = &garage.object_table;
|
||||
async move {
|
||||
@@ -269,15 +270,14 @@ pub async fn handle_list_multipart_upload(
|
||||
}
|
||||
|
||||
pub async fn handle_list_parts(
|
||||
garage: Arc<Garage>,
|
||||
ctx: ReqCtx,
|
||||
query: &ListPartsQuery,
|
||||
) -> Result<Response<ResBody>, Error> {
|
||||
debug!("ListParts {:?}", query);
|
||||
|
||||
let upload_id = s3_multipart::decode_upload_id(&query.upload_id)?;
|
||||
|
||||
let (_, _, mpu) =
|
||||
s3_multipart::get_upload(&garage, &query.bucket_id, &query.key, &upload_id).await?;
|
||||
let (_, _, mpu) = s3_multipart::get_upload(&ctx, &query.key, &upload_id).await?;
|
||||
|
||||
let (info, next) = fetch_part_info(query, &mpu)?;
|
||||
|
||||
|
||||
+39
-29
@@ -8,7 +8,6 @@ use md5::{Digest as Md5Digest, Md5};
|
||||
use garage_table::*;
|
||||
use garage_util::data::*;
|
||||
|
||||
use garage_model::bucket_table::Bucket;
|
||||
use garage_model::garage::Garage;
|
||||
use garage_model::s3::block_ref_table::*;
|
||||
use garage_model::s3::mpu_table::*;
|
||||
@@ -25,12 +24,16 @@ use crate::signature::verify_signed_content;
|
||||
// ----
|
||||
|
||||
pub async fn handle_create_multipart_upload(
|
||||
garage: Arc<Garage>,
|
||||
ctx: ReqCtx,
|
||||
req: &Request<ReqBody>,
|
||||
bucket_name: &str,
|
||||
bucket_id: Uuid,
|
||||
key: &String,
|
||||
) -> Result<Response<ResBody>, Error> {
|
||||
let ReqCtx {
|
||||
garage,
|
||||
bucket_id,
|
||||
bucket_name,
|
||||
..
|
||||
} = &ctx;
|
||||
let existing_object = garage.object_table.get(&bucket_id, &key).await?;
|
||||
|
||||
let upload_id = gen_uuid();
|
||||
@@ -47,13 +50,13 @@ pub async fn handle_create_multipart_upload(
|
||||
headers,
|
||||
},
|
||||
};
|
||||
let object = Object::new(bucket_id, key.to_string(), vec![object_version]);
|
||||
let object = Object::new(*bucket_id, key.to_string(), vec![object_version]);
|
||||
garage.object_table.insert(&object).await?;
|
||||
|
||||
// Create multipart upload in mpu table
|
||||
// This multipart upload will hold references to uploaded parts
|
||||
// (which are entries in the Version table)
|
||||
let mpu = MultipartUpload::new(upload_id, timestamp, bucket_id, key.into(), false);
|
||||
let mpu = MultipartUpload::new(upload_id, timestamp, *bucket_id, key.into(), false);
|
||||
garage.mpu_table.insert(&mpu).await?;
|
||||
|
||||
// Send success response
|
||||
@@ -69,14 +72,15 @@ pub async fn handle_create_multipart_upload(
|
||||
}
|
||||
|
||||
pub async fn handle_put_part(
|
||||
garage: Arc<Garage>,
|
||||
ctx: ReqCtx,
|
||||
req: Request<ReqBody>,
|
||||
bucket_id: Uuid,
|
||||
key: &str,
|
||||
part_number: u64,
|
||||
upload_id: &str,
|
||||
content_sha256: Option<Hash>,
|
||||
) -> Result<Response<ResBody>, Error> {
|
||||
let ReqCtx { garage, .. } = &ctx;
|
||||
|
||||
let upload_id = decode_upload_id(upload_id)?;
|
||||
|
||||
let content_md5 = match req.headers().get("content-md5") {
|
||||
@@ -90,10 +94,8 @@ pub async fn handle_put_part(
|
||||
let stream = body_stream(req.into_body());
|
||||
let mut chunker = StreamChunker::new(stream, garage.config.block_size);
|
||||
|
||||
let ((_, _, mut mpu), first_block) = futures::try_join!(
|
||||
get_upload(&garage, &bucket_id, &key, &upload_id),
|
||||
chunker.next(),
|
||||
)?;
|
||||
let ((_, _, mut mpu), first_block) =
|
||||
futures::try_join!(get_upload(&ctx, &key, &upload_id), chunker.next(),)?;
|
||||
|
||||
// Check object is valid and part can be accepted
|
||||
let first_block = first_block.ok_or_bad_request("Empty body")?;
|
||||
@@ -135,7 +137,7 @@ pub async fn handle_put_part(
|
||||
|
||||
// Copy data to version
|
||||
let (total_size, data_md5sum, data_sha256sum, _) =
|
||||
read_and_put_blocks(&garage, &version, part_number, first_block, &mut chunker).await?;
|
||||
read_and_put_blocks(&ctx, &version, part_number, first_block, &mut chunker).await?;
|
||||
|
||||
// Verify that checksums map
|
||||
ensure_checksum_matches(
|
||||
@@ -200,14 +202,19 @@ impl Drop for InterruptedCleanup {
|
||||
}
|
||||
|
||||
pub async fn handle_complete_multipart_upload(
|
||||
garage: Arc<Garage>,
|
||||
ctx: ReqCtx,
|
||||
req: Request<ReqBody>,
|
||||
bucket_name: &str,
|
||||
bucket: &Bucket,
|
||||
key: &str,
|
||||
upload_id: &str,
|
||||
content_sha256: Option<Hash>,
|
||||
) -> Result<Response<ResBody>, Error> {
|
||||
let ReqCtx {
|
||||
garage,
|
||||
bucket_id,
|
||||
bucket_name,
|
||||
..
|
||||
} = &ctx;
|
||||
|
||||
let body = http_body_util::BodyExt::collect(req.into_body())
|
||||
.await?
|
||||
.to_bytes();
|
||||
@@ -228,8 +235,7 @@ pub async fn handle_complete_multipart_upload(
|
||||
|
||||
// Get object and multipart upload
|
||||
let key = key.to_string();
|
||||
let (object, mut object_version, mpu) =
|
||||
get_upload(&garage, &bucket.id, &key, &upload_id).await?;
|
||||
let (object, mut object_version, mpu) = get_upload(&ctx, &key, &upload_id).await?;
|
||||
|
||||
if mpu.parts.is_empty() {
|
||||
return Err(Error::bad_request("No data was uploaded"));
|
||||
@@ -283,7 +289,7 @@ pub async fn handle_complete_multipart_upload(
|
||||
let mut final_version = Version::new(
|
||||
upload_id,
|
||||
VersionBacklink::Object {
|
||||
bucket_id: bucket.id,
|
||||
bucket_id: *bucket_id,
|
||||
key: key.to_string(),
|
||||
},
|
||||
false,
|
||||
@@ -327,9 +333,9 @@ pub async fn handle_complete_multipart_upload(
|
||||
// Calculate total size of final object
|
||||
let total_size = parts.iter().map(|x| x.size.unwrap()).sum();
|
||||
|
||||
if let Err(e) = check_quotas(&garage, bucket, total_size, Some(&object)).await {
|
||||
if let Err(e) = check_quotas(&ctx, total_size, Some(&object)).await {
|
||||
object_version.state = ObjectVersionState::Aborted;
|
||||
let final_object = Object::new(bucket.id, key.clone(), vec![object_version]);
|
||||
let final_object = Object::new(*bucket_id, key.clone(), vec![object_version]);
|
||||
garage.object_table.insert(&final_object).await?;
|
||||
|
||||
return Err(e);
|
||||
@@ -345,7 +351,7 @@ pub async fn handle_complete_multipart_upload(
|
||||
final_version.blocks.items()[0].1.hash,
|
||||
));
|
||||
|
||||
let final_object = Object::new(bucket.id, key.clone(), vec![object_version]);
|
||||
let final_object = Object::new(*bucket_id, key.clone(), vec![object_version]);
|
||||
garage.object_table.insert(&final_object).await?;
|
||||
|
||||
// Send response saying ok we're done
|
||||
@@ -362,18 +368,20 @@ pub async fn handle_complete_multipart_upload(
|
||||
}
|
||||
|
||||
pub async fn handle_abort_multipart_upload(
|
||||
garage: Arc<Garage>,
|
||||
bucket_id: Uuid,
|
||||
ctx: ReqCtx,
|
||||
key: &str,
|
||||
upload_id: &str,
|
||||
) -> Result<Response<ResBody>, Error> {
|
||||
let ReqCtx {
|
||||
garage, bucket_id, ..
|
||||
} = &ctx;
|
||||
|
||||
let upload_id = decode_upload_id(upload_id)?;
|
||||
|
||||
let (_, mut object_version, _) =
|
||||
get_upload(&garage, &bucket_id, &key.to_string(), &upload_id).await?;
|
||||
let (_, mut object_version, _) = get_upload(&ctx, &key.to_string(), &upload_id).await?;
|
||||
|
||||
object_version.state = ObjectVersionState::Aborted;
|
||||
let final_object = Object::new(bucket_id, key.to_string(), vec![object_version]);
|
||||
let final_object = Object::new(*bucket_id, key.to_string(), vec![object_version]);
|
||||
garage.object_table.insert(&final_object).await?;
|
||||
|
||||
Ok(Response::new(empty_body()))
|
||||
@@ -383,11 +391,13 @@ pub async fn handle_abort_multipart_upload(
|
||||
|
||||
#[allow(clippy::ptr_arg)]
|
||||
pub(crate) async fn get_upload(
|
||||
garage: &Garage,
|
||||
bucket_id: &Uuid,
|
||||
ctx: &ReqCtx,
|
||||
key: &String,
|
||||
upload_id: &Uuid,
|
||||
) -> Result<(Object, ObjectVersion, MultipartUpload), Error> {
|
||||
let ReqCtx {
|
||||
garage, bucket_id, ..
|
||||
} = ctx;
|
||||
let (object, mpu) = futures::try_join!(
|
||||
garage.object_table.get(bucket_id, key).map_err(Error::from),
|
||||
garage
|
||||
|
||||
@@ -120,6 +120,12 @@ pub async fn handle_post_object(
|
||||
.bucket_helper()
|
||||
.get_existing_bucket(bucket_id)
|
||||
.await?;
|
||||
let bucket_params = bucket.state.into_option().unwrap();
|
||||
let matching_cors_rule = find_matching_cors_rule(
|
||||
&bucket_params,
|
||||
&Request::from_parts(head.clone(), empty_body::<Infallible>()),
|
||||
)?
|
||||
.cloned();
|
||||
|
||||
let decoded_policy = BASE64_STANDARD
|
||||
.decode(policy)
|
||||
@@ -213,11 +219,19 @@ pub async fn handle_post_object(
|
||||
let headers = get_headers(¶ms)?;
|
||||
|
||||
let stream = field.map(|r| r.map_err(Into::into));
|
||||
let (_, md5) = save_stream(
|
||||
|
||||
let ctx = ReqCtx {
|
||||
garage,
|
||||
bucket_id,
|
||||
bucket_name,
|
||||
bucket_params,
|
||||
api_key,
|
||||
};
|
||||
|
||||
let (_, md5) = save_stream(
|
||||
&ctx,
|
||||
headers,
|
||||
StreamLimiter::new(stream, conditions.content_length),
|
||||
&bucket,
|
||||
&key,
|
||||
None,
|
||||
None,
|
||||
@@ -234,7 +248,7 @@ pub async fn handle_post_object(
|
||||
{
|
||||
target
|
||||
.query_pairs_mut()
|
||||
.append_pair("bucket", &bucket_name)
|
||||
.append_pair("bucket", &ctx.bucket_name)
|
||||
.append_pair("key", &key)
|
||||
.append_pair("etag", &etag);
|
||||
let target = target.to_string();
|
||||
@@ -278,7 +292,7 @@ pub async fn handle_post_object(
|
||||
let xml = s3_xml::PostObject {
|
||||
xmlns: (),
|
||||
location: s3_xml::Value(location),
|
||||
bucket: s3_xml::Value(bucket_name),
|
||||
bucket: s3_xml::Value(ctx.bucket_name),
|
||||
key: s3_xml::Value(key),
|
||||
etag: s3_xml::Value(etag),
|
||||
};
|
||||
@@ -291,12 +305,8 @@ pub async fn handle_post_object(
|
||||
}
|
||||
};
|
||||
|
||||
let matching_cors_rule = find_matching_cors_rule(
|
||||
&bucket,
|
||||
&Request::from_parts(head, empty_body::<Infallible>()),
|
||||
)?;
|
||||
if let Some(rule) = matching_cors_rule {
|
||||
add_cors_headers(&mut resp, rule)
|
||||
add_cors_headers(&mut resp, &rule)
|
||||
.ok_or_internal_error("Invalid bucket CORS configuration")?;
|
||||
}
|
||||
|
||||
|
||||
+34
-36
@@ -28,7 +28,6 @@ use garage_util::error::Error as GarageError;
|
||||
use garage_util::time::*;
|
||||
|
||||
use garage_block::manager::INLINE_THRESHOLD;
|
||||
use garage_model::bucket_table::Bucket;
|
||||
use garage_model::garage::Garage;
|
||||
use garage_model::index_counter::CountedItem;
|
||||
use garage_model::s3::block_ref_table::*;
|
||||
@@ -42,9 +41,8 @@ use crate::s3::error::*;
|
||||
const PUT_BLOCKS_MAX_PARALLEL: usize = 3;
|
||||
|
||||
pub async fn handle_put(
|
||||
garage: Arc<Garage>,
|
||||
ctx: ReqCtx,
|
||||
req: Request<ReqBody>,
|
||||
bucket: &Bucket,
|
||||
key: &String,
|
||||
content_sha256: Option<Hash>,
|
||||
) -> Result<Response<ResBody>, Error> {
|
||||
@@ -59,35 +57,27 @@ pub async fn handle_put(
|
||||
|
||||
let stream = body_stream(req.into_body());
|
||||
|
||||
save_stream(
|
||||
garage,
|
||||
headers,
|
||||
stream,
|
||||
bucket,
|
||||
key,
|
||||
content_md5,
|
||||
content_sha256,
|
||||
)
|
||||
.await
|
||||
.map(|(uuid, md5)| put_response(uuid, md5))
|
||||
save_stream(&ctx, headers, stream, key, content_md5, content_sha256)
|
||||
.await
|
||||
.map(|(uuid, md5)| put_response(uuid, md5))
|
||||
}
|
||||
|
||||
pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
|
||||
garage: Arc<Garage>,
|
||||
ctx: &ReqCtx,
|
||||
headers: ObjectVersionHeaders,
|
||||
body: S,
|
||||
bucket: &Bucket,
|
||||
key: &String,
|
||||
content_md5: Option<String>,
|
||||
content_sha256: Option<FixedBytes32>,
|
||||
) -> Result<(Uuid, String), Error> {
|
||||
let ReqCtx {
|
||||
garage, bucket_id, ..
|
||||
} = ctx;
|
||||
|
||||
let mut chunker = StreamChunker::new(body, garage.config.block_size);
|
||||
let (first_block_opt, existing_object) = try_join!(
|
||||
chunker.next(),
|
||||
garage
|
||||
.object_table
|
||||
.get(&bucket.id, key)
|
||||
.map_err(Error::from),
|
||||
garage.object_table.get(bucket_id, key).map_err(Error::from),
|
||||
)?;
|
||||
|
||||
let first_block = first_block_opt.unwrap_or_default();
|
||||
@@ -114,7 +104,7 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
|
||||
content_sha256,
|
||||
)?;
|
||||
|
||||
check_quotas(&garage, bucket, size, existing_object.as_ref()).await?;
|
||||
check_quotas(ctx, size, existing_object.as_ref()).await?;
|
||||
|
||||
let object_version = ObjectVersion {
|
||||
uuid: version_uuid,
|
||||
@@ -129,7 +119,7 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
|
||||
)),
|
||||
};
|
||||
|
||||
let object = Object::new(bucket.id, key.into(), vec![object_version]);
|
||||
let object = Object::new(*bucket_id, key.into(), vec![object_version]);
|
||||
garage.object_table.insert(&object).await?;
|
||||
|
||||
return Ok((version_uuid, data_md5sum_hex));
|
||||
@@ -140,7 +130,7 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
|
||||
// before everything is finished (cleanup is done using the Drop trait).
|
||||
let mut interrupted_cleanup = InterruptedCleanup(Some(InterruptedCleanupInner {
|
||||
garage: garage.clone(),
|
||||
bucket_id: bucket.id,
|
||||
bucket_id: *bucket_id,
|
||||
key: key.into(),
|
||||
version_uuid,
|
||||
version_timestamp,
|
||||
@@ -156,7 +146,7 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
|
||||
multipart: false,
|
||||
},
|
||||
};
|
||||
let object = Object::new(bucket.id, key.into(), vec![object_version.clone()]);
|
||||
let object = Object::new(*bucket_id, key.into(), vec![object_version.clone()]);
|
||||
garage.object_table.insert(&object).await?;
|
||||
|
||||
// Initialize corresponding entry in version table
|
||||
@@ -166,7 +156,7 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
|
||||
let version = Version::new(
|
||||
version_uuid,
|
||||
VersionBacklink::Object {
|
||||
bucket_id: bucket.id,
|
||||
bucket_id: *bucket_id,
|
||||
key: key.into(),
|
||||
},
|
||||
false,
|
||||
@@ -175,7 +165,7 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
|
||||
|
||||
// Transfer data and verify checksum
|
||||
let (total_size, data_md5sum, data_sha256sum, first_block_hash) =
|
||||
read_and_put_blocks(&garage, &version, 1, first_block, &mut chunker).await?;
|
||||
read_and_put_blocks(ctx, &version, 1, first_block, &mut chunker).await?;
|
||||
|
||||
ensure_checksum_matches(
|
||||
data_md5sum.as_slice(),
|
||||
@@ -184,7 +174,7 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
|
||||
content_sha256,
|
||||
)?;
|
||||
|
||||
check_quotas(&garage, bucket, total_size, existing_object.as_ref()).await?;
|
||||
check_quotas(ctx, total_size, existing_object.as_ref()).await?;
|
||||
|
||||
// Save final object state, marked as Complete
|
||||
let md5sum_hex = hex::encode(data_md5sum);
|
||||
@@ -196,7 +186,7 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
|
||||
},
|
||||
first_block_hash,
|
||||
));
|
||||
let object = Object::new(bucket.id, key.into(), vec![object_version]);
|
||||
let object = Object::new(*bucket_id, key.into(), vec![object_version]);
|
||||
garage.object_table.insert(&object).await?;
|
||||
|
||||
// We were not interrupted, everything went fine.
|
||||
@@ -235,12 +225,18 @@ pub(crate) fn ensure_checksum_matches(
|
||||
|
||||
/// Check that inserting this object with this size doesn't exceed bucket quotas
|
||||
pub(crate) async fn check_quotas(
|
||||
garage: &Arc<Garage>,
|
||||
bucket: &Bucket,
|
||||
ctx: &ReqCtx,
|
||||
size: u64,
|
||||
prev_object: Option<&Object>,
|
||||
) -> Result<(), Error> {
|
||||
let quotas = bucket.state.as_option().unwrap().quotas.get();
|
||||
let ReqCtx {
|
||||
garage,
|
||||
bucket_id,
|
||||
bucket_params,
|
||||
..
|
||||
} = ctx;
|
||||
|
||||
let quotas = bucket_params.quotas.get();
|
||||
if quotas.max_objects.is_none() && quotas.max_size.is_none() {
|
||||
return Ok(());
|
||||
};
|
||||
@@ -248,7 +244,7 @@ pub(crate) async fn check_quotas(
|
||||
let counters = garage
|
||||
.object_counter_table
|
||||
.table
|
||||
.get(&bucket.id, &EmptyKey)
|
||||
.get(bucket_id, &EmptyKey)
|
||||
.await?;
|
||||
|
||||
let counters = counters
|
||||
@@ -292,7 +288,7 @@ pub(crate) async fn check_quotas(
|
||||
}
|
||||
|
||||
pub(crate) async fn read_and_put_blocks<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
|
||||
garage: &Garage,
|
||||
ctx: &ReqCtx,
|
||||
version: &Version,
|
||||
part_number: u64,
|
||||
first_block: Bytes,
|
||||
@@ -417,7 +413,7 @@ pub(crate) async fn read_and_put_blocks<S: Stream<Item = Result<Bytes, Error>> +
|
||||
let offset = written_bytes;
|
||||
written_bytes += block.len() as u64;
|
||||
write_futs.push_back(put_block_and_meta(
|
||||
garage,
|
||||
ctx,
|
||||
version,
|
||||
part_number,
|
||||
offset,
|
||||
@@ -447,7 +443,7 @@ pub(crate) async fn read_and_put_blocks<S: Stream<Item = Result<Bytes, Error>> +
|
||||
}
|
||||
|
||||
async fn put_block_and_meta(
|
||||
garage: &Garage,
|
||||
ctx: &ReqCtx,
|
||||
version: &Version,
|
||||
part_number: u64,
|
||||
offset: u64,
|
||||
@@ -455,6 +451,8 @@ async fn put_block_and_meta(
|
||||
block: Bytes,
|
||||
order_tag: OrderTag,
|
||||
) -> Result<(), GarageError> {
|
||||
let ReqCtx { garage, .. } = ctx;
|
||||
|
||||
let mut version = version.clone();
|
||||
version.blocks.put(
|
||||
VersionBlockKey {
|
||||
@@ -597,7 +595,7 @@ pub(crate) fn get_headers(headers: &HeaderMap<HeaderValue>) -> Result<ObjectVers
|
||||
// Preserve x-amz-meta- headers
|
||||
for (k, v) in headers.iter() {
|
||||
if k.as_str().starts_with("x-amz-meta-") {
|
||||
match v.to_str() {
|
||||
match std::str::from_utf8(v.as_bytes()) {
|
||||
Ok(v_str) => {
|
||||
other.insert(k.to_string(), v_str.to_string());
|
||||
}
|
||||
|
||||
+28
-26
@@ -1,5 +1,4 @@
|
||||
use quick_xml::de::from_reader;
|
||||
use std::sync::Arc;
|
||||
|
||||
use http_body_util::BodyExt;
|
||||
use hyper::{Request, Response, StatusCode};
|
||||
@@ -12,15 +11,11 @@ use crate::s3::xml::{to_xml_with_header, xmlns_tag, IntValue, Value};
|
||||
use crate::signature::verify_signed_content;
|
||||
|
||||
use garage_model::bucket_table::*;
|
||||
use garage_model::garage::Garage;
|
||||
use garage_util::data::*;
|
||||
|
||||
pub async fn handle_get_website(bucket: &Bucket) -> Result<Response<ResBody>, Error> {
|
||||
let param = bucket
|
||||
.params()
|
||||
.ok_or_internal_error("Bucket should not be deleted at this point")?;
|
||||
|
||||
if let Some(website) = param.website_config.get() {
|
||||
pub async fn handle_get_website(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
|
||||
let ReqCtx { bucket_params, .. } = ctx;
|
||||
if let Some(website) = bucket_params.website_config.get() {
|
||||
let wc = WebsiteConfiguration {
|
||||
xmlns: (),
|
||||
error_document: website.error_document.as_ref().map(|v| Key {
|
||||
@@ -44,16 +39,18 @@ pub async fn handle_get_website(bucket: &Bucket) -> Result<Response<ResBody>, Er
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle_delete_website(
|
||||
garage: Arc<Garage>,
|
||||
mut bucket: Bucket,
|
||||
) -> Result<Response<ResBody>, Error> {
|
||||
let param = bucket
|
||||
.params_mut()
|
||||
.ok_or_internal_error("Bucket should not be deleted at this point")?;
|
||||
|
||||
param.website_config.update(None);
|
||||
garage.bucket_table.insert(&bucket).await?;
|
||||
pub async fn handle_delete_website(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
|
||||
let ReqCtx {
|
||||
garage,
|
||||
bucket_id,
|
||||
mut bucket_params,
|
||||
..
|
||||
} = ctx;
|
||||
bucket_params.website_config.update(None);
|
||||
garage
|
||||
.bucket_table
|
||||
.insert(&Bucket::present(bucket_id, bucket_params))
|
||||
.await?;
|
||||
|
||||
Ok(Response::builder()
|
||||
.status(StatusCode::NO_CONTENT)
|
||||
@@ -61,28 +58,33 @@ pub async fn handle_delete_website(
|
||||
}
|
||||
|
||||
pub async fn handle_put_website(
|
||||
garage: Arc<Garage>,
|
||||
mut bucket: Bucket,
|
||||
ctx: ReqCtx,
|
||||
req: Request<ReqBody>,
|
||||
content_sha256: Option<Hash>,
|
||||
) -> Result<Response<ResBody>, Error> {
|
||||
let ReqCtx {
|
||||
garage,
|
||||
bucket_id,
|
||||
mut bucket_params,
|
||||
..
|
||||
} = ctx;
|
||||
|
||||
let body = BodyExt::collect(req.into_body()).await?.to_bytes();
|
||||
|
||||
if let Some(content_sha256) = content_sha256 {
|
||||
verify_signed_content(content_sha256, &body[..])?;
|
||||
}
|
||||
|
||||
let param = bucket
|
||||
.params_mut()
|
||||
.ok_or_internal_error("Bucket should not be deleted at this point")?;
|
||||
|
||||
let conf: WebsiteConfiguration = from_reader(&body as &[u8])?;
|
||||
conf.validate()?;
|
||||
|
||||
param
|
||||
bucket_params
|
||||
.website_config
|
||||
.update(Some(conf.into_garage_website_config()?));
|
||||
garage.bucket_table.insert(&bucket).await?;
|
||||
garage
|
||||
.bucket_table
|
||||
.insert(&Bucket::present(bucket_id, bucket_params))
|
||||
.await?;
|
||||
|
||||
Ok(Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
|
||||
@@ -3,7 +3,7 @@ use std::convert::TryFrom;
|
||||
|
||||
use chrono::{DateTime, Duration, NaiveDateTime, TimeZone, Utc};
|
||||
use hmac::Mac;
|
||||
use hyper::header::{HeaderMap, HeaderName, HeaderValue, AUTHORIZATION, CONTENT_TYPE, HOST};
|
||||
use hyper::header::{HeaderMap, HeaderName, HeaderValue, AUTHORIZATION, HOST};
|
||||
use hyper::{body::Incoming as IncomingBody, Method, Request};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
@@ -31,7 +31,13 @@ pub const AWS4_HMAC_SHA256: &str = "AWS4-HMAC-SHA256";
|
||||
pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
|
||||
pub const STREAMING_AWS4_HMAC_SHA256_PAYLOAD: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD";
|
||||
|
||||
pub type QueryMap = HashMap<String, String>;
|
||||
pub type QueryMap = HeaderMap<QueryValue>;
|
||||
pub struct QueryValue {
|
||||
/// Original key with potential uppercase characters,
|
||||
/// for use in signature calculation
|
||||
key: String,
|
||||
value: String,
|
||||
}
|
||||
|
||||
pub async fn check_payload_signature(
|
||||
garage: &Garage,
|
||||
@@ -40,7 +46,7 @@ pub async fn check_payload_signature(
|
||||
) -> Result<(Option<Key>, Option<Hash>), Error> {
|
||||
let query = parse_query_map(request.uri())?;
|
||||
|
||||
if query.contains_key(X_AMZ_ALGORITHM.as_str()) {
|
||||
if query.contains_key(&X_AMZ_ALGORITHM) {
|
||||
// We check for presigned-URL-style authentification first, because
|
||||
// the browser or someting else could inject an Authorization header
|
||||
// that is totally unrelated to AWS signatures.
|
||||
@@ -74,12 +80,13 @@ async fn check_standard_signature(
|
||||
let authorization = Authorization::parse_header(request.headers())?;
|
||||
|
||||
// Verify that all necessary request headers are included in signed_headers
|
||||
// For standard AWSv4 signatures, the following must be included:
|
||||
// The following must be included for all signatures:
|
||||
// - the Host header (mandatory)
|
||||
// - the Content-Type header, if it is used in the request
|
||||
// - all x-amz-* headers used in the request
|
||||
// AWS also indicates that the Content-Type header should be signed if
|
||||
// it is used, but Minio client doesn't sign it so we don't check it for compatibility.
|
||||
let signed_headers = split_signed_headers(&authorization)?;
|
||||
verify_signed_headers(request.headers(), &signed_headers, &[CONTENT_TYPE])?;
|
||||
verify_signed_headers(request.headers(), &signed_headers)?;
|
||||
|
||||
let canonical_request = canonical_request(
|
||||
service,
|
||||
@@ -121,21 +128,21 @@ async fn check_presigned_signature(
|
||||
request: &mut Request<IncomingBody>,
|
||||
mut query: QueryMap,
|
||||
) -> Result<(Option<Key>, Option<Hash>), Error> {
|
||||
let algorithm = query.get(X_AMZ_ALGORITHM.as_str()).unwrap();
|
||||
let authorization = Authorization::parse_presigned(algorithm, &query)?;
|
||||
let algorithm = query.get(&X_AMZ_ALGORITHM).unwrap();
|
||||
let authorization = Authorization::parse_presigned(&algorithm.value, &query)?;
|
||||
|
||||
// Verify that all necessary request headers are included in signed_headers
|
||||
// For AWSv4 pre-signed URLs, the following must be incldued:
|
||||
// - the Host header (mandatory)
|
||||
// - all x-amz-* headers used in the request
|
||||
let signed_headers = split_signed_headers(&authorization)?;
|
||||
verify_signed_headers(request.headers(), &signed_headers, &[])?;
|
||||
verify_signed_headers(request.headers(), &signed_headers)?;
|
||||
|
||||
// The X-Amz-Signature value is passed as a query parameter,
|
||||
// but the signature cannot be computed from a string that contains itself.
|
||||
// AWS specifies that all query params except X-Amz-Signature are included
|
||||
// in the canonical request.
|
||||
query.remove(X_AMZ_SIGNATURE.as_str());
|
||||
query.remove(&X_AMZ_SIGNATURE);
|
||||
let canonical_request = canonical_request(
|
||||
service,
|
||||
request.method(),
|
||||
@@ -161,10 +168,8 @@ async fn check_presigned_signature(
|
||||
// then an InvalidRequest error is raised.
|
||||
let headers_mut = request.headers_mut();
|
||||
for (name, value) in query.iter() {
|
||||
let name =
|
||||
HeaderName::from_bytes(name.as_bytes()).ok_or_bad_request("Invalid header name")?;
|
||||
if let Some(existing) = headers_mut.get(&name) {
|
||||
if signed_headers.contains(&name) && existing.as_bytes() != value.as_bytes() {
|
||||
if let Some(existing) = headers_mut.get(name) {
|
||||
if signed_headers.contains(&name) && existing.as_bytes() != value.value.as_bytes() {
|
||||
return Err(Error::bad_request(format!(
|
||||
"Conflicting values for `{}` in query parameters and request headers",
|
||||
name
|
||||
@@ -180,7 +185,7 @@ async fn check_presigned_signature(
|
||||
// that are not signed, however there is not much reason that this would happen)
|
||||
headers_mut.insert(
|
||||
name,
|
||||
HeaderValue::from_bytes(value.as_bytes())
|
||||
HeaderValue::from_bytes(value.value.as_bytes())
|
||||
.ok_or_bad_request("invalid query parameter value")?,
|
||||
);
|
||||
}
|
||||
@@ -192,11 +197,19 @@ async fn check_presigned_signature(
|
||||
}
|
||||
|
||||
pub fn parse_query_map(uri: &http::uri::Uri) -> Result<QueryMap, Error> {
|
||||
let mut query = QueryMap::new();
|
||||
let mut query = QueryMap::with_capacity(0);
|
||||
if let Some(query_str) = uri.query() {
|
||||
let query_pairs = url::form_urlencoded::parse(query_str.as_bytes());
|
||||
for (key, val) in query_pairs {
|
||||
if query.insert(key.to_string(), val.into_owned()).is_some() {
|
||||
let name =
|
||||
HeaderName::from_bytes(key.as_bytes()).ok_or_bad_request("Invalid header name")?;
|
||||
|
||||
let value = QueryValue {
|
||||
key: key.to_string(),
|
||||
value: val.into_owned(),
|
||||
};
|
||||
|
||||
if query.insert(name, value).is_some() {
|
||||
return Err(Error::bad_request(format!(
|
||||
"duplicate query parameter: `{}`",
|
||||
key
|
||||
@@ -229,16 +242,12 @@ fn split_signed_headers(authorization: &Authorization) -> Result<Vec<HeaderName>
|
||||
Ok(signed_headers)
|
||||
}
|
||||
|
||||
fn verify_signed_headers(
|
||||
headers: &HeaderMap,
|
||||
signed_headers: &[HeaderName],
|
||||
extra_headers: &[HeaderName],
|
||||
) -> Result<(), Error> {
|
||||
fn verify_signed_headers(headers: &HeaderMap, signed_headers: &[HeaderName]) -> Result<(), Error> {
|
||||
if !signed_headers.contains(&HOST) {
|
||||
return Err(Error::bad_request("Header `Host` should be signed"));
|
||||
}
|
||||
for (name, _) in headers.iter() {
|
||||
if name.as_str().starts_with("x-amz-") || extra_headers.contains(name) {
|
||||
if name.as_str().starts_with("x-amz-") {
|
||||
if !signed_headers.contains(name) {
|
||||
return Err(Error::bad_request(format!(
|
||||
"Header `{}` should be signed",
|
||||
@@ -309,7 +318,7 @@ pub fn canonical_request(
|
||||
// Canonical query string from passed HeaderMap
|
||||
let canonical_query_string = {
|
||||
let mut items = Vec::with_capacity(query.len());
|
||||
for (key, value) in query.iter() {
|
||||
for (_, QueryValue { key, value }) in query.iter() {
|
||||
items.push(uri_encode(&key, true) + "=" + &uri_encode(&value, true));
|
||||
}
|
||||
items.sort();
|
||||
@@ -322,8 +331,8 @@ pub fn canonical_request(
|
||||
.map(|name| {
|
||||
let value = headers
|
||||
.get(name)
|
||||
.ok_or_bad_request(format!("signed header `{}` is not present", name))?
|
||||
.to_str()?;
|
||||
.ok_or_bad_request(format!("signed header `{}` is not present", name))?;
|
||||
let value = std::str::from_utf8(value.as_bytes())?;
|
||||
Ok(format!("{}:{}", name.as_str(), value.trim()))
|
||||
})
|
||||
.collect::<Result<Vec<String>, Error>>()?
|
||||
@@ -467,18 +476,19 @@ impl Authorization {
|
||||
}
|
||||
|
||||
let cred = query
|
||||
.get(X_AMZ_CREDENTIAL.as_str())
|
||||
.get(&X_AMZ_CREDENTIAL)
|
||||
.ok_or_bad_request("X-Amz-Credential not found in query parameters")?;
|
||||
let signed_headers = query
|
||||
.get(X_AMZ_SIGNEDHEADERS.as_str())
|
||||
.get(&X_AMZ_SIGNEDHEADERS)
|
||||
.ok_or_bad_request("X-Amz-SignedHeaders not found in query parameters")?;
|
||||
let signature = query
|
||||
.get(X_AMZ_SIGNATURE.as_str())
|
||||
.get(&X_AMZ_SIGNATURE)
|
||||
.ok_or_bad_request("X-Amz-Signature not found in query parameters")?;
|
||||
|
||||
let duration = query
|
||||
.get(X_AMZ_EXPIRES.as_str())
|
||||
.get(&X_AMZ_EXPIRES)
|
||||
.ok_or_bad_request("X-Amz-Expires not found in query parameters")?
|
||||
.value
|
||||
.parse()
|
||||
.map_err(|_| Error::bad_request("X-Amz-Expires is not a number".to_string()))?;
|
||||
|
||||
@@ -489,20 +499,20 @@ impl Authorization {
|
||||
}
|
||||
|
||||
let date = query
|
||||
.get(X_AMZ_DATE.as_str())
|
||||
.get(&X_AMZ_DATE)
|
||||
.ok_or_bad_request("Missing X-Amz-Date field")?;
|
||||
let date = parse_date(date)?;
|
||||
let date = parse_date(&date.value)?;
|
||||
|
||||
if Utc::now() - date > Duration::seconds(duration) {
|
||||
return Err(Error::bad_request("Date is too old".to_string()));
|
||||
}
|
||||
|
||||
let (key_id, scope) = parse_credential(cred)?;
|
||||
let (key_id, scope) = parse_credential(&cred.value)?;
|
||||
Ok(Authorization {
|
||||
key_id,
|
||||
scope,
|
||||
signed_headers: signed_headers.to_string(),
|
||||
signature: signature.to_string(),
|
||||
signed_headers: signed_headers.value.clone(),
|
||||
signature: signature.value.clone(),
|
||||
content_sha256: UNSIGNED_PAYLOAD.to_string(),
|
||||
date,
|
||||
})
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_block"
|
||||
version = "0.9.1"
|
||||
version = "0.9.4"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
|
||||
+48
-12
@@ -1,3 +1,4 @@
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -13,9 +14,12 @@ const DRIVE_NPART: usize = 1024;
|
||||
|
||||
const HASH_DRIVE_BYTES: (usize, usize) = (2, 3);
|
||||
|
||||
const MARKER_FILE_NAME: &str = "garage-marker";
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub(crate) struct DataLayout {
|
||||
pub(crate) data_dirs: Vec<DataDir>,
|
||||
markers: HashMap<PathBuf, String>,
|
||||
|
||||
/// Primary storage location (index in data_dirs) for each partition
|
||||
/// = the location where the data is supposed to be, blocks are always
|
||||
@@ -75,16 +79,17 @@ impl DataLayout {
|
||||
|
||||
Ok(Self {
|
||||
data_dirs,
|
||||
markers: HashMap::new(),
|
||||
part_prim,
|
||||
part_sec,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn update(&mut self, dirs: &DataDirEnum) -> Result<(), Error> {
|
||||
pub(crate) fn update(self, dirs: &DataDirEnum) -> Result<Self, Error> {
|
||||
// Make list of new data directories, exit if nothing changed
|
||||
let data_dirs = make_data_dirs(dirs)?;
|
||||
if data_dirs == self.data_dirs {
|
||||
return Ok(());
|
||||
return Ok(self);
|
||||
}
|
||||
|
||||
let total_cap = data_dirs.iter().filter_map(|x| x.capacity()).sum::<u64>();
|
||||
@@ -214,11 +219,43 @@ impl DataLayout {
|
||||
}
|
||||
|
||||
// Apply newly generated config
|
||||
*self = Self {
|
||||
Ok(Self {
|
||||
data_dirs,
|
||||
markers: self.markers,
|
||||
part_prim,
|
||||
part_sec,
|
||||
};
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn check_markers(&mut self) -> Result<(), Error> {
|
||||
let data_dirs = &self.data_dirs;
|
||||
self.markers
|
||||
.retain(|k, _| data_dirs.iter().any(|x| x.path == *k));
|
||||
|
||||
for dir in self.data_dirs.iter() {
|
||||
let mut marker_path = dir.path.clone();
|
||||
marker_path.push(MARKER_FILE_NAME);
|
||||
let existing_marker = std::fs::read_to_string(&marker_path).ok();
|
||||
match (existing_marker, self.markers.get(&dir.path)) {
|
||||
(Some(m1), Some(m2)) => {
|
||||
if m1 != *m2 {
|
||||
return Err(Error::Message(format!("Mismatched content for marker file `{}` in data directory `{}`. If you moved data directories or changed their mountpoints, you should remove the `data_layout` file in Garage's metadata directory and restart Garage.", MARKER_FILE_NAME, dir.path.display())));
|
||||
}
|
||||
}
|
||||
(None, Some(_)) => {
|
||||
return Err(Error::Message(format!("Could not find expected marker file `{}` in data directory `{}`, make sure this data directory is mounted correctly.", MARKER_FILE_NAME, dir.path.display())));
|
||||
}
|
||||
(Some(mkr), None) => {
|
||||
self.markers.insert(dir.path.clone(), mkr);
|
||||
}
|
||||
(None, None) => {
|
||||
let mkr = hex::encode(garage_util::data::gen_uuid().as_slice());
|
||||
std::fs::write(&marker_path, &mkr)?;
|
||||
self.markers.insert(dir.path.clone(), mkr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -255,6 +292,7 @@ impl DataLayout {
|
||||
pub(crate) fn without_secondary_locations(&self) -> Self {
|
||||
Self {
|
||||
data_dirs: self.data_dirs.clone(),
|
||||
markers: self.markers.clone(),
|
||||
part_prim: self.part_prim.clone(),
|
||||
part_sec: self.part_sec.iter().map(|_| vec![]).collect::<Vec<_>>(),
|
||||
}
|
||||
@@ -322,14 +360,12 @@ fn make_data_dirs(dirs: &DataDirEnum) -> Result<Vec<DataDir>, Error> {
|
||||
fn dir_not_empty(path: &PathBuf) -> Result<bool, Error> {
|
||||
for entry in std::fs::read_dir(&path)? {
|
||||
let dir = entry?;
|
||||
if dir.file_type()?.is_dir()
|
||||
&& dir
|
||||
.file_name()
|
||||
.into_string()
|
||||
.ok()
|
||||
.and_then(|hex| hex::decode(&hex).ok())
|
||||
.is_some()
|
||||
{
|
||||
let ft = dir.file_type()?;
|
||||
let name = dir.file_name().into_string().ok();
|
||||
if ft.is_file() && name.as_deref() == Some(MARKER_FILE_NAME) {
|
||||
return Ok(true);
|
||||
}
|
||||
if ft.is_dir() && name.and_then(|hex| hex::decode(&hex).ok()).is_some() {
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
|
||||
+71
-42
@@ -1,3 +1,4 @@
|
||||
use std::convert::TryInto;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
@@ -10,7 +11,7 @@ use serde::{Deserialize, Serialize};
|
||||
|
||||
use tokio::fs;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
|
||||
use tokio::sync::{mpsc, Mutex, MutexGuard};
|
||||
use tokio::sync::{mpsc, Mutex, MutexGuard, Semaphore};
|
||||
|
||||
use opentelemetry::{
|
||||
trace::{FutureExt as OtelFutureExt, TraceContextExt, Tracer},
|
||||
@@ -22,7 +23,7 @@ use garage_net::stream::{read_stream_to_end, stream_asyncread, ByteStream};
|
||||
use garage_db as db;
|
||||
|
||||
use garage_util::background::{vars, BackgroundRunner};
|
||||
use garage_util::config::DataDirEnum;
|
||||
use garage_util::config::Config;
|
||||
use garage_util::data::*;
|
||||
use garage_util::error::*;
|
||||
use garage_util::metrics::RecordDuration;
|
||||
@@ -84,6 +85,7 @@ pub struct BlockManager {
|
||||
|
||||
data_fsync: bool,
|
||||
compression_level: Option<i32>,
|
||||
disable_scrub: bool,
|
||||
|
||||
mutation_lock: Vec<Mutex<BlockManagerLocked>>,
|
||||
|
||||
@@ -92,6 +94,7 @@ pub struct BlockManager {
|
||||
|
||||
pub(crate) system: Arc<System>,
|
||||
pub(crate) endpoint: Arc<Endpoint<BlockRpc, Self>>,
|
||||
buffer_kb_semaphore: Arc<Semaphore>,
|
||||
|
||||
pub(crate) metrics: BlockManagerMetrics,
|
||||
|
||||
@@ -119,24 +122,22 @@ struct BlockManagerLocked();
|
||||
impl BlockManager {
|
||||
pub fn new(
|
||||
db: &db::Db,
|
||||
data_dir: DataDirEnum,
|
||||
data_fsync: bool,
|
||||
compression_level: Option<i32>,
|
||||
config: &Config,
|
||||
replication: TableShardedReplication,
|
||||
system: Arc<System>,
|
||||
) -> Result<Arc<Self>, Error> {
|
||||
// Load or compute layout, i.e. assignment of data blocks to the different data directories
|
||||
let data_layout_persister: Persister<DataLayout> =
|
||||
Persister::new(&system.metadata_dir, "data_layout");
|
||||
let data_layout = match data_layout_persister.load() {
|
||||
Ok(mut layout) => {
|
||||
layout
|
||||
.update(&data_dir)
|
||||
.ok_or_message("invalid data_dir config")?;
|
||||
layout
|
||||
let mut data_layout = match data_layout_persister.load() {
|
||||
Ok(layout) => layout
|
||||
.update(&config.data_dir)
|
||||
.ok_or_message("invalid data_dir config")?,
|
||||
Err(_) => {
|
||||
DataLayout::initialize(&config.data_dir).ok_or_message("invalid data_dir config")?
|
||||
}
|
||||
Err(_) => DataLayout::initialize(&data_dir).ok_or_message("invalid data_dir config")?,
|
||||
};
|
||||
data_layout.check_markers()?;
|
||||
data_layout_persister
|
||||
.save(&data_layout)
|
||||
.expect("cannot save data_layout");
|
||||
@@ -153,11 +154,14 @@ impl BlockManager {
|
||||
.netapp
|
||||
.endpoint("garage_block/manager.rs/Rpc".to_string());
|
||||
|
||||
let buffer_kb_semaphore = Arc::new(Semaphore::new(config.block_ram_buffer_max / 1024));
|
||||
|
||||
let metrics = BlockManagerMetrics::new(
|
||||
compression_level,
|
||||
config.compression_level,
|
||||
rc.rc.clone(),
|
||||
resync.queue.clone(),
|
||||
resync.errors.clone(),
|
||||
buffer_kb_semaphore.clone(),
|
||||
);
|
||||
|
||||
let scrub_persister = PersisterShared::new(&system.metadata_dir, "scrub_info");
|
||||
@@ -166,8 +170,9 @@ impl BlockManager {
|
||||
replication,
|
||||
data_layout: ArcSwap::new(Arc::new(data_layout)),
|
||||
data_layout_persister,
|
||||
data_fsync,
|
||||
compression_level,
|
||||
data_fsync: config.data_fsync,
|
||||
disable_scrub: config.disable_scrub,
|
||||
compression_level: config.compression_level,
|
||||
mutation_lock: vec![(); MUTEX_COUNT]
|
||||
.iter()
|
||||
.map(|_| Mutex::new(BlockManagerLocked()))
|
||||
@@ -176,6 +181,7 @@ impl BlockManager {
|
||||
resync,
|
||||
system,
|
||||
endpoint,
|
||||
buffer_kb_semaphore,
|
||||
metrics,
|
||||
scrub_persister,
|
||||
tx_scrub_command: ArcSwapOption::new(None),
|
||||
@@ -194,33 +200,37 @@ impl BlockManager {
|
||||
}
|
||||
|
||||
// Spawn scrub worker
|
||||
let (scrub_tx, scrub_rx) = mpsc::channel(1);
|
||||
self.tx_scrub_command.store(Some(Arc::new(scrub_tx)));
|
||||
bg.spawn_worker(ScrubWorker::new(
|
||||
self.clone(),
|
||||
scrub_rx,
|
||||
self.scrub_persister.clone(),
|
||||
));
|
||||
if !self.disable_scrub {
|
||||
let (scrub_tx, scrub_rx) = mpsc::channel(1);
|
||||
self.tx_scrub_command.store(Some(Arc::new(scrub_tx)));
|
||||
bg.spawn_worker(ScrubWorker::new(
|
||||
self.clone(),
|
||||
scrub_rx,
|
||||
self.scrub_persister.clone(),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
pub fn register_bg_vars(&self, vars: &mut vars::BgVars) {
|
||||
self.resync.register_bg_vars(vars);
|
||||
|
||||
vars.register_rw(
|
||||
&self.scrub_persister,
|
||||
"scrub-tranquility",
|
||||
|p| p.get_with(|x| x.tranquility),
|
||||
|p, tranquility| p.set_with(|x| x.tranquility = tranquility),
|
||||
);
|
||||
vars.register_ro(&self.scrub_persister, "scrub-last-completed", |p| {
|
||||
p.get_with(|x| msec_to_rfc3339(x.time_last_complete_scrub))
|
||||
});
|
||||
vars.register_ro(&self.scrub_persister, "scrub-next-run", |p| {
|
||||
p.get_with(|x| msec_to_rfc3339(x.time_next_run_scrub))
|
||||
});
|
||||
vars.register_ro(&self.scrub_persister, "scrub-corruptions_detected", |p| {
|
||||
p.get_with(|x| x.corruptions_detected)
|
||||
});
|
||||
if !self.disable_scrub {
|
||||
vars.register_rw(
|
||||
&self.scrub_persister,
|
||||
"scrub-tranquility",
|
||||
|p| p.get_with(|x| x.tranquility),
|
||||
|p, tranquility| p.set_with(|x| x.tranquility = tranquility),
|
||||
);
|
||||
vars.register_ro(&self.scrub_persister, "scrub-last-completed", |p| {
|
||||
p.get_with(|x| msec_to_rfc3339(x.time_last_complete_scrub))
|
||||
});
|
||||
vars.register_ro(&self.scrub_persister, "scrub-next-run", |p| {
|
||||
p.get_with(|x| msec_to_rfc3339(x.time_next_run_scrub))
|
||||
});
|
||||
vars.register_ro(&self.scrub_persister, "scrub-corruptions_detected", |p| {
|
||||
p.get_with(|x| x.corruptions_detected)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// Ask nodes that might have a (possibly compressed) block for it
|
||||
@@ -228,10 +238,16 @@ impl BlockManager {
|
||||
async fn rpc_get_raw_block_streaming(
|
||||
&self,
|
||||
hash: &Hash,
|
||||
priority: RequestPriority,
|
||||
order_tag: Option<OrderTag>,
|
||||
) -> Result<DataBlockStream, Error> {
|
||||
self.rpc_get_raw_block_internal(hash, order_tag, |stream| async move { Ok(stream) })
|
||||
.await
|
||||
self.rpc_get_raw_block_internal(
|
||||
hash,
|
||||
priority,
|
||||
order_tag,
|
||||
|stream| async move { Ok(stream) },
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Ask nodes that might have a (possibly compressed) block for it
|
||||
@@ -239,9 +255,10 @@ impl BlockManager {
|
||||
pub(crate) async fn rpc_get_raw_block(
|
||||
&self,
|
||||
hash: &Hash,
|
||||
priority: RequestPriority,
|
||||
order_tag: Option<OrderTag>,
|
||||
) -> Result<DataBlock, Error> {
|
||||
self.rpc_get_raw_block_internal(hash, order_tag, |block_stream| async move {
|
||||
self.rpc_get_raw_block_internal(hash, priority, order_tag, |block_stream| async move {
|
||||
let (header, stream) = block_stream.into_parts();
|
||||
read_stream_to_end(stream)
|
||||
.await
|
||||
@@ -254,6 +271,7 @@ impl BlockManager {
|
||||
async fn rpc_get_raw_block_internal<F, Fut, T>(
|
||||
&self,
|
||||
hash: &Hash,
|
||||
priority: RequestPriority,
|
||||
order_tag: Option<OrderTag>,
|
||||
f: F,
|
||||
) -> Result<T, Error>
|
||||
@@ -269,7 +287,7 @@ impl BlockManager {
|
||||
let rpc = self.endpoint.call_streaming(
|
||||
&node_id,
|
||||
BlockRpc::GetBlock(*hash, order_tag),
|
||||
PRIO_NORMAL | PRIO_SECONDARY,
|
||||
priority,
|
||||
);
|
||||
tokio::select! {
|
||||
res = rpc => {
|
||||
@@ -321,7 +339,9 @@ impl BlockManager {
|
||||
hash: &Hash,
|
||||
order_tag: Option<OrderTag>,
|
||||
) -> Result<ByteStream, Error> {
|
||||
let block_stream = self.rpc_get_raw_block_streaming(hash, order_tag).await?;
|
||||
let block_stream = self
|
||||
.rpc_get_raw_block_streaming(hash, PRIO_NORMAL | PRIO_SECONDARY, order_tag)
|
||||
.await?;
|
||||
let (header, stream) = block_stream.into_parts();
|
||||
match header {
|
||||
DataBlockHeader::Plain => Ok(stream),
|
||||
@@ -357,6 +377,14 @@ impl BlockManager {
|
||||
let (header, bytes) = DataBlock::from_buffer(data, self.compression_level)
|
||||
.await
|
||||
.into_parts();
|
||||
|
||||
let permit = self
|
||||
.buffer_kb_semaphore
|
||||
.clone()
|
||||
.acquire_many_owned((bytes.len() / 1024).try_into().unwrap())
|
||||
.await
|
||||
.ok_or_message("could not reserve space for buffer of data to send to remote nodes")?;
|
||||
|
||||
let put_block_rpc =
|
||||
Req::new(BlockRpc::PutBlock { hash, header })?.with_stream_from_buffer(bytes);
|
||||
let put_block_rpc = if let Some(tag) = order_tag {
|
||||
@@ -372,6 +400,7 @@ impl BlockManager {
|
||||
&who[..],
|
||||
put_block_rpc,
|
||||
RequestStrategy::with_priority(PRIO_NORMAL | PRIO_SECONDARY)
|
||||
.with_drop_on_completion(permit)
|
||||
.with_quorum(self.replication.write_quorum()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
use opentelemetry::{global, metrics::*};
|
||||
|
||||
use garage_db as db;
|
||||
@@ -9,6 +13,7 @@ pub struct BlockManagerMetrics {
|
||||
pub(crate) _rc_size: ValueObserver<u64>,
|
||||
pub(crate) _resync_queue_len: ValueObserver<u64>,
|
||||
pub(crate) _resync_errored_blocks: ValueObserver<u64>,
|
||||
pub(crate) _buffer_free_kb: ValueObserver<u64>,
|
||||
|
||||
pub(crate) resync_counter: BoundCounter<u64>,
|
||||
pub(crate) resync_error_counter: BoundCounter<u64>,
|
||||
@@ -31,6 +36,7 @@ impl BlockManagerMetrics {
|
||||
rc_tree: db::Tree,
|
||||
resync_queue: CountedTree,
|
||||
resync_errors: CountedTree,
|
||||
buffer_semaphore: Arc<Semaphore>,
|
||||
) -> Self {
|
||||
let meter = global::meter("garage_model/block");
|
||||
Self {
|
||||
@@ -66,6 +72,15 @@ impl BlockManagerMetrics {
|
||||
.with_description("Number of block hashes whose last resync resulted in an error")
|
||||
.init(),
|
||||
|
||||
_buffer_free_kb: meter
|
||||
.u64_value_observer("block.ram_buffer_free_kb", move |observer| {
|
||||
observer.observe(buffer_semaphore.available_permits() as u64, &[])
|
||||
})
|
||||
.with_description(
|
||||
"Available RAM in KiB to use for buffering data blocks to be written to remote nodes",
|
||||
)
|
||||
.init(),
|
||||
|
||||
resync_counter: meter
|
||||
.u64_counter("block.resync_counter")
|
||||
.with_description("Number of calls to resync_block")
|
||||
|
||||
+4
-2
@@ -436,7 +436,7 @@ impl BlockResyncManager {
|
||||
&manager.endpoint,
|
||||
&need_nodes[..],
|
||||
put_block_message,
|
||||
RequestStrategy::with_priority(PRIO_BACKGROUND)
|
||||
RequestStrategy::with_priority(PRIO_BACKGROUND | PRIO_SECONDARY)
|
||||
.with_quorum(need_nodes.len()),
|
||||
)
|
||||
.await
|
||||
@@ -460,7 +460,9 @@ impl BlockResyncManager {
|
||||
hash
|
||||
);
|
||||
|
||||
let block_data = manager.rpc_get_raw_block(hash, None).await?;
|
||||
let block_data = manager
|
||||
.rpc_get_raw_block(hash, PRIO_BACKGROUND | PRIO_SECONDARY, None)
|
||||
.await?;
|
||||
|
||||
manager.metrics.resync_recv_counter.add(1);
|
||||
|
||||
|
||||
+5
-3
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_db"
|
||||
version = "0.9.1"
|
||||
version = "0.9.4"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
@@ -17,7 +17,9 @@ hexdump.workspace = true
|
||||
tracing.workspace = true
|
||||
|
||||
heed = { workspace = true, optional = true }
|
||||
rusqlite = { workspace = true, optional = true }
|
||||
rusqlite = { workspace = true, optional = true, features = ["backup"] }
|
||||
r2d2 = { workspace = true, optional = true }
|
||||
r2d2_sqlite = { workspace = true, optional = true }
|
||||
sled = { workspace = true, optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
@@ -27,4 +29,4 @@ mktemp.workspace = true
|
||||
default = [ "sled", "lmdb", "sqlite" ]
|
||||
bundled-libs = [ "rusqlite?/bundled" ]
|
||||
lmdb = [ "heed" ]
|
||||
sqlite = [ "rusqlite" ]
|
||||
sqlite = [ "rusqlite", "r2d2", "r2d2_sqlite" ]
|
||||
|
||||
+16
-43
@@ -1,5 +1,4 @@
|
||||
#[macro_use]
|
||||
#[cfg(feature = "sqlite")]
|
||||
extern crate tracing;
|
||||
|
||||
#[cfg(feature = "lmdb")]
|
||||
@@ -11,6 +10,8 @@ pub mod sqlite_adapter;
|
||||
|
||||
pub mod counted_tree_hack;
|
||||
|
||||
pub mod open;
|
||||
|
||||
#[cfg(test)]
|
||||
pub mod test;
|
||||
|
||||
@@ -18,10 +19,13 @@ use core::ops::{Bound, RangeBounds};
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::cell::Cell;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use err_derive::Error;
|
||||
|
||||
pub use open::*;
|
||||
|
||||
pub(crate) type OnCommit = Vec<Box<dyn FnOnce()>>;
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -45,6 +49,12 @@ pub type TxValueIter<'a> = Box<dyn std::iter::Iterator<Item = TxOpResult<(Value,
|
||||
#[error(display = "{}", _0)]
|
||||
pub struct Error(pub Cow<'static, str>);
|
||||
|
||||
impl From<std::io::Error> for Error {
|
||||
fn from(e: std::io::Error) -> Error {
|
||||
Error(format!("IO: {}", e).into())
|
||||
}
|
||||
}
|
||||
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
@@ -126,6 +136,10 @@ impl Db {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn snapshot(&self, path: &PathBuf) -> Result<()> {
|
||||
self.0.snapshot(path)
|
||||
}
|
||||
|
||||
pub fn import(&self, other: &Db) -> Result<()> {
|
||||
let existing_trees = self.list_trees()?;
|
||||
if !existing_trees.is_empty() {
|
||||
@@ -171,48 +185,6 @@ impl Db {
|
||||
}
|
||||
}
|
||||
|
||||
/// List of supported database engine types
|
||||
///
|
||||
/// The `enum` holds list of *all* database engines that are are be supported by crate, no matter
|
||||
/// if relevant feature is enabled or not. It allows us to distinguish between invalid engine
|
||||
/// and valid engine, whose support is not enabled via feature flag.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Engine {
|
||||
Lmdb,
|
||||
Sqlite,
|
||||
Sled,
|
||||
}
|
||||
|
||||
impl Engine {
|
||||
/// Return variant name as static `&str`
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Lmdb => "lmdb",
|
||||
Self::Sqlite => "sqlite",
|
||||
Self::Sled => "sled",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for Engine {
|
||||
fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
self.as_str().fmt(fmt)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::str::FromStr for Engine {
|
||||
type Err = Error;
|
||||
|
||||
fn from_str(text: &str) -> Result<Engine> {
|
||||
match text {
|
||||
"lmdb" | "heed" => Ok(Self::Lmdb),
|
||||
"sqlite" | "sqlite3" | "rusqlite" => Ok(Self::Sqlite),
|
||||
"sled" => Ok(Self::Sled),
|
||||
kind => Err(Error(format!("Invalid DB engine: {}", kind).into())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::len_without_is_empty)]
|
||||
impl Tree {
|
||||
#[inline]
|
||||
@@ -364,6 +336,7 @@ pub(crate) trait IDb: Send + Sync {
|
||||
fn engine(&self) -> String;
|
||||
fn open_tree(&self, name: &str) -> Result<usize>;
|
||||
fn list_trees(&self) -> Result<Vec<String>>;
|
||||
fn snapshot(&self, path: &PathBuf) -> Result<()>;
|
||||
|
||||
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>;
|
||||
fn len(&self, tree: usize) -> Result<usize>;
|
||||
|
||||
@@ -3,6 +3,7 @@ use core::ptr::NonNull;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::convert::TryInto;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, RwLock};
|
||||
|
||||
use heed::types::ByteSlice;
|
||||
@@ -102,6 +103,15 @@ impl IDb for LmdbDb {
|
||||
Ok(ret2)
|
||||
}
|
||||
|
||||
fn snapshot(&self, to: &PathBuf) -> Result<()> {
|
||||
std::fs::create_dir_all(to)?;
|
||||
let mut path = to.clone();
|
||||
path.push("data.mdb");
|
||||
self.db
|
||||
.copy_to_path(path, heed::CompactionOption::Disabled)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ----
|
||||
|
||||
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>> {
|
||||
|
||||
+147
@@ -0,0 +1,147 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
use crate::{Db, Error, Result};
|
||||
|
||||
/// List of supported database engine types
|
||||
///
|
||||
/// The `enum` holds list of *all* database engines that are are be supported by crate, no matter
|
||||
/// if relevant feature is enabled or not. It allows us to distinguish between invalid engine
|
||||
/// and valid engine, whose support is not enabled via feature flag.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Engine {
|
||||
Lmdb,
|
||||
Sqlite,
|
||||
Sled,
|
||||
}
|
||||
|
||||
impl Engine {
|
||||
/// Return variant name as static `&str`
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Lmdb => "lmdb",
|
||||
Self::Sqlite => "sqlite",
|
||||
Self::Sled => "sled",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for Engine {
|
||||
fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
self.as_str().fmt(fmt)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::str::FromStr for Engine {
|
||||
type Err = Error;
|
||||
|
||||
fn from_str(text: &str) -> Result<Engine> {
|
||||
match text {
|
||||
"lmdb" | "heed" => Ok(Self::Lmdb),
|
||||
"sqlite" | "sqlite3" | "rusqlite" => Ok(Self::Sqlite),
|
||||
"sled" => Ok(Self::Sled),
|
||||
kind => Err(Error(
|
||||
format!(
|
||||
"Invalid DB engine: {} (options are: lmdb, sled, sqlite)",
|
||||
kind
|
||||
)
|
||||
.into(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct OpenOpt {
|
||||
pub fsync: bool,
|
||||
pub lmdb_map_size: Option<usize>,
|
||||
pub sled_cache_capacity: usize,
|
||||
pub sled_flush_every_ms: u64,
|
||||
}
|
||||
|
||||
impl Default for OpenOpt {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
fsync: false,
|
||||
lmdb_map_size: None,
|
||||
sled_cache_capacity: 1024 * 1024 * 1024,
|
||||
sled_flush_every_ms: 2000,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn open_db(path: &PathBuf, engine: Engine, opt: &OpenOpt) -> Result<Db> {
|
||||
match engine {
|
||||
// ---- Sled DB ----
|
||||
#[cfg(feature = "sled")]
|
||||
Engine::Sled => {
|
||||
if opt.fsync {
|
||||
return Err(Error(
|
||||
"`metadata_fsync = true` is not supported with the Sled database engine".into(),
|
||||
));
|
||||
}
|
||||
info!("Opening Sled database at: {}", path.display());
|
||||
let db = crate::sled_adapter::sled::Config::default()
|
||||
.path(&path)
|
||||
.cache_capacity(opt.sled_cache_capacity as u64)
|
||||
.flush_every_ms(Some(opt.sled_flush_every_ms))
|
||||
.open()?;
|
||||
Ok(crate::sled_adapter::SledDb::init(db))
|
||||
}
|
||||
|
||||
// ---- Sqlite DB ----
|
||||
#[cfg(feature = "sqlite")]
|
||||
Engine::Sqlite => {
|
||||
info!("Opening Sqlite database at: {}", path.display());
|
||||
let manager = r2d2_sqlite::SqliteConnectionManager::file(path);
|
||||
Ok(crate::sqlite_adapter::SqliteDb::new(manager, opt.fsync)?)
|
||||
}
|
||||
|
||||
// ---- LMDB DB ----
|
||||
#[cfg(feature = "lmdb")]
|
||||
Engine::Lmdb => {
|
||||
info!("Opening LMDB database at: {}", path.display());
|
||||
if let Err(e) = std::fs::create_dir_all(&path) {
|
||||
return Err(Error(
|
||||
format!("Unable to create LMDB data directory: {}", e).into(),
|
||||
));
|
||||
}
|
||||
|
||||
let map_size = match opt.lmdb_map_size {
|
||||
None => crate::lmdb_adapter::recommended_map_size(),
|
||||
Some(v) => v - (v % 4096),
|
||||
};
|
||||
|
||||
let mut env_builder = heed::EnvOpenOptions::new();
|
||||
env_builder.max_dbs(100);
|
||||
env_builder.map_size(map_size);
|
||||
env_builder.max_readers(2048);
|
||||
unsafe {
|
||||
env_builder.flag(crate::lmdb_adapter::heed::flags::Flags::MdbNoMetaSync);
|
||||
if !opt.fsync {
|
||||
env_builder.flag(heed::flags::Flags::MdbNoSync);
|
||||
}
|
||||
}
|
||||
match env_builder.open(&path) {
|
||||
Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => {
|
||||
return Err(Error(
|
||||
"OutOfMemory error while trying to open LMDB database. This can happen \
|
||||
if your operating system is not allowing you to use sufficient virtual \
|
||||
memory address space. Please check that no limit is set (ulimit -v). \
|
||||
You may also try to set a smaller `lmdb_map_size` configuration parameter. \
|
||||
On 32-bit machines, you should probably switch to another database engine."
|
||||
.into(),
|
||||
))
|
||||
}
|
||||
Err(e) => Err(Error(format!("Cannot open LMDB database: {}", e).into())),
|
||||
Ok(db) => Ok(crate::lmdb_adapter::LmdbDb::init(db)),
|
||||
}
|
||||
}
|
||||
|
||||
// Pattern is unreachable when all supported DB engines are compiled into binary. The allow
|
||||
// attribute is added so that we won't have to change this match in case stop building
|
||||
// support for one or more engines by default.
|
||||
#[allow(unreachable_patterns)]
|
||||
engine => Err(Error(
|
||||
format!("DB engine support not available in this build: {}", engine).into(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ use core::ops::Bound;
|
||||
|
||||
use std::cell::Cell;
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, RwLock};
|
||||
|
||||
use sled::transaction::{
|
||||
@@ -96,6 +97,13 @@ impl IDb for SledDb {
|
||||
Ok(trees)
|
||||
}
|
||||
|
||||
fn snapshot(&self, to: &PathBuf) -> Result<()> {
|
||||
let to_db = sled::open(to)?;
|
||||
let export = self.db.export();
|
||||
to_db.import(export);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ----
|
||||
|
||||
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>> {
|
||||
|
||||
+106
-106
@@ -1,12 +1,14 @@
|
||||
use core::ops::Bound;
|
||||
|
||||
use std::borrow::BorrowMut;
|
||||
use std::marker::PhantomPinned;
|
||||
use std::path::PathBuf;
|
||||
use std::pin::Pin;
|
||||
use std::ptr::NonNull;
|
||||
use std::sync::{Arc, Mutex, MutexGuard};
|
||||
use std::sync::{Arc, Mutex, RwLock};
|
||||
|
||||
use rusqlite::{params, Connection, Rows, Statement, Transaction};
|
||||
use r2d2::Pool;
|
||||
use r2d2_sqlite::SqliteConnectionManager;
|
||||
use rusqlite::{params, Rows, Statement, Transaction};
|
||||
|
||||
use crate::{
|
||||
Db, Error, IDb, ITx, ITxFn, OnCommit, Result, TxError, TxFnResult, TxOpError, TxOpResult,
|
||||
@@ -15,6 +17,8 @@ use crate::{
|
||||
|
||||
pub use rusqlite;
|
||||
|
||||
type Connection = r2d2::PooledConnection<SqliteConnectionManager>;
|
||||
|
||||
// --- err
|
||||
|
||||
impl From<rusqlite::Error> for Error {
|
||||
@@ -23,6 +27,12 @@ impl From<rusqlite::Error> for Error {
|
||||
}
|
||||
}
|
||||
|
||||
impl From<r2d2::Error> for Error {
|
||||
fn from(e: r2d2::Error) -> Error {
|
||||
Error(format!("Sqlite: {}", e).into())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<rusqlite::Error> for TxOpError {
|
||||
fn from(e: rusqlite::Error) -> TxOpError {
|
||||
TxOpError(e.into())
|
||||
@@ -31,35 +41,47 @@ impl From<rusqlite::Error> for TxOpError {
|
||||
|
||||
// -- db
|
||||
|
||||
pub struct SqliteDb(Mutex<SqliteDbInner>);
|
||||
|
||||
struct SqliteDbInner {
|
||||
db: Connection,
|
||||
trees: Vec<String>,
|
||||
pub struct SqliteDb {
|
||||
db: Pool<SqliteConnectionManager>,
|
||||
trees: RwLock<Vec<Arc<str>>>,
|
||||
// All operations that might write on the DB must take this lock first.
|
||||
// This emulates LMDB's approach where a single writer can be
|
||||
// active at once.
|
||||
write_lock: Mutex<()>,
|
||||
}
|
||||
|
||||
impl SqliteDb {
|
||||
pub fn init(db: rusqlite::Connection) -> Db {
|
||||
let s = Self(Mutex::new(SqliteDbInner {
|
||||
db,
|
||||
trees: Vec::new(),
|
||||
}));
|
||||
Db(Arc::new(s))
|
||||
pub fn new(manager: SqliteConnectionManager, sync_mode: bool) -> Result<Db> {
|
||||
let manager = manager.with_init(move |db| {
|
||||
db.pragma_update(None, "journal_mode", "WAL")?;
|
||||
if sync_mode {
|
||||
db.pragma_update(None, "synchronous", "NORMAL")?;
|
||||
} else {
|
||||
db.pragma_update(None, "synchronous", "OFF")?;
|
||||
}
|
||||
Ok(())
|
||||
});
|
||||
let s = Self {
|
||||
db: Pool::builder().build(manager)?,
|
||||
trees: RwLock::new(vec![]),
|
||||
write_lock: Mutex::new(()),
|
||||
};
|
||||
Ok(Db(Arc::new(s)))
|
||||
}
|
||||
}
|
||||
|
||||
impl SqliteDbInner {
|
||||
fn get_tree(&self, i: usize) -> Result<&'_ str> {
|
||||
impl SqliteDb {
|
||||
fn get_tree(&self, i: usize) -> Result<Arc<str>> {
|
||||
self.trees
|
||||
.read()
|
||||
.unwrap()
|
||||
.get(i)
|
||||
.map(String::as_str)
|
||||
.cloned()
|
||||
.ok_or_else(|| Error("invalid tree id".into()))
|
||||
}
|
||||
|
||||
fn internal_get(&self, tree: &str, key: &[u8]) -> Result<Option<Value>> {
|
||||
let mut stmt = self
|
||||
.db
|
||||
.prepare(&format!("SELECT v FROM {} WHERE k = ?1", tree))?;
|
||||
fn internal_get(&self, db: &Connection, tree: &str, key: &[u8]) -> Result<Option<Value>> {
|
||||
let mut stmt = db.prepare(&format!("SELECT v FROM {} WHERE k = ?1", tree))?;
|
||||
let mut res_iter = stmt.query([key])?;
|
||||
match res_iter.next()? {
|
||||
None => Ok(None),
|
||||
@@ -75,13 +97,14 @@ impl IDb for SqliteDb {
|
||||
|
||||
fn open_tree(&self, name: &str) -> Result<usize> {
|
||||
let name = format!("tree_{}", name.replace(':', "_COLON_"));
|
||||
let mut this = self.0.lock().unwrap();
|
||||
let mut trees = self.trees.write().unwrap();
|
||||
|
||||
if let Some(i) = this.trees.iter().position(|x| x == &name) {
|
||||
if let Some(i) = trees.iter().position(|x| x.as_ref() == &name) {
|
||||
Ok(i)
|
||||
} else {
|
||||
let db = self.db.get()?;
|
||||
trace!("create table {}", name);
|
||||
this.db.execute(
|
||||
db.execute(
|
||||
&format!(
|
||||
"CREATE TABLE IF NOT EXISTS {} (
|
||||
k BLOB PRIMARY KEY,
|
||||
@@ -93,8 +116,8 @@ impl IDb for SqliteDb {
|
||||
)?;
|
||||
trace!("table created: {}, unlocking", name);
|
||||
|
||||
let i = this.trees.len();
|
||||
this.trees.push(name.to_string());
|
||||
let i = trees.len();
|
||||
trees.push(name.to_string().into_boxed_str().into());
|
||||
Ok(i)
|
||||
}
|
||||
}
|
||||
@@ -102,11 +125,8 @@ impl IDb for SqliteDb {
|
||||
fn list_trees(&self) -> Result<Vec<String>> {
|
||||
let mut trees = vec![];
|
||||
|
||||
trace!("list_trees: lock db");
|
||||
let this = self.0.lock().unwrap();
|
||||
trace!("list_trees: lock acquired");
|
||||
|
||||
let mut stmt = this.db.prepare(
|
||||
let db = self.db.get()?;
|
||||
let mut stmt = db.prepare(
|
||||
"SELECT name FROM sqlite_schema WHERE type = 'table' AND name LIKE 'tree_%'",
|
||||
)?;
|
||||
let mut rows = stmt.query([])?;
|
||||
@@ -119,24 +139,29 @@ impl IDb for SqliteDb {
|
||||
Ok(trees)
|
||||
}
|
||||
|
||||
fn snapshot(&self, to: &PathBuf) -> Result<()> {
|
||||
fn progress(p: rusqlite::backup::Progress) {
|
||||
let percent = (p.pagecount - p.remaining) * 100 / p.pagecount;
|
||||
info!("Sqlite snapshot progres: {}%", percent);
|
||||
}
|
||||
self.db
|
||||
.get()?
|
||||
.backup(rusqlite::DatabaseName::Main, to, Some(progress))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ----
|
||||
|
||||
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>> {
|
||||
trace!("get {}: lock db", tree);
|
||||
let this = self.0.lock().unwrap();
|
||||
trace!("get {}: lock acquired", tree);
|
||||
|
||||
let tree = this.get_tree(tree)?;
|
||||
this.internal_get(tree, key)
|
||||
let tree = self.get_tree(tree)?;
|
||||
self.internal_get(&self.db.get()?, &tree, key)
|
||||
}
|
||||
|
||||
fn len(&self, tree: usize) -> Result<usize> {
|
||||
trace!("len {}: lock db", tree);
|
||||
let this = self.0.lock().unwrap();
|
||||
trace!("len {}: lock acquired", tree);
|
||||
let tree = self.get_tree(tree)?;
|
||||
let db = self.db.get()?;
|
||||
|
||||
let tree = this.get_tree(tree)?;
|
||||
let mut stmt = this.db.prepare(&format!("SELECT COUNT(*) FROM {}", tree))?;
|
||||
let mut stmt = db.prepare(&format!("SELECT COUNT(*) FROM {}", tree))?;
|
||||
let mut res_iter = stmt.query([])?;
|
||||
match res_iter.next()? {
|
||||
None => Ok(0),
|
||||
@@ -149,69 +174,60 @@ impl IDb for SqliteDb {
|
||||
}
|
||||
|
||||
fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<Option<Value>> {
|
||||
trace!("insert {}: lock db", tree);
|
||||
let this = self.0.lock().unwrap();
|
||||
trace!("insert {}: lock acquired", tree);
|
||||
let tree = self.get_tree(tree)?;
|
||||
let db = self.db.get()?;
|
||||
let lock = self.write_lock.lock();
|
||||
|
||||
let tree = this.get_tree(tree)?;
|
||||
let old_val = this.internal_get(tree, key)?;
|
||||
let old_val = self.internal_get(&db, &tree, key)?;
|
||||
|
||||
let sql = match &old_val {
|
||||
Some(_) => format!("UPDATE {} SET v = ?2 WHERE k = ?1", tree),
|
||||
None => format!("INSERT INTO {} (k, v) VALUES (?1, ?2)", tree),
|
||||
};
|
||||
let n = this.db.execute(&sql, params![key, value])?;
|
||||
let n = db.execute(&sql, params![key, value])?;
|
||||
assert_eq!(n, 1);
|
||||
|
||||
drop(lock);
|
||||
Ok(old_val)
|
||||
}
|
||||
|
||||
fn remove(&self, tree: usize, key: &[u8]) -> Result<Option<Value>> {
|
||||
trace!("remove {}: lock db", tree);
|
||||
let this = self.0.lock().unwrap();
|
||||
trace!("remove {}: lock acquired", tree);
|
||||
let tree = self.get_tree(tree)?;
|
||||
let db = self.db.get()?;
|
||||
let lock = self.write_lock.lock();
|
||||
|
||||
let tree = this.get_tree(tree)?;
|
||||
let old_val = this.internal_get(tree, key)?;
|
||||
let old_val = self.internal_get(&db, &tree, key)?;
|
||||
|
||||
if old_val.is_some() {
|
||||
let n = this
|
||||
.db
|
||||
.execute(&format!("DELETE FROM {} WHERE k = ?1", tree), params![key])?;
|
||||
let n = db.execute(&format!("DELETE FROM {} WHERE k = ?1", tree), params![key])?;
|
||||
assert_eq!(n, 1);
|
||||
}
|
||||
|
||||
drop(lock);
|
||||
Ok(old_val)
|
||||
}
|
||||
|
||||
fn clear(&self, tree: usize) -> Result<()> {
|
||||
trace!("clear {}: lock db", tree);
|
||||
let this = self.0.lock().unwrap();
|
||||
trace!("clear {}: lock acquired", tree);
|
||||
let tree = self.get_tree(tree)?;
|
||||
let db = self.db.get()?;
|
||||
let lock = self.write_lock.lock();
|
||||
|
||||
let tree = this.get_tree(tree)?;
|
||||
this.db.execute(&format!("DELETE FROM {}", tree), [])?;
|
||||
db.execute(&format!("DELETE FROM {}", tree), [])?;
|
||||
|
||||
drop(lock);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn iter(&self, tree: usize) -> Result<ValueIter<'_>> {
|
||||
trace!("iter {}: lock db", tree);
|
||||
let this = self.0.lock().unwrap();
|
||||
trace!("iter {}: lock acquired", tree);
|
||||
|
||||
let tree = this.get_tree(tree)?;
|
||||
let tree = self.get_tree(tree)?;
|
||||
let sql = format!("SELECT k, v FROM {} ORDER BY k ASC", tree);
|
||||
DbValueIterator::make(this, &sql, [])
|
||||
DbValueIterator::make(self.db.get()?, &sql, [])
|
||||
}
|
||||
|
||||
fn iter_rev(&self, tree: usize) -> Result<ValueIter<'_>> {
|
||||
trace!("iter_rev {}: lock db", tree);
|
||||
let this = self.0.lock().unwrap();
|
||||
trace!("iter_rev {}: lock acquired", tree);
|
||||
|
||||
let tree = this.get_tree(tree)?;
|
||||
let tree = self.get_tree(tree)?;
|
||||
let sql = format!("SELECT k, v FROM {} ORDER BY k DESC", tree);
|
||||
DbValueIterator::make(this, &sql, [])
|
||||
DbValueIterator::make(self.db.get()?, &sql, [])
|
||||
}
|
||||
|
||||
fn range<'r>(
|
||||
@@ -220,11 +236,7 @@ impl IDb for SqliteDb {
|
||||
low: Bound<&'r [u8]>,
|
||||
high: Bound<&'r [u8]>,
|
||||
) -> Result<ValueIter<'_>> {
|
||||
trace!("range {}: lock db", tree);
|
||||
let this = self.0.lock().unwrap();
|
||||
trace!("range {}: lock acquired", tree);
|
||||
|
||||
let tree = this.get_tree(tree)?;
|
||||
let tree = self.get_tree(tree)?;
|
||||
|
||||
let (bounds_sql, params) = bounds_sql(low, high);
|
||||
let sql = format!("SELECT k, v FROM {} {} ORDER BY k ASC", tree, bounds_sql);
|
||||
@@ -234,7 +246,7 @@ impl IDb for SqliteDb {
|
||||
.map(|x| x as &dyn rusqlite::ToSql)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
DbValueIterator::make::<&[&dyn rusqlite::ToSql]>(this, &sql, params.as_ref())
|
||||
DbValueIterator::make::<&[&dyn rusqlite::ToSql]>(self.db.get()?, &sql, params.as_ref())
|
||||
}
|
||||
fn range_rev<'r>(
|
||||
&self,
|
||||
@@ -242,11 +254,7 @@ impl IDb for SqliteDb {
|
||||
low: Bound<&'r [u8]>,
|
||||
high: Bound<&'r [u8]>,
|
||||
) -> Result<ValueIter<'_>> {
|
||||
trace!("range_rev {}: lock db", tree);
|
||||
let this = self.0.lock().unwrap();
|
||||
trace!("range_rev {}: lock acquired", tree);
|
||||
|
||||
let tree = this.get_tree(tree)?;
|
||||
let tree = self.get_tree(tree)?;
|
||||
|
||||
let (bounds_sql, params) = bounds_sql(low, high);
|
||||
let sql = format!("SELECT k, v FROM {} {} ORDER BY k DESC", tree, bounds_sql);
|
||||
@@ -256,25 +264,20 @@ impl IDb for SqliteDb {
|
||||
.map(|x| x as &dyn rusqlite::ToSql)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
DbValueIterator::make::<&[&dyn rusqlite::ToSql]>(this, &sql, params.as_ref())
|
||||
DbValueIterator::make::<&[&dyn rusqlite::ToSql]>(self.db.get()?, &sql, params.as_ref())
|
||||
}
|
||||
|
||||
// ----
|
||||
|
||||
fn transaction(&self, f: &dyn ITxFn) -> TxResult<OnCommit, ()> {
|
||||
trace!("transaction: lock db");
|
||||
let mut this = self.0.lock().unwrap();
|
||||
trace!("transaction: lock acquired");
|
||||
|
||||
let this_mut_ref: &mut SqliteDbInner = this.borrow_mut();
|
||||
let mut db = self.db.get().map_err(Error::from).map_err(TxError::Db)?;
|
||||
let trees = self.trees.read().unwrap();
|
||||
let lock = self.write_lock.lock();
|
||||
|
||||
trace!("trying transaction");
|
||||
let mut tx = SqliteTx {
|
||||
tx: this_mut_ref
|
||||
.db
|
||||
.transaction()
|
||||
.map_err(Error::from)
|
||||
.map_err(TxError::Db)?,
|
||||
trees: &this_mut_ref.trees,
|
||||
tx: db.transaction().map_err(Error::from).map_err(TxError::Db)?,
|
||||
trees: &trees,
|
||||
};
|
||||
let res = match f.try_on(&mut tx) {
|
||||
TxFnResult::Ok(on_commit) => {
|
||||
@@ -294,7 +297,8 @@ impl IDb for SqliteDb {
|
||||
};
|
||||
|
||||
trace!("transaction done");
|
||||
res
|
||||
drop(lock);
|
||||
return res;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -302,12 +306,12 @@ impl IDb for SqliteDb {
|
||||
|
||||
struct SqliteTx<'a> {
|
||||
tx: Transaction<'a>,
|
||||
trees: &'a [String],
|
||||
trees: &'a [Arc<str>],
|
||||
}
|
||||
|
||||
impl<'a> SqliteTx<'a> {
|
||||
fn get_tree(&self, i: usize) -> TxOpResult<&'_ str> {
|
||||
self.trees.get(i).map(String::as_ref).ok_or_else(|| {
|
||||
self.trees.get(i).map(Arc::as_ref).ok_or_else(|| {
|
||||
TxOpError(Error(
|
||||
"invalid tree id (it might have been openned after the transaction started)".into(),
|
||||
))
|
||||
@@ -396,18 +400,14 @@ impl<'a> ITx for SqliteTx<'a> {
|
||||
// ----
|
||||
|
||||
struct DbValueIterator<'a> {
|
||||
db: MutexGuard<'a, SqliteDbInner>,
|
||||
db: Connection,
|
||||
stmt: Option<Statement<'a>>,
|
||||
iter: Option<Rows<'a>>,
|
||||
_pin: PhantomPinned,
|
||||
}
|
||||
|
||||
impl<'a> DbValueIterator<'a> {
|
||||
fn make<P: rusqlite::Params>(
|
||||
db: MutexGuard<'a, SqliteDbInner>,
|
||||
sql: &str,
|
||||
args: P,
|
||||
) -> Result<ValueIter<'a>> {
|
||||
fn make<P: rusqlite::Params>(db: Connection, sql: &str, args: P) -> Result<ValueIter<'a>> {
|
||||
let res = DbValueIterator {
|
||||
db,
|
||||
stmt: None,
|
||||
@@ -419,7 +419,7 @@ impl<'a> DbValueIterator<'a> {
|
||||
|
||||
unsafe {
|
||||
let db = NonNull::from(&boxed.db);
|
||||
let stmt = db.as_ref().db.prepare(sql)?;
|
||||
let stmt = db.as_ref().prepare(sql)?;
|
||||
|
||||
let mut_ref: Pin<&mut DbValueIterator<'a>> = Pin::as_mut(&mut boxed);
|
||||
Pin::get_unchecked_mut(mut_ref).stmt = Some(stmt);
|
||||
|
||||
+2
-1
@@ -106,6 +106,7 @@ fn test_sled_db() {
|
||||
fn test_sqlite_db() {
|
||||
use crate::sqlite_adapter::SqliteDb;
|
||||
|
||||
let db = SqliteDb::init(rusqlite::Connection::open_in_memory().unwrap());
|
||||
let manager = r2d2_sqlite::SqliteConnectionManager::memory();
|
||||
let db = SqliteDb::new(manager, false).unwrap();
|
||||
test_suite(db);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage"
|
||||
version = "0.9.1"
|
||||
version = "0.9.4"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
@@ -58,6 +58,7 @@ opentelemetry.workspace = true
|
||||
opentelemetry-prometheus = { workspace = true, optional = true }
|
||||
opentelemetry-otlp = { workspace = true, optional = true }
|
||||
prometheus = { workspace = true, optional = true }
|
||||
syslog-tracing = { workspace = true, optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
aws-config.workspace = true
|
||||
@@ -97,6 +98,8 @@ kubernetes-discovery = [ "garage_rpc/kubernetes-discovery" ]
|
||||
metrics = [ "garage_api/metrics", "opentelemetry-prometheus", "prometheus" ]
|
||||
# Exporter for the OpenTelemetry Collector.
|
||||
telemetry-otlp = [ "opentelemetry-otlp" ]
|
||||
# Logging to syslog
|
||||
syslog = [ "syslog-tracing" ]
|
||||
|
||||
# NOTE: bundled-libs and system-libs should be treat as mutually exclusive;
|
||||
# exactly one of them should be enabled.
|
||||
|
||||
+16
-24
@@ -54,9 +54,8 @@ impl AdminRpcHandler {
|
||||
let bucket_id = self
|
||||
.garage
|
||||
.bucket_helper()
|
||||
.resolve_global_bucket_name(&query.name)
|
||||
.await?
|
||||
.ok_or_bad_request("Bucket not found")?;
|
||||
.admin_get_existing_matching_bucket(&query.name)
|
||||
.await?;
|
||||
|
||||
let bucket = self
|
||||
.garage
|
||||
@@ -157,9 +156,8 @@ impl AdminRpcHandler {
|
||||
|
||||
let bucket_id = helper
|
||||
.bucket()
|
||||
.resolve_global_bucket_name(&query.name)
|
||||
.await?
|
||||
.ok_or_bad_request("Bucket not found")?;
|
||||
.admin_get_existing_matching_bucket(&query.name)
|
||||
.await?;
|
||||
|
||||
// Get the alias, but keep in minde here the bucket name
|
||||
// given in parameter can also be directly the bucket's ID.
|
||||
@@ -235,9 +233,8 @@ impl AdminRpcHandler {
|
||||
|
||||
let bucket_id = helper
|
||||
.bucket()
|
||||
.resolve_global_bucket_name(&query.existing_bucket)
|
||||
.await?
|
||||
.ok_or_bad_request("Bucket not found")?;
|
||||
.admin_get_existing_matching_bucket(&query.existing_bucket)
|
||||
.await?;
|
||||
|
||||
if let Some(key_pattern) = &query.local {
|
||||
let key = helper.key().get_existing_matching_key(key_pattern).await?;
|
||||
@@ -307,9 +304,8 @@ impl AdminRpcHandler {
|
||||
|
||||
let bucket_id = helper
|
||||
.bucket()
|
||||
.resolve_global_bucket_name(&query.bucket)
|
||||
.await?
|
||||
.ok_or_bad_request("Bucket not found")?;
|
||||
.admin_get_existing_matching_bucket(&query.bucket)
|
||||
.await?;
|
||||
let key = helper
|
||||
.key()
|
||||
.get_existing_matching_key(&query.key_pattern)
|
||||
@@ -343,9 +339,8 @@ impl AdminRpcHandler {
|
||||
|
||||
let bucket_id = helper
|
||||
.bucket()
|
||||
.resolve_global_bucket_name(&query.bucket)
|
||||
.await?
|
||||
.ok_or_bad_request("Bucket not found")?;
|
||||
.admin_get_existing_matching_bucket(&query.bucket)
|
||||
.await?;
|
||||
let key = helper
|
||||
.key()
|
||||
.get_existing_matching_key(&query.key_pattern)
|
||||
@@ -378,9 +373,8 @@ impl AdminRpcHandler {
|
||||
let bucket_id = self
|
||||
.garage
|
||||
.bucket_helper()
|
||||
.resolve_global_bucket_name(&query.bucket)
|
||||
.await?
|
||||
.ok_or_bad_request("Bucket not found")?;
|
||||
.admin_get_existing_matching_bucket(&query.bucket)
|
||||
.await?;
|
||||
|
||||
let mut bucket = self
|
||||
.garage
|
||||
@@ -420,9 +414,8 @@ impl AdminRpcHandler {
|
||||
let bucket_id = self
|
||||
.garage
|
||||
.bucket_helper()
|
||||
.resolve_global_bucket_name(&query.bucket)
|
||||
.await?
|
||||
.ok_or_bad_request("Bucket not found")?;
|
||||
.admin_get_existing_matching_bucket(&query.bucket)
|
||||
.await?;
|
||||
|
||||
let mut bucket = self
|
||||
.garage
|
||||
@@ -479,9 +472,8 @@ impl AdminRpcHandler {
|
||||
bucket_ids.push(
|
||||
self.garage
|
||||
.bucket_helper()
|
||||
.resolve_global_bucket_name(b)
|
||||
.await?
|
||||
.ok_or_bad_request(format!("Bucket not found: {}", b))?,
|
||||
.admin_get_existing_matching_bucket(b)
|
||||
.await?,
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -46,6 +46,7 @@ pub enum AdminRpc {
|
||||
Stats(StatsOpt),
|
||||
Worker(WorkerOperation),
|
||||
BlockOperation(BlockOperation),
|
||||
MetaOperation(MetaOperation),
|
||||
|
||||
// Replies
|
||||
Ok(String),
|
||||
@@ -518,6 +519,44 @@ impl AdminRpcHandler {
|
||||
)]))
|
||||
}
|
||||
}
|
||||
|
||||
// ================ META DB COMMANDS ====================
|
||||
|
||||
async fn handle_meta_cmd(self: &Arc<Self>, mo: &MetaOperation) -> Result<AdminRpc, Error> {
|
||||
match mo {
|
||||
MetaOperation::Snapshot { all: true } => {
|
||||
let ring = self.garage.system.ring.borrow().clone();
|
||||
let to = ring.layout.node_ids().to_vec();
|
||||
|
||||
let resps = futures::future::join_all(to.iter().map(|to| async move {
|
||||
let to = (*to).into();
|
||||
self.endpoint
|
||||
.call(
|
||||
&to,
|
||||
AdminRpc::MetaOperation(MetaOperation::Snapshot { all: false }),
|
||||
PRIO_NORMAL,
|
||||
)
|
||||
.await
|
||||
}))
|
||||
.await;
|
||||
|
||||
let mut ret = vec![];
|
||||
for (to, resp) in to.iter().zip(resps.iter()) {
|
||||
let res_str = match resp {
|
||||
Ok(_) => "ok".to_string(),
|
||||
Err(e) => format!("error: {}", e),
|
||||
};
|
||||
ret.push(format!("{:?}\t{}", to, res_str));
|
||||
}
|
||||
|
||||
Ok(AdminRpc::Ok(format_table_to_string(ret)))
|
||||
}
|
||||
MetaOperation::Snapshot { all: false } => {
|
||||
garage_model::snapshot::async_snapshot_metadata(&self.garage).await?;
|
||||
Ok(AdminRpc::Ok("Snapshot has been saved.".into()))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -535,6 +574,7 @@ impl EndpointHandler<AdminRpc> for AdminRpcHandler {
|
||||
AdminRpc::Stats(opt) => self.handle_stats(opt.clone()).await,
|
||||
AdminRpc::Worker(wo) => self.handle_worker_cmd(wo).await,
|
||||
AdminRpc::BlockOperation(bo) => self.handle_block_cmd(bo).await,
|
||||
AdminRpc::MetaOperation(mo) => self.handle_meta_cmd(mo).await,
|
||||
m => Err(GarageError::unexpected_rpc_message(m).into()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,6 +44,9 @@ pub async fn cli_command_dispatch(
|
||||
Command::Block(bo) => {
|
||||
cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::BlockOperation(bo)).await
|
||||
}
|
||||
Command::Meta(mo) => {
|
||||
cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::MetaOperation(mo)).await
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -51,53 +51,13 @@ pub(crate) fn do_conversion(args: ConvertDbOpt) -> Result<()> {
|
||||
return Err(Error("input and output database engine must differ".into()));
|
||||
}
|
||||
|
||||
let input = open_db(args.input_path, args.input_engine, &args.db_open)?;
|
||||
let output = open_db(args.output_path, args.output_engine, &args.db_open)?;
|
||||
let opt = OpenOpt {
|
||||
lmdb_map_size: args.db_open.lmdb.map_size.map(|x| x.as_u64() as usize),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let input = open_db(&args.input_path, args.input_engine, &opt)?;
|
||||
let output = open_db(&args.output_path, args.output_engine, &opt)?;
|
||||
output.import(&input)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn open_db(path: PathBuf, engine: Engine, open: &OpenDbOpt) -> Result<Db> {
|
||||
match engine {
|
||||
#[cfg(feature = "sled")]
|
||||
Engine::Sled => {
|
||||
let db = sled_adapter::sled::Config::default().path(&path).open()?;
|
||||
Ok(sled_adapter::SledDb::init(db))
|
||||
}
|
||||
#[cfg(feature = "sqlite")]
|
||||
Engine::Sqlite => {
|
||||
let db = sqlite_adapter::rusqlite::Connection::open(&path)?;
|
||||
db.pragma_update(None, "journal_mode", "WAL")?;
|
||||
db.pragma_update(None, "synchronous", "NORMAL")?;
|
||||
Ok(sqlite_adapter::SqliteDb::init(db))
|
||||
}
|
||||
#[cfg(feature = "lmdb")]
|
||||
Engine::Lmdb => {
|
||||
std::fs::create_dir_all(&path).map_err(|e| {
|
||||
Error(format!("Unable to create LMDB data directory: {}", e).into())
|
||||
})?;
|
||||
|
||||
let map_size = match open.lmdb.map_size {
|
||||
Some(c) => c.as_u64() as usize,
|
||||
None => lmdb_adapter::recommended_map_size(),
|
||||
};
|
||||
|
||||
let mut env_builder = lmdb_adapter::heed::EnvOpenOptions::new();
|
||||
env_builder.max_dbs(100);
|
||||
env_builder.map_size(map_size);
|
||||
unsafe {
|
||||
env_builder.flag(lmdb_adapter::heed::flags::Flags::MdbNoMetaSync);
|
||||
}
|
||||
let db = env_builder.open(&path)?;
|
||||
Ok(lmdb_adapter::LmdbDb::init(db))
|
||||
}
|
||||
|
||||
// Pattern is unreachable when all supported DB engines are compiled into binary. The allow
|
||||
// attribute is added so that we won't have to change this match in case stop building
|
||||
// support for one or more engines by default.
|
||||
#[allow(unreachable_patterns)]
|
||||
engine => Err(Error(
|
||||
format!("Engine support not available in this build: {}", engine).into(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,6 +57,10 @@ pub enum Command {
|
||||
#[structopt(name = "block", version = garage_version())]
|
||||
Block(BlockOperation),
|
||||
|
||||
/// Operations on the metadata db
|
||||
#[structopt(name = "meta", version = garage_version())]
|
||||
Meta(MetaOperation),
|
||||
|
||||
/// Convert metadata db between database engine formats
|
||||
#[structopt(name = "convert-db", version = garage_version())]
|
||||
ConvertDb(convert_db::ConvertDbOpt),
|
||||
@@ -617,3 +621,14 @@ pub enum BlockOperation {
|
||||
blocks: Vec<String>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone, Copy)]
|
||||
pub enum MetaOperation {
|
||||
/// Save a snapshot of the metadata db file
|
||||
#[structopt(name = "snapshot", version = garage_version())]
|
||||
Snapshot {
|
||||
/// Run on all nodes instead of only local node
|
||||
#[structopt(long = "all")]
|
||||
all: bool,
|
||||
},
|
||||
}
|
||||
|
||||
+54
-11
@@ -140,17 +140,8 @@ async fn main() {
|
||||
let opt = Opt::from_clap(&Opt::clap().version(version.as_str()).get_matches());
|
||||
|
||||
// Initialize logging as well as other libraries used in Garage
|
||||
if std::env::var("RUST_LOG").is_err() {
|
||||
let default_log = match &opt.cmd {
|
||||
Command::Server => "netapp=info,garage=info",
|
||||
_ => "netapp=warn,garage=warn",
|
||||
};
|
||||
std::env::set_var("RUST_LOG", default_log)
|
||||
}
|
||||
tracing_subscriber::fmt()
|
||||
.with_writer(std::io::stderr)
|
||||
.with_env_filter(tracing_subscriber::filter::EnvFilter::from_default_env())
|
||||
.init();
|
||||
init_logging(&opt);
|
||||
|
||||
sodiumoxide::init().expect("Unable to init sodiumoxide");
|
||||
|
||||
let res = match opt.cmd {
|
||||
@@ -173,6 +164,58 @@ async fn main() {
|
||||
}
|
||||
}
|
||||
|
||||
fn init_logging(opt: &Opt) {
|
||||
if std::env::var("RUST_LOG").is_err() {
|
||||
let default_log = match &opt.cmd {
|
||||
Command::Server => "netapp=info,garage=info",
|
||||
_ => "netapp=warn,garage=warn",
|
||||
};
|
||||
std::env::set_var("RUST_LOG", default_log)
|
||||
}
|
||||
|
||||
let env_filter = tracing_subscriber::filter::EnvFilter::from_default_env();
|
||||
|
||||
if std::env::var("GARAGE_LOG_TO_SYSLOG")
|
||||
.map(|x| x == "1" || x == "true")
|
||||
.unwrap_or(false)
|
||||
{
|
||||
#[cfg(feature = "syslog")]
|
||||
{
|
||||
use std::ffi::CStr;
|
||||
use syslog_tracing::{Facility, Options, Syslog};
|
||||
|
||||
let syslog = Syslog::new(
|
||||
CStr::from_bytes_with_nul(b"garage\0").unwrap(),
|
||||
Options::LOG_PID | Options::LOG_PERROR,
|
||||
Facility::Daemon,
|
||||
)
|
||||
.expect("Unable to init syslog");
|
||||
|
||||
tracing_subscriber::fmt()
|
||||
.with_writer(syslog)
|
||||
.with_env_filter(env_filter)
|
||||
.with_ansi(false) // disable ANSI escape sequences (colours)
|
||||
.with_file(false)
|
||||
.with_level(false)
|
||||
.without_time()
|
||||
.compact()
|
||||
.init();
|
||||
|
||||
return;
|
||||
}
|
||||
#[cfg(not(feature = "syslog"))]
|
||||
{
|
||||
eprintln!("Syslog support is not enabled in this build.");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
tracing_subscriber::fmt()
|
||||
.with_writer(std::io::stderr)
|
||||
.with_env_filter(env_filter)
|
||||
.init();
|
||||
}
|
||||
|
||||
async fn cli_command(opt: Opt) -> Result<(), Error> {
|
||||
let config = if (opt.secrets.rpc_secret.is_none() && opt.secrets.rpc_secret_file.is_none())
|
||||
|| opt.rpc_host.is_none()
|
||||
|
||||
@@ -51,7 +51,7 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er
|
||||
let (background, await_background_done) = BackgroundRunner::new(watch_cancel.clone());
|
||||
|
||||
info!("Spawning Garage workers...");
|
||||
garage.spawn_workers(&background);
|
||||
garage.spawn_workers(&background)?;
|
||||
|
||||
if config.admin.trace_sink.is_some() {
|
||||
info!("Initialize tracing...");
|
||||
|
||||
@@ -64,6 +64,10 @@ impl CustomRequester {
|
||||
vhost_style: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn client(&self) -> &Client<HttpConnector, Body> {
|
||||
&self.client
|
||||
}
|
||||
}
|
||||
|
||||
pub struct RequestBuilder<'a> {
|
||||
|
||||
@@ -42,6 +42,10 @@ impl Instance {
|
||||
.ok()
|
||||
.unwrap_or_else(|| env::temp_dir().join(format!("garage-integ-test-{}", port)));
|
||||
|
||||
let db_engine = env::var("GARAGE_TEST_INTEGRATION_DB_ENGINE")
|
||||
.ok()
|
||||
.unwrap_or_else(|| "lmdb".into());
|
||||
|
||||
// Clean test runtime directory
|
||||
if path.exists() {
|
||||
fs::remove_dir_all(&path).expect("Could not clean test runtime directory");
|
||||
@@ -52,7 +56,7 @@ impl Instance {
|
||||
r#"
|
||||
metadata_dir = "{path}/meta"
|
||||
data_dir = "{path}/data"
|
||||
db_engine = "lmdb"
|
||||
db_engine = "{db_engine}"
|
||||
|
||||
replication_mode = "1"
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ use crate::common;
|
||||
use crate::json_body;
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "currently broken"]
|
||||
async fn test_poll_item() {
|
||||
let ctx = common::context();
|
||||
let bucket = ctx.create_bucket("test-k2v-poll-item");
|
||||
@@ -98,6 +99,7 @@ async fn test_poll_item() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "currently broken"]
|
||||
async fn test_poll_range() {
|
||||
let ctx = common::context();
|
||||
let bucket = ctx.create_bucket("test-k2v-poll-range");
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
mod list;
|
||||
mod multipart;
|
||||
mod objects;
|
||||
mod presigned;
|
||||
mod simple;
|
||||
mod streaming_signature;
|
||||
mod website;
|
||||
|
||||
@@ -185,8 +185,51 @@ async fn test_getobject() {
|
||||
assert_eq!(o.content_range.unwrap().as_str(), "bytes 57-61/62");
|
||||
assert_bytes_eq!(o.body, &BODY[57..]);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_metadata() {
|
||||
let ctx = common::context();
|
||||
let bucket = ctx.create_bucket("testmetadata");
|
||||
|
||||
let etag = "\"46cf18a9b447991b450cad3facf5937e\"";
|
||||
let exp = aws_sdk_s3::primitives::DateTime::from_secs(10000000000);
|
||||
let exp2 = aws_sdk_s3::primitives::DateTime::from_secs(10000500000);
|
||||
|
||||
{
|
||||
let exp = aws_sdk_s3::primitives::DateTime::from_secs(10000000000);
|
||||
// Note. The AWS client SDK adds a Content-Type header
|
||||
// with value application/octet-stream if it is not given,
|
||||
// so here we force it to a known different value.
|
||||
let data = ByteStream::from_static(BODY);
|
||||
let r = ctx
|
||||
.client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(STD_KEY)
|
||||
.body(data)
|
||||
.content_type("application/test")
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(r.e_tag.unwrap().as_str(), etag);
|
||||
|
||||
let o = ctx
|
||||
.client
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(STD_KEY)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(o.e_tag.unwrap().as_str(), etag);
|
||||
assert_eq!(o.content_type.unwrap().as_str(), "application/test");
|
||||
assert_eq!(o.cache_control, None);
|
||||
assert_eq!(o.content_disposition, None);
|
||||
assert_eq!(o.content_encoding, None);
|
||||
assert_eq!(o.content_language, None);
|
||||
assert_eq!(o.expires, None);
|
||||
assert_eq!(o.metadata.unwrap_or_default().len(), 0);
|
||||
|
||||
let o = ctx
|
||||
.client
|
||||
.get_object()
|
||||
@@ -201,13 +244,77 @@ async fn test_getobject() {
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(o.e_tag.unwrap().as_str(), etag);
|
||||
assert_eq!(o.content_type.unwrap().as_str(), "application/x-dummy-test");
|
||||
assert_eq!(o.cache_control.unwrap().as_str(), "ccdummy");
|
||||
assert_eq!(o.content_disposition.unwrap().as_str(), "cddummy");
|
||||
assert_eq!(o.content_encoding.unwrap().as_str(), "cedummy");
|
||||
assert_eq!(o.content_language.unwrap().as_str(), "cldummy");
|
||||
assert_eq!(o.expires.unwrap(), exp);
|
||||
}
|
||||
|
||||
{
|
||||
let data = ByteStream::from_static(BODY);
|
||||
let r = ctx
|
||||
.client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(STD_KEY)
|
||||
.body(data)
|
||||
.content_type("application/test")
|
||||
.cache_control("cctest")
|
||||
.content_disposition("cdtest")
|
||||
.content_encoding("cetest")
|
||||
.content_language("cltest")
|
||||
.expires(exp2)
|
||||
.metadata("testmeta", "hello people")
|
||||
.metadata("nice-unicode-meta", "宅配便")
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(r.e_tag.unwrap().as_str(), etag);
|
||||
|
||||
let o = ctx
|
||||
.client
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(STD_KEY)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(o.e_tag.unwrap().as_str(), etag);
|
||||
assert_eq!(o.content_type.unwrap().as_str(), "application/test");
|
||||
assert_eq!(o.cache_control.unwrap().as_str(), "cctest");
|
||||
assert_eq!(o.content_disposition.unwrap().as_str(), "cdtest");
|
||||
assert_eq!(o.content_encoding.unwrap().as_str(), "cetest");
|
||||
assert_eq!(o.content_language.unwrap().as_str(), "cltest");
|
||||
assert_eq!(o.expires.unwrap(), exp2);
|
||||
let mut meta = o.metadata.unwrap();
|
||||
assert_eq!(meta.remove("testmeta").unwrap(), "hello people");
|
||||
assert_eq!(meta.remove("nice-unicode-meta").unwrap(), "宅配便");
|
||||
assert_eq!(meta.len(), 0);
|
||||
|
||||
let o = ctx
|
||||
.client
|
||||
.get_object()
|
||||
.bucket(&bucket)
|
||||
.key(STD_KEY)
|
||||
.response_content_type("application/x-dummy-test")
|
||||
.response_cache_control("ccdummy")
|
||||
.response_content_disposition("cddummy")
|
||||
.response_content_encoding("cedummy")
|
||||
.response_content_language("cldummy")
|
||||
.response_expires(exp)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(o.e_tag.unwrap().as_str(), etag);
|
||||
assert_eq!(o.content_type.unwrap().as_str(), "application/x-dummy-test");
|
||||
assert_eq!(o.cache_control.unwrap().as_str(), "ccdummy");
|
||||
assert_eq!(o.content_disposition.unwrap().as_str(), "cddummy");
|
||||
assert_eq!(o.content_encoding.unwrap().as_str(), "cedummy");
|
||||
assert_eq!(o.content_language.unwrap().as_str(), "cldummy");
|
||||
assert_eq!(o.expires.unwrap(), exp);
|
||||
assert_bytes_eq!(o.body, &BODY[..]);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use crate::common;
|
||||
use aws_sdk_s3::presigning::PresigningConfig;
|
||||
use bytes::Bytes;
|
||||
use http_body_util::{BodyExt, Full};
|
||||
use hyper::Request;
|
||||
|
||||
const STD_KEY: &str = "hello world";
|
||||
const BODY: &[u8; 62] = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_presigned_url() {
|
||||
let ctx = common::context();
|
||||
let bucket = ctx.create_bucket("presigned");
|
||||
|
||||
let etag = "\"46cf18a9b447991b450cad3facf5937e\"";
|
||||
let body = Bytes::from(BODY.to_vec());
|
||||
|
||||
let psc = PresigningConfig::builder()
|
||||
.start_time(SystemTime::now() - Duration::from_secs(60))
|
||||
.expires_in(Duration::from_secs(3600))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
{
|
||||
// PutObject
|
||||
let req = ctx
|
||||
.client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(STD_KEY)
|
||||
.presigned(psc.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let client = ctx.custom_request.client();
|
||||
let req = Request::builder()
|
||||
.method("PUT")
|
||||
.uri(req.uri())
|
||||
.body(Full::new(body.clone()))
|
||||
.unwrap();
|
||||
let res = client.request(req).await.unwrap();
|
||||
assert_eq!(res.status(), 200);
|
||||
assert_eq!(res.headers().get("etag").unwrap(), etag);
|
||||
}
|
||||
|
||||
{
|
||||
// GetObject
|
||||
let req = ctx
|
||||
.client
|
||||
.get_object()
|
||||
.bucket(&bucket)
|
||||
.key(STD_KEY)
|
||||
.presigned(psc)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let client = ctx.custom_request.client();
|
||||
let req = Request::builder()
|
||||
.method("GET")
|
||||
.uri(req.uri())
|
||||
.body(Full::new(Bytes::new()))
|
||||
.unwrap();
|
||||
let res = client.request(req).await.unwrap();
|
||||
assert_eq!(res.status(), 200);
|
||||
assert_eq!(res.headers().get("etag").unwrap(), etag);
|
||||
|
||||
let body2 = BodyExt::collect(res.into_body()).await.unwrap().to_bytes();
|
||||
assert_eq!(body, body2);
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_model"
|
||||
version = "0.9.1"
|
||||
version = "0.9.4"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
@@ -28,6 +28,7 @@ chrono.workspace = true
|
||||
err-derive.workspace = true
|
||||
hex.workspace = true
|
||||
base64.workspace = true
|
||||
parse_duration.workspace = true
|
||||
tracing.workspace = true
|
||||
rand.workspace = true
|
||||
zstd.workspace = true
|
||||
|
||||
@@ -191,6 +191,13 @@ impl Bucket {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn present(id: Uuid, params: BucketParams) -> Self {
|
||||
Bucket {
|
||||
id,
|
||||
state: crdt::Deletable::present(params),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns true if this represents a deleted bucket
|
||||
pub fn is_deleted(&self) -> bool {
|
||||
self.state.is_deleted()
|
||||
|
||||
+38
-102
@@ -1,3 +1,4 @@
|
||||
use std::str::FromStr;
|
||||
use std::sync::Arc;
|
||||
|
||||
use garage_net::NetworkKey;
|
||||
@@ -113,108 +114,33 @@ impl Garage {
|
||||
}
|
||||
|
||||
info!("Opening database...");
|
||||
let db_engine = db::Engine::from_str(&config.db_engine)
|
||||
.ok_or_message("Invalid `db_engine` value in configuration file")?;
|
||||
let mut db_path = config.metadata_dir.clone();
|
||||
let db = match config.db_engine.as_str() {
|
||||
// ---- Sled DB ----
|
||||
#[cfg(feature = "sled")]
|
||||
"sled" => {
|
||||
if config.metadata_fsync {
|
||||
return Err(Error::Message(format!(
|
||||
"`metadata_fsync = true` is not supported with the Sled database engine"
|
||||
)));
|
||||
}
|
||||
match db_engine {
|
||||
db::Engine::Sled => {
|
||||
db_path.push("db");
|
||||
info!("Opening Sled database at: {}", db_path.display());
|
||||
let db = db::sled_adapter::sled::Config::default()
|
||||
.path(&db_path)
|
||||
.cache_capacity(config.sled_cache_capacity as u64)
|
||||
.flush_every_ms(Some(config.sled_flush_every_ms))
|
||||
.open()
|
||||
.ok_or_message("Unable to open sled DB")?;
|
||||
db::sled_adapter::SledDb::init(db)
|
||||
}
|
||||
#[cfg(not(feature = "sled"))]
|
||||
"sled" => return Err(Error::Message("sled db not available in this build".into())),
|
||||
// ---- Sqlite DB ----
|
||||
#[cfg(feature = "sqlite")]
|
||||
"sqlite" | "sqlite3" | "rusqlite" => {
|
||||
db::Engine::Sqlite => {
|
||||
db_path.push("db.sqlite");
|
||||
info!("Opening Sqlite database at: {}", db_path.display());
|
||||
let db = db::sqlite_adapter::rusqlite::Connection::open(db_path)
|
||||
.and_then(|db| {
|
||||
db.pragma_update(None, "journal_mode", &"WAL")?;
|
||||
if config.metadata_fsync {
|
||||
db.pragma_update(None, "synchronous", &"NORMAL")?;
|
||||
} else {
|
||||
db.pragma_update(None, "synchronous", &"OFF")?;
|
||||
}
|
||||
Ok(db)
|
||||
})
|
||||
.ok_or_message("Unable to open sqlite DB")?;
|
||||
db::sqlite_adapter::SqliteDb::init(db)
|
||||
}
|
||||
#[cfg(not(feature = "sqlite"))]
|
||||
"sqlite" | "sqlite3" | "rusqlite" => {
|
||||
return Err(Error::Message(
|
||||
"sqlite db not available in this build".into(),
|
||||
))
|
||||
}
|
||||
// ---- LMDB DB ----
|
||||
#[cfg(feature = "lmdb")]
|
||||
"lmdb" | "heed" => {
|
||||
db::Engine::Lmdb => {
|
||||
db_path.push("db.lmdb");
|
||||
info!("Opening LMDB database at: {}", db_path.display());
|
||||
std::fs::create_dir_all(&db_path)
|
||||
.ok_or_message("Unable to create LMDB data directory")?;
|
||||
let map_size = match config.lmdb_map_size {
|
||||
v if v == usize::default() => garage_db::lmdb_adapter::recommended_map_size(),
|
||||
v => v - (v % 4096),
|
||||
};
|
||||
|
||||
use db::lmdb_adapter::heed;
|
||||
let mut env_builder = heed::EnvOpenOptions::new();
|
||||
env_builder.max_dbs(100);
|
||||
env_builder.max_readers(500);
|
||||
env_builder.map_size(map_size);
|
||||
unsafe {
|
||||
env_builder.flag(heed::flags::Flags::MdbNoMetaSync);
|
||||
if !config.metadata_fsync {
|
||||
env_builder.flag(heed::flags::Flags::MdbNoSync);
|
||||
}
|
||||
}
|
||||
let db = match env_builder.open(&db_path) {
|
||||
Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => {
|
||||
return Err(Error::Message(
|
||||
"OutOfMemory error while trying to open LMDB database. This can happen \
|
||||
if your operating system is not allowing you to use sufficient virtual \
|
||||
memory address space. Please check that no limit is set (ulimit -v). \
|
||||
You may also try to set a smaller `lmdb_map_size` configuration parameter. \
|
||||
On 32-bit machines, you should probably switch to another database engine.".into()))
|
||||
}
|
||||
x => x.ok_or_message("Unable to open LMDB DB")?,
|
||||
};
|
||||
db::lmdb_adapter::LmdbDb::init(db)
|
||||
}
|
||||
#[cfg(not(feature = "lmdb"))]
|
||||
"lmdb" | "heed" => return Err(Error::Message("lmdb db not available in this build".into())),
|
||||
// ---- Unavailable DB engine ----
|
||||
e => {
|
||||
return Err(Error::Message(format!(
|
||||
"Unsupported DB engine: {} (options: {})",
|
||||
e,
|
||||
vec![
|
||||
#[cfg(feature = "sled")]
|
||||
"sled",
|
||||
#[cfg(feature = "sqlite")]
|
||||
"sqlite",
|
||||
#[cfg(feature = "lmdb")]
|
||||
"lmdb",
|
||||
]
|
||||
.join(", ")
|
||||
)));
|
||||
}
|
||||
}
|
||||
let db_opt = db::OpenOpt {
|
||||
fsync: config.metadata_fsync,
|
||||
lmdb_map_size: match config.lmdb_map_size {
|
||||
v if v == usize::default() => None,
|
||||
v => Some(v),
|
||||
},
|
||||
sled_cache_capacity: config.sled_cache_capacity,
|
||||
sled_flush_every_ms: config.sled_flush_every_ms,
|
||||
};
|
||||
let db = db::open_db(&db_path, db_engine, &db_opt)
|
||||
.ok_or_message("Unable to open metadata db")?;
|
||||
|
||||
info!("Initializing RPC...");
|
||||
let network_key = hex::decode(config.rpc_secret.as_ref().ok_or_message(
|
||||
"rpc_secret value is missing, not present in config file or in environment",
|
||||
)?)
|
||||
@@ -251,14 +177,7 @@ impl Garage {
|
||||
};
|
||||
|
||||
info!("Initialize block manager...");
|
||||
let block_manager = BlockManager::new(
|
||||
&db,
|
||||
config.data_dir.clone(),
|
||||
config.data_fsync,
|
||||
config.compression_level,
|
||||
data_rep_param,
|
||||
system.clone(),
|
||||
)?;
|
||||
let block_manager = BlockManager::new(&db, &config, data_rep_param, system.clone())?;
|
||||
block_manager.register_bg_vars(&mut bg_vars);
|
||||
|
||||
// ---- admin tables ----
|
||||
@@ -359,7 +278,7 @@ impl Garage {
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn spawn_workers(self: &Arc<Self>, bg: &BackgroundRunner) {
|
||||
pub fn spawn_workers(self: &Arc<Self>, bg: &BackgroundRunner) -> Result<(), Error> {
|
||||
self.block_manager.spawn_workers(bg);
|
||||
|
||||
self.bucket_table.spawn_workers(bg);
|
||||
@@ -380,6 +299,23 @@ impl Garage {
|
||||
|
||||
#[cfg(feature = "k2v")]
|
||||
self.k2v.spawn_workers(bg);
|
||||
|
||||
if let Some(itv) = self.config.metadata_auto_snapshot_interval.as_deref() {
|
||||
let interval = parse_duration::parse(itv)
|
||||
.ok_or_message("Invalid `metadata_auto_snapshot_interval`")?;
|
||||
if interval < std::time::Duration::from_secs(600) {
|
||||
return Err(Error::Message(
|
||||
"metadata_auto_snapshot_interval too small or negative".into(),
|
||||
));
|
||||
}
|
||||
|
||||
bg.spawn_worker(crate::snapshot::AutoSnapshotWorker::new(
|
||||
self.clone(),
|
||||
interval,
|
||||
));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn bucket_helper(&self) -> helper::bucket::BucketHelper {
|
||||
|
||||
@@ -67,6 +67,49 @@ impl<'a> BucketHelper<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Find a bucket by its global alias or a prefix of its uuid
|
||||
pub async fn admin_get_existing_matching_bucket(
|
||||
&self,
|
||||
pattern: &String,
|
||||
) -> Result<Uuid, Error> {
|
||||
if let Some(uuid) = self.resolve_global_bucket_name(pattern).await? {
|
||||
return Ok(uuid);
|
||||
} else if pattern.len() >= 2 {
|
||||
let hexdec = pattern
|
||||
.get(..pattern.len() & !1)
|
||||
.and_then(|x| hex::decode(x).ok());
|
||||
if let Some(hex) = hexdec {
|
||||
let mut start = [0u8; 32];
|
||||
start
|
||||
.as_mut_slice()
|
||||
.get_mut(..hex.len())
|
||||
.ok_or_bad_request("invalid length")?
|
||||
.copy_from_slice(&hex);
|
||||
let mut candidates = self
|
||||
.0
|
||||
.bucket_table
|
||||
.get_range(
|
||||
&EmptyKey,
|
||||
Some(start.into()),
|
||||
Some(DeletedFilter::NotDeleted),
|
||||
10,
|
||||
EnumerationOrder::Forward,
|
||||
)
|
||||
.await?
|
||||
.into_iter()
|
||||
.collect::<Vec<_>>();
|
||||
candidates.retain(|x| hex::encode(x.id).starts_with(pattern));
|
||||
if candidates.len() == 1 {
|
||||
return Ok(candidates.into_iter().next().unwrap().id);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(Error::BadRequest(format!(
|
||||
"Bucket not found / several matching buckets: {}",
|
||||
pattern
|
||||
)))
|
||||
}
|
||||
|
||||
/// Returns a Bucket if it is present in bucket table,
|
||||
/// even if it is in deleted state. Querying a non-existing
|
||||
/// bucket ID returns an internal error.
|
||||
|
||||
@@ -300,7 +300,11 @@ impl K2VRpcHandler {
|
||||
let rs = RequestStrategy::with_priority(PRIO_NORMAL).without_timeout();
|
||||
let mut requests = nodes
|
||||
.iter()
|
||||
.map(|node| self.system.rpc.call(&self.endpoint, *node, msg.clone(), rs))
|
||||
.map(|node| {
|
||||
self.system
|
||||
.rpc
|
||||
.call(&self.endpoint, *node, msg.clone(), rs.clone())
|
||||
})
|
||||
.collect::<FuturesUnordered<_>>();
|
||||
|
||||
// Fetch responses. This procedure stops fetching responses when any of the following
|
||||
|
||||
@@ -19,3 +19,4 @@ pub mod s3;
|
||||
pub mod garage;
|
||||
pub mod helper;
|
||||
pub mod migrate;
|
||||
pub mod snapshot;
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::Mutex;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use rand::prelude::*;
|
||||
use tokio::sync::watch;
|
||||
|
||||
use garage_util::background::*;
|
||||
use garage_util::error::*;
|
||||
|
||||
use crate::garage::Garage;
|
||||
|
||||
// The two most recent snapshots are kept
|
||||
const KEEP_SNAPSHOTS: usize = 2;
|
||||
|
||||
static SNAPSHOT_MUTEX: Mutex<()> = Mutex::new(());
|
||||
|
||||
// ================ snapshotting logic =====================
|
||||
|
||||
/// Run snashot_metadata in a blocking thread and async await on it
|
||||
pub async fn async_snapshot_metadata(garage: &Arc<Garage>) -> Result<(), Error> {
|
||||
let garage = garage.clone();
|
||||
let worker = tokio::task::spawn_blocking(move || snapshot_metadata(&garage));
|
||||
worker.await.unwrap()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Take a snapshot of the metadata database, and erase older
|
||||
/// snapshots if necessary.
|
||||
/// This is not an async function, it should be spawned on a thread pool
|
||||
pub fn snapshot_metadata(garage: &Garage) -> Result<(), Error> {
|
||||
let lock = match SNAPSHOT_MUTEX.try_lock() {
|
||||
Ok(lock) => lock,
|
||||
Err(_) => {
|
||||
return Err(Error::Message(
|
||||
"Cannot acquire lock, another snapshot might be in progress".into(),
|
||||
))
|
||||
}
|
||||
};
|
||||
|
||||
let mut snapshots_dir = garage.config.metadata_dir.clone();
|
||||
snapshots_dir.push("snapshots");
|
||||
fs::create_dir_all(&snapshots_dir)?;
|
||||
|
||||
let mut new_path = snapshots_dir.clone();
|
||||
new_path.push(chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true));
|
||||
|
||||
info!("Snapshotting metadata db to {}", new_path.display());
|
||||
garage.db.snapshot(&new_path)?;
|
||||
info!("Metadata db snapshot finished");
|
||||
|
||||
if let Err(e) = cleanup_snapshots(&snapshots_dir) {
|
||||
error!("Failed to do cleanup in snapshots directory: {}", e);
|
||||
}
|
||||
|
||||
drop(lock);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cleanup_snapshots(snapshots_dir: &PathBuf) -> Result<(), Error> {
|
||||
let mut snapshots =
|
||||
fs::read_dir(&snapshots_dir)?.collect::<Result<Vec<fs::DirEntry>, std::io::Error>>()?;
|
||||
|
||||
snapshots.retain(|x| x.file_name().len() > 8);
|
||||
snapshots.sort_by_key(|x| x.file_name());
|
||||
|
||||
for to_delete in snapshots.iter().rev().skip(KEEP_SNAPSHOTS) {
|
||||
let path = snapshots_dir.join(to_delete.path());
|
||||
if to_delete.metadata()?.file_type().is_dir() {
|
||||
for file in fs::read_dir(&path)? {
|
||||
let file = file?;
|
||||
if file.metadata()?.is_file() {
|
||||
fs::remove_file(path.join(file.path()))?;
|
||||
}
|
||||
}
|
||||
std::fs::remove_dir(&path)?;
|
||||
} else {
|
||||
std::fs::remove_file(&path)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ================ auto snapshot worker =====================
|
||||
|
||||
pub struct AutoSnapshotWorker {
|
||||
garage: Arc<Garage>,
|
||||
next_snapshot: Instant,
|
||||
snapshot_interval: Duration,
|
||||
}
|
||||
|
||||
impl AutoSnapshotWorker {
|
||||
pub(crate) fn new(garage: Arc<Garage>, snapshot_interval: Duration) -> Self {
|
||||
Self {
|
||||
garage,
|
||||
snapshot_interval,
|
||||
next_snapshot: Instant::now() + (snapshot_interval / 2),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Worker for AutoSnapshotWorker {
|
||||
fn name(&self) -> String {
|
||||
"Metadata snapshot worker".into()
|
||||
}
|
||||
fn status(&self) -> WorkerStatus {
|
||||
WorkerStatus {
|
||||
freeform: vec![format!(
|
||||
"Next snapshot: {}",
|
||||
(chrono::Utc::now() + (self.next_snapshot - Instant::now())).to_rfc3339()
|
||||
)],
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
|
||||
if Instant::now() < self.next_snapshot {
|
||||
return Ok(WorkerState::Idle);
|
||||
}
|
||||
|
||||
async_snapshot_metadata(&self.garage).await?;
|
||||
|
||||
let rand_factor = 1f32 + thread_rng().gen::<f32>() / 5f32;
|
||||
self.next_snapshot = Instant::now() + self.snapshot_interval.mul_f32(rand_factor);
|
||||
|
||||
Ok(WorkerState::Idle)
|
||||
}
|
||||
async fn wait_for_work(&mut self) -> WorkerState {
|
||||
tokio::time::sleep_until(self.next_snapshot.into()).await;
|
||||
WorkerState::Busy
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -1,9 +1,9 @@
|
||||
[package]
|
||||
name = "garage_net"
|
||||
version = "0.9.1"
|
||||
version = "0.9.4"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license-file = "AGPL-3.0"
|
||||
license = "AGPL-3.0"
|
||||
description = "Networking library for Garage RPC communication, forked from Netapp"
|
||||
repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage"
|
||||
readme = "../../README.md"
|
||||
|
||||
@@ -28,12 +28,30 @@ use crate::util::*;
|
||||
/// The same priority value is given to a request and to its associated response.
|
||||
pub type RequestPriority = u8;
|
||||
|
||||
// Usage of priority levels in Garage:
|
||||
//
|
||||
// PRIO_HIGH
|
||||
// for liveness check events such as pings and important
|
||||
// reconfiguration events such as layout changes
|
||||
//
|
||||
// PRIO_NORMAL
|
||||
// for standard interactive requests to exchange metadata
|
||||
//
|
||||
// PRIO_NORMAL | PRIO_SECONDARY
|
||||
// for standard interactive requests to exchange block data
|
||||
//
|
||||
// PRIO_BACKGROUND
|
||||
// for background resync requests to exchange metadata
|
||||
// PRIO_BACKGROUND | PRIO_SECONDARY
|
||||
// for background resync requests to exchange block data
|
||||
|
||||
/// Priority class: high
|
||||
pub const PRIO_HIGH: RequestPriority = 0x20;
|
||||
/// Priority class: normal
|
||||
pub const PRIO_NORMAL: RequestPriority = 0x40;
|
||||
/// Priority class: background
|
||||
pub const PRIO_BACKGROUND: RequestPriority = 0x80;
|
||||
|
||||
/// Priority: primary among given class
|
||||
pub const PRIO_PRIMARY: RequestPriority = 0x00;
|
||||
/// Priority: secondary among given class (ex: `PRIO_HIGH | PRIO_SECONDARY`)
|
||||
|
||||
+42
-37
@@ -109,7 +109,7 @@ impl SendQueuePriority {
|
||||
let i = order_vec.iter().take_while(|o2| **o2 < order).count();
|
||||
order_vec.insert(i, order);
|
||||
}
|
||||
self.items.push_front(item);
|
||||
self.items.push_back(item);
|
||||
}
|
||||
fn remove(&mut self, id: RequestID) {
|
||||
if let Some(i) = self.items.iter().position(|x| x.id == id) {
|
||||
@@ -128,51 +128,56 @@ impl SendQueuePriority {
|
||||
self.items.is_empty()
|
||||
}
|
||||
fn poll_next_ready(&mut self, ctx: &mut Context<'_>) -> Poll<(RequestID, DataFrame)> {
|
||||
for (j, item) in self.items.iter_mut().enumerate() {
|
||||
if let Some(OrderTag(stream, order)) = item.order_tag {
|
||||
if order > *self.order.get(&stream).unwrap().front().unwrap() {
|
||||
// in step 1: poll only streams that have sent 0 bytes, we want to send them in priority
|
||||
// as they most likely represent small requests to be sent first
|
||||
// in step 2: poll all streams
|
||||
for step in 0..2 {
|
||||
for (j, item) in self.items.iter_mut().enumerate() {
|
||||
if let Some(OrderTag(stream, order)) = item.order_tag {
|
||||
if order > *self.order.get(&stream).unwrap().front().unwrap() {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if step == 0 && item.sent > 0 {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let mut item_reader = item.data.read_exact_or_eos(MAX_CHUNK_LENGTH as usize);
|
||||
if let Poll::Ready(bytes_or_err) = Pin::new(&mut item_reader).poll(ctx) {
|
||||
let id = item.id;
|
||||
let eos = item.data.eos();
|
||||
let mut item_reader = item.data.read_exact_or_eos(MAX_CHUNK_LENGTH as usize);
|
||||
if let Poll::Ready(bytes_or_err) = Pin::new(&mut item_reader).poll(ctx) {
|
||||
let id = item.id;
|
||||
let eos = item.data.eos();
|
||||
|
||||
let packet = bytes_or_err.map_err(|e| match e {
|
||||
ReadExactError::Stream(err) => err,
|
||||
_ => unreachable!(),
|
||||
});
|
||||
let packet = bytes_or_err.map_err(|e| match e {
|
||||
ReadExactError::Stream(err) => err,
|
||||
_ => unreachable!(),
|
||||
});
|
||||
|
||||
let is_err = packet.is_err();
|
||||
let data_frame = DataFrame::from_packet(packet, !eos);
|
||||
item.sent += data_frame.data().len();
|
||||
let is_err = packet.is_err();
|
||||
let data_frame = DataFrame::from_packet(packet, !eos);
|
||||
item.sent += data_frame.data().len();
|
||||
|
||||
if eos || is_err {
|
||||
// If item had an order tag, remove it from the corresponding ordering list
|
||||
if let Some(OrderTag(stream, order)) = item.order_tag {
|
||||
let order_stream = self.order.get_mut(&stream).unwrap();
|
||||
assert_eq!(order_stream.pop_front(), Some(order));
|
||||
if order_stream.is_empty() {
|
||||
self.order.remove(&stream);
|
||||
}
|
||||
}
|
||||
// Remove item from sending queue
|
||||
self.items.remove(j);
|
||||
} else {
|
||||
// Move item later in send queue to implement LAS scheduling
|
||||
// (LAS = Least Attained Service)
|
||||
for k in j..self.items.len() - 1 {
|
||||
if self.items[k].sent >= self.items[k + 1].sent {
|
||||
self.items.swap(k, k + 1);
|
||||
} else {
|
||||
break;
|
||||
if eos || is_err {
|
||||
// If item had an order tag, remove it from the corresponding ordering list
|
||||
if let Some(OrderTag(stream, order)) = item.order_tag {
|
||||
let order_stream = self.order.get_mut(&stream).unwrap();
|
||||
assert_eq!(order_stream.pop_front(), Some(order));
|
||||
if order_stream.is_empty() {
|
||||
self.order.remove(&stream);
|
||||
}
|
||||
}
|
||||
// Remove item from sending queue
|
||||
self.items.remove(j);
|
||||
} else if step == 0 {
|
||||
// Step 0 means that this stream had not sent any bytes yet.
|
||||
// Now that it has, and it was not an EOS, we know that it is bigger
|
||||
// than one chunk so move it at the end of the queue.
|
||||
let item = self.items.remove(j).unwrap();
|
||||
self.items.push_back(item);
|
||||
}
|
||||
|
||||
return Poll::Ready((id, data_frame));
|
||||
}
|
||||
|
||||
return Poll::Ready((id, data_frame));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -190,7 +190,7 @@ impl RecvLoop for ServerConn {
|
||||
|
||||
let (prio, resp_enc_result) = match ReqEnc::decode(stream).await {
|
||||
Ok(req_enc) => (req_enc.prio, self2.recv_handler_aux(req_enc).await),
|
||||
Err(e) => (PRIO_HIGH, Err(e)),
|
||||
Err(e) => (PRIO_NORMAL, Err(e)),
|
||||
};
|
||||
|
||||
debug!("server: sending response to {}", id);
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_rpc"
|
||||
version = "0.9.1"
|
||||
version = "0.9.4"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
|
||||
+57
-11
@@ -33,8 +33,7 @@ use crate::ring::Ring;
|
||||
const DEFAULT_TIMEOUT: Duration = Duration::from_secs(300);
|
||||
|
||||
/// Strategy to apply when making RPC
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct RequestStrategy {
|
||||
pub struct RequestStrategy<T> {
|
||||
/// Min number of response to consider the request successful
|
||||
pub rs_quorum: Option<usize>,
|
||||
/// Should requests be dropped after enough response are received
|
||||
@@ -43,6 +42,8 @@ pub struct RequestStrategy {
|
||||
pub rs_priority: RequestPriority,
|
||||
/// Custom timeout for this request
|
||||
rs_timeout: Timeout,
|
||||
/// Data to drop when everything completes
|
||||
rs_drop_on_complete: T,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone)]
|
||||
@@ -52,7 +53,19 @@ enum Timeout {
|
||||
Custom(Duration),
|
||||
}
|
||||
|
||||
impl RequestStrategy {
|
||||
impl Clone for RequestStrategy<()> {
|
||||
fn clone(&self) -> Self {
|
||||
RequestStrategy {
|
||||
rs_quorum: self.rs_quorum,
|
||||
rs_interrupt_after_quorum: self.rs_interrupt_after_quorum,
|
||||
rs_priority: self.rs_priority,
|
||||
rs_timeout: self.rs_timeout,
|
||||
rs_drop_on_complete: (),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RequestStrategy<()> {
|
||||
/// Create a RequestStrategy with default timeout and not interrupting when quorum reached
|
||||
pub fn with_priority(prio: RequestPriority) -> Self {
|
||||
RequestStrategy {
|
||||
@@ -60,8 +73,22 @@ impl RequestStrategy {
|
||||
rs_interrupt_after_quorum: false,
|
||||
rs_priority: prio,
|
||||
rs_timeout: Timeout::Default,
|
||||
rs_drop_on_complete: (),
|
||||
}
|
||||
}
|
||||
/// Add an item to be dropped on completion
|
||||
pub fn with_drop_on_completion<T>(self, drop_on_complete: T) -> RequestStrategy<T> {
|
||||
RequestStrategy {
|
||||
rs_quorum: self.rs_quorum,
|
||||
rs_interrupt_after_quorum: self.rs_interrupt_after_quorum,
|
||||
rs_priority: self.rs_priority,
|
||||
rs_timeout: self.rs_timeout,
|
||||
rs_drop_on_complete: drop_on_complete,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> RequestStrategy<T> {
|
||||
/// Set quorum to be reached for request
|
||||
pub fn with_quorum(mut self, quorum: usize) -> Self {
|
||||
self.rs_quorum = Some(quorum);
|
||||
@@ -83,6 +110,19 @@ impl RequestStrategy {
|
||||
self.rs_timeout = Timeout::Custom(timeout);
|
||||
self
|
||||
}
|
||||
/// Extract drop_on_complete item
|
||||
fn extract_drop_on_complete(self) -> (RequestStrategy<()>, T) {
|
||||
(
|
||||
RequestStrategy {
|
||||
rs_quorum: self.rs_quorum,
|
||||
rs_interrupt_after_quorum: self.rs_interrupt_after_quorum,
|
||||
rs_priority: self.rs_priority,
|
||||
rs_timeout: self.rs_timeout,
|
||||
rs_drop_on_complete: (),
|
||||
},
|
||||
self.rs_drop_on_complete,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -123,7 +163,7 @@ impl RpcHelper {
|
||||
endpoint: &Endpoint<M, H>,
|
||||
to: Uuid,
|
||||
msg: N,
|
||||
strat: RequestStrategy,
|
||||
strat: RequestStrategy<()>,
|
||||
) -> Result<S, Error>
|
||||
where
|
||||
M: Rpc<Response = Result<S, Error>>,
|
||||
@@ -176,7 +216,7 @@ impl RpcHelper {
|
||||
endpoint: &Endpoint<M, H>,
|
||||
to: &[Uuid],
|
||||
msg: N,
|
||||
strat: RequestStrategy,
|
||||
strat: RequestStrategy<()>,
|
||||
) -> Result<Vec<(Uuid, Result<S, Error>)>, Error>
|
||||
where
|
||||
M: Rpc<Response = Result<S, Error>>,
|
||||
@@ -187,7 +227,7 @@ impl RpcHelper {
|
||||
|
||||
let resps = join_all(
|
||||
to.iter()
|
||||
.map(|to| self.call(endpoint, *to, msg.clone(), strat)),
|
||||
.map(|to| self.call(endpoint, *to, msg.clone(), strat.clone())),
|
||||
)
|
||||
.await;
|
||||
Ok(to
|
||||
@@ -201,7 +241,7 @@ impl RpcHelper {
|
||||
&self,
|
||||
endpoint: &Endpoint<M, H>,
|
||||
msg: N,
|
||||
strat: RequestStrategy,
|
||||
strat: RequestStrategy<()>,
|
||||
) -> Result<Vec<(Uuid, Result<S, Error>)>, Error>
|
||||
where
|
||||
M: Rpc<Response = Result<S, Error>>,
|
||||
@@ -220,18 +260,19 @@ impl RpcHelper {
|
||||
|
||||
/// Make a RPC call to multiple servers, returning either a Vec of responses,
|
||||
/// or an error if quorum could not be reached due to too many errors
|
||||
pub async fn try_call_many<M, N, H, S>(
|
||||
pub async fn try_call_many<M, N, H, S, T>(
|
||||
&self,
|
||||
endpoint: &Arc<Endpoint<M, H>>,
|
||||
to: &[Uuid],
|
||||
msg: N,
|
||||
strategy: RequestStrategy,
|
||||
strategy: RequestStrategy<T>,
|
||||
) -> Result<Vec<S>, Error>
|
||||
where
|
||||
M: Rpc<Response = Result<S, Error>> + 'static,
|
||||
N: IntoReq<M>,
|
||||
H: StreamingEndpointHandler<M> + 'static,
|
||||
S: Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
let quorum = strategy.rs_quorum.unwrap_or(to.len());
|
||||
|
||||
@@ -260,12 +301,12 @@ impl RpcHelper {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn try_call_many_internal<M, N, H, S>(
|
||||
async fn try_call_many_internal<M, N, H, S, T>(
|
||||
&self,
|
||||
endpoint: &Arc<Endpoint<M, H>>,
|
||||
to: &[Uuid],
|
||||
msg: N,
|
||||
strategy: RequestStrategy,
|
||||
strategy: RequestStrategy<T>,
|
||||
quorum: usize,
|
||||
) -> Result<Vec<S>, Error>
|
||||
where
|
||||
@@ -273,9 +314,12 @@ impl RpcHelper {
|
||||
N: IntoReq<M>,
|
||||
H: StreamingEndpointHandler<M> + 'static,
|
||||
S: Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
let msg = msg.into_req().map_err(garage_net::error::Error::from)?;
|
||||
|
||||
let (strategy, drop_on_complete) = strategy.extract_drop_on_complete();
|
||||
|
||||
// Build future for each request
|
||||
// They are not started now: they are added below in a FuturesUnordered
|
||||
// object that will take care of polling them (see below)
|
||||
@@ -283,6 +327,7 @@ impl RpcHelper {
|
||||
let self2 = self.clone();
|
||||
let msg = msg.clone();
|
||||
let endpoint2 = endpoint.clone();
|
||||
let strategy = strategy.clone();
|
||||
(to, async move {
|
||||
self2.call(&endpoint2, to, msg, strategy).await
|
||||
})
|
||||
@@ -377,6 +422,7 @@ impl RpcHelper {
|
||||
// they have to be put in a proper queue that is persisted to disk.
|
||||
tokio::spawn(async move {
|
||||
resp_stream.collect::<Vec<Result<_, _>>>().await;
|
||||
drop(drop_on_complete);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_table"
|
||||
version = "0.9.1"
|
||||
version = "0.9.4"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_util"
|
||||
version = "0.9.1"
|
||||
version = "0.9.4"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
|
||||
@@ -23,6 +23,14 @@ pub struct Config {
|
||||
#[serde(default)]
|
||||
pub data_fsync: bool,
|
||||
|
||||
/// Disable automatic scrubbing of the data directory
|
||||
#[serde(default)]
|
||||
pub disable_scrub: bool,
|
||||
|
||||
/// Automatic snapshot interval for metadata
|
||||
#[serde(default)]
|
||||
pub metadata_auto_snapshot_interval: Option<String>,
|
||||
|
||||
/// Size of data blocks to save to disk
|
||||
#[serde(
|
||||
deserialize_with = "deserialize_capacity",
|
||||
@@ -44,6 +52,14 @@ pub struct Config {
|
||||
)]
|
||||
pub compression_level: Option<i32>,
|
||||
|
||||
/// Maximum amount of block data to buffer in RAM for sending to
|
||||
/// remote nodes when these nodes are on slower links
|
||||
#[serde(
|
||||
deserialize_with = "deserialize_capacity",
|
||||
default = "default_block_ram_buffer_max"
|
||||
)]
|
||||
pub block_ram_buffer_max: usize,
|
||||
|
||||
/// Skip the permission check of secret files. Useful when
|
||||
/// POSIX ACLs (or more complex chmods) are used.
|
||||
#[serde(default)]
|
||||
@@ -247,6 +263,9 @@ fn default_sled_flush_every_ms() -> u64 {
|
||||
fn default_block_size() -> usize {
|
||||
1048576
|
||||
}
|
||||
fn default_block_ram_buffer_max() -> usize {
|
||||
256 * 1024 * 1024
|
||||
}
|
||||
|
||||
fn default_compression() -> Option<i32> {
|
||||
Some(1)
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_web"
|
||||
version = "0.9.1"
|
||||
version = "0.9.4"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
|
||||
+15
-14
@@ -26,7 +26,7 @@ use garage_api::s3::cors::{add_cors_headers, find_matching_cors_rule, handle_opt
|
||||
use garage_api::s3::error::{
|
||||
CommonErrorDerivative, Error as ApiError, OkOrBadRequest, OkOrInternalError,
|
||||
};
|
||||
use garage_api::s3::get::{handle_get, handle_head};
|
||||
use garage_api::s3::get::{handle_get_without_ctx, handle_head_without_ctx};
|
||||
|
||||
use garage_model::garage::Garage;
|
||||
|
||||
@@ -219,14 +219,13 @@ impl WebServer {
|
||||
// Check bucket isn't deleted and has website access enabled
|
||||
let bucket = self
|
||||
.garage
|
||||
.bucket_table
|
||||
.get(&EmptyKey, &bucket_id)
|
||||
.await?
|
||||
.ok_or(Error::NotFound)?;
|
||||
.bucket_helper()
|
||||
.get_existing_bucket(bucket_id)
|
||||
.await
|
||||
.map_err(|_| Error::NotFound)?;
|
||||
let bucket_params = bucket.state.into_option().unwrap();
|
||||
|
||||
let website_config = bucket
|
||||
.params()
|
||||
.ok_or(Error::NotFound)?
|
||||
let website_config = bucket_params
|
||||
.website_config
|
||||
.get()
|
||||
.as_ref()
|
||||
@@ -243,14 +242,16 @@ impl WebServer {
|
||||
);
|
||||
|
||||
let ret_doc = match *req.method() {
|
||||
Method::OPTIONS => handle_options_for_bucket(req, &bucket)
|
||||
Method::OPTIONS => handle_options_for_bucket(req, &bucket_params)
|
||||
.map_err(ApiError::from)
|
||||
.map(|res| res.map(|_empty_body: EmptyBody| empty_body())),
|
||||
Method::HEAD => handle_head(self.garage.clone(), &req, bucket_id, &key, None).await,
|
||||
Method::HEAD => {
|
||||
handle_head_without_ctx(self.garage.clone(), req, bucket_id, &key, None).await
|
||||
}
|
||||
Method::GET => {
|
||||
handle_get(
|
||||
handle_get_without_ctx(
|
||||
self.garage.clone(),
|
||||
&req,
|
||||
req,
|
||||
bucket_id,
|
||||
&key,
|
||||
None,
|
||||
@@ -301,7 +302,7 @@ impl WebServer {
|
||||
.body(empty_body::<Infallible>())
|
||||
.unwrap();
|
||||
|
||||
match handle_get(
|
||||
match handle_get_without_ctx(
|
||||
self.garage.clone(),
|
||||
&req2,
|
||||
bucket_id,
|
||||
@@ -344,7 +345,7 @@ impl WebServer {
|
||||
}
|
||||
Ok(mut resp) => {
|
||||
// Maybe add CORS headers
|
||||
if let Some(rule) = find_matching_cors_rule(&bucket, req)? {
|
||||
if let Some(rule) = find_matching_cors_rule(&bucket_params, req)? {
|
||||
add_cors_headers(&mut resp, rule)
|
||||
.ok_or_internal_error("Invalid bucket CORS configuration")?;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user