mirror of
https://github.com/deuxfleurs-org/garage.git
synced 2026-08-15 08:33:12 +00:00
Compare commits
9 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Alex Auvolat
|
398ddb1300 | ||
|
trinity-1686a
|
945b75dbf1 | ||
|
trinity-1686a
|
ca7b438f3f | ||
|
trinity-1686a
|
1eb972b1ac | ||
|
trinity-1686a
|
60d4459926 | ||
|
Quentin Dufour
|
3b3a1f275f | ||
|
Quentin Dufour
|
dba9af2968 | ||
|
Quentin Dufour
|
e9358054ac | ||
|
Quentin Dufour
|
f9e5520ffb |
Generated
+49
-7
@@ -106,6 +106,9 @@ name = "cc"
|
||||
version = "1.0.71"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "79c2681d6594606957bbb8631c4b90a7fcaaa72cdb714743a437b156d6a7eedd"
|
||||
dependencies = [
|
||||
"jobserver",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
@@ -379,7 +382,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "garage"
|
||||
version = "0.5.0"
|
||||
version = "0.5.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes 1.1.0",
|
||||
@@ -408,7 +411,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "garage_api"
|
||||
version = "0.5.0"
|
||||
version = "0.5.1"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"bytes 1.1.0",
|
||||
@@ -440,7 +443,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "garage_model"
|
||||
version = "0.5.0"
|
||||
version = "0.5.1"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -458,11 +461,12 @@ dependencies = [
|
||||
"serde_bytes",
|
||||
"sled",
|
||||
"tokio",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "garage_rpc"
|
||||
version = "0.5.0"
|
||||
version = "0.5.1"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -487,7 +491,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "garage_table"
|
||||
version = "0.5.0"
|
||||
version = "0.5.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes 1.1.0",
|
||||
@@ -507,7 +511,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "garage_util"
|
||||
version = "0.5.0"
|
||||
version = "0.5.1"
|
||||
dependencies = [
|
||||
"blake2",
|
||||
"chrono",
|
||||
@@ -531,7 +535,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "garage_web"
|
||||
version = "0.5.0"
|
||||
version = "0.5.1"
|
||||
dependencies = [
|
||||
"err-derive 0.3.0",
|
||||
"futures",
|
||||
@@ -752,6 +756,15 @@ version = "0.4.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
|
||||
|
||||
[[package]]
|
||||
name = "jobserver"
|
||||
version = "0.1.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "kuska-handshake"
|
||||
version = "0.2.0"
|
||||
@@ -1652,3 +1665,32 @@ name = "xxhash-rust"
|
||||
version = "0.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e575e15bedf6e57b5c2d763ffc6c3c760143466cbd09d762d539680ab5992ded"
|
||||
|
||||
[[package]]
|
||||
name = "zstd"
|
||||
version = "0.9.0+zstd.1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "07749a5dc2cb6b36661290245e350f15ec3bbb304e493db54a1d354480522ccd"
|
||||
dependencies = [
|
||||
"zstd-safe",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd-safe"
|
||||
version = "4.1.1+zstd.1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c91c90f2c593b003603e5e0493c837088df4469da25aafff8bce42ba48caf079"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"zstd-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd-sys"
|
||||
version = "1.6.1+zstd.1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "615120c7a2431d16cf1cf979e7fc31ba7a5b5e5707b29c8a99e5dbf8a8392a33"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
]
|
||||
|
||||
@@ -40,13 +40,13 @@ in
|
||||
{
|
||||
cargo2nixVersion = "0.9.0";
|
||||
workspace = {
|
||||
garage_util = rustPackages.unknown.garage_util."0.5.0";
|
||||
garage_rpc = rustPackages.unknown.garage_rpc."0.5.0";
|
||||
garage_table = rustPackages.unknown.garage_table."0.5.0";
|
||||
garage_model = rustPackages.unknown.garage_model."0.5.0";
|
||||
garage_api = rustPackages.unknown.garage_api."0.5.0";
|
||||
garage_web = rustPackages.unknown.garage_web."0.5.0";
|
||||
garage = rustPackages.unknown.garage."0.5.0";
|
||||
garage_util = rustPackages.unknown.garage_util."0.5.1";
|
||||
garage_rpc = rustPackages.unknown.garage_rpc."0.5.1";
|
||||
garage_table = rustPackages.unknown.garage_table."0.5.1";
|
||||
garage_model = rustPackages.unknown.garage_model."0.5.1";
|
||||
garage_api = rustPackages.unknown.garage_api."0.5.1";
|
||||
garage_web = rustPackages.unknown.garage_web."0.5.1";
|
||||
garage = rustPackages.unknown.garage."0.5.1";
|
||||
};
|
||||
"registry+https://github.com/rust-lang/crates.io-index".aho-corasick."0.7.18" = overridableMkRustCrate (profileName: rec {
|
||||
name = "aho-corasick";
|
||||
@@ -196,6 +196,13 @@ in
|
||||
version = "1.0.71";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "79c2681d6594606957bbb8631c4b90a7fcaaa72cdb714743a437b156d6a7eedd"; };
|
||||
features = builtins.concatLists [
|
||||
[ "jobserver" ]
|
||||
[ "parallel" ]
|
||||
];
|
||||
dependencies = {
|
||||
jobserver = rustPackages."registry+https://github.com/rust-lang/crates.io-index".jobserver."0.1.24" { inherit profileName; };
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" = overridableMkRustCrate (profileName: rec {
|
||||
@@ -246,7 +253,7 @@ in
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469"; };
|
||||
dependencies = {
|
||||
${ if hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" || hostPlatform.config == "aarch64-apple-darwin" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.103" { inherit profileName; };
|
||||
${ if hostPlatform.config == "aarch64-apple-darwin" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.103" { inherit profileName; };
|
||||
};
|
||||
});
|
||||
|
||||
@@ -606,9 +613,9 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"unknown".garage."0.5.0" = overridableMkRustCrate (profileName: rec {
|
||||
"unknown".garage."0.5.1" = overridableMkRustCrate (profileName: rec {
|
||||
name = "garage";
|
||||
version = "0.5.0";
|
||||
version = "0.5.1";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/garage");
|
||||
dependencies = {
|
||||
@@ -616,12 +623,12 @@ in
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.17" { inherit profileName; };
|
||||
futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.17" { inherit profileName; };
|
||||
garage_api = rustPackages."unknown".garage_api."0.5.0" { inherit profileName; };
|
||||
garage_model = rustPackages."unknown".garage_model."0.5.0" { inherit profileName; };
|
||||
garage_rpc = rustPackages."unknown".garage_rpc."0.5.0" { inherit profileName; };
|
||||
garage_table = rustPackages."unknown".garage_table."0.5.0" { inherit profileName; };
|
||||
garage_util = rustPackages."unknown".garage_util."0.5.0" { inherit profileName; };
|
||||
garage_web = rustPackages."unknown".garage_web."0.5.0" { inherit profileName; };
|
||||
garage_api = rustPackages."unknown".garage_api."0.5.1" { inherit profileName; };
|
||||
garage_model = rustPackages."unknown".garage_model."0.5.1" { inherit profileName; };
|
||||
garage_rpc = rustPackages."unknown".garage_rpc."0.5.1" { inherit profileName; };
|
||||
garage_table = rustPackages."unknown".garage_table."0.5.1" { inherit profileName; };
|
||||
garage_util = rustPackages."unknown".garage_util."0.5.1" { inherit profileName; };
|
||||
garage_web = rustPackages."unknown".garage_web."0.5.1" { inherit profileName; };
|
||||
git_version = rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; };
|
||||
hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
|
||||
sodiumoxide = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; };
|
||||
@@ -638,9 +645,9 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"unknown".garage_api."0.5.0" = overridableMkRustCrate (profileName: rec {
|
||||
"unknown".garage_api."0.5.1" = overridableMkRustCrate (profileName: rec {
|
||||
name = "garage_api";
|
||||
version = "0.5.0";
|
||||
version = "0.5.1";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/api");
|
||||
dependencies = {
|
||||
@@ -651,9 +658,9 @@ in
|
||||
err_derive = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.0" { profileName = "__noProfile"; };
|
||||
futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.17" { inherit profileName; };
|
||||
futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.17" { inherit profileName; };
|
||||
garage_model = rustPackages."unknown".garage_model."0.5.0" { inherit profileName; };
|
||||
garage_table = rustPackages."unknown".garage_table."0.5.0" { inherit profileName; };
|
||||
garage_util = rustPackages."unknown".garage_util."0.5.0" { inherit profileName; };
|
||||
garage_model = rustPackages."unknown".garage_model."0.5.1" { inherit profileName; };
|
||||
garage_table = rustPackages."unknown".garage_table."0.5.1" { inherit profileName; };
|
||||
garage_util = rustPackages."unknown".garage_util."0.5.1" { inherit profileName; };
|
||||
hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
|
||||
hmac = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.10.1" { inherit profileName; };
|
||||
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.5" { inherit profileName; };
|
||||
@@ -673,9 +680,9 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"unknown".garage_model."0.5.0" = overridableMkRustCrate (profileName: rec {
|
||||
"unknown".garage_model."0.5.1" = overridableMkRustCrate (profileName: rec {
|
||||
name = "garage_model";
|
||||
version = "0.5.0";
|
||||
version = "0.5.1";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/model");
|
||||
dependencies = {
|
||||
@@ -683,9 +690,9 @@ in
|
||||
async_trait = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.51" { profileName = "__noProfile"; };
|
||||
futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.17" { inherit profileName; };
|
||||
futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.17" { inherit profileName; };
|
||||
garage_rpc = rustPackages."unknown".garage_rpc."0.5.0" { inherit profileName; };
|
||||
garage_table = rustPackages."unknown".garage_table."0.5.0" { inherit profileName; };
|
||||
garage_util = rustPackages."unknown".garage_util."0.5.0" { inherit profileName; };
|
||||
garage_rpc = rustPackages."unknown".garage_rpc."0.5.1" { inherit profileName; };
|
||||
garage_table = rustPackages."unknown".garage_table."0.5.1" { inherit profileName; };
|
||||
garage_util = rustPackages."unknown".garage_util."0.5.1" { inherit profileName; };
|
||||
hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
|
||||
log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; };
|
||||
netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.0" { inherit profileName; };
|
||||
@@ -695,12 +702,13 @@ in
|
||||
serde_bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; };
|
||||
sled = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sled."0.34.7" { inherit profileName; };
|
||||
tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.12.0" { inherit profileName; };
|
||||
zstd = rustPackages."registry+https://github.com/rust-lang/crates.io-index".zstd."0.9.0+zstd.1.5.0" { inherit profileName; };
|
||||
};
|
||||
});
|
||||
|
||||
"unknown".garage_rpc."0.5.0" = overridableMkRustCrate (profileName: rec {
|
||||
"unknown".garage_rpc."0.5.1" = overridableMkRustCrate (profileName: rec {
|
||||
name = "garage_rpc";
|
||||
version = "0.5.0";
|
||||
version = "0.5.1";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/rpc");
|
||||
dependencies = {
|
||||
@@ -709,7 +717,7 @@ in
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.17" { inherit profileName; };
|
||||
futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.17" { inherit profileName; };
|
||||
garage_util = rustPackages."unknown".garage_util."0.5.0" { inherit profileName; };
|
||||
garage_util = rustPackages."unknown".garage_util."0.5.1" { inherit profileName; };
|
||||
gethostname = rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.2.1" { inherit profileName; };
|
||||
hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
|
||||
hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.13" { inherit profileName; };
|
||||
@@ -726,9 +734,9 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"unknown".garage_table."0.5.0" = overridableMkRustCrate (profileName: rec {
|
||||
"unknown".garage_table."0.5.1" = overridableMkRustCrate (profileName: rec {
|
||||
name = "garage_table";
|
||||
version = "0.5.0";
|
||||
version = "0.5.1";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/table");
|
||||
dependencies = {
|
||||
@@ -736,8 +744,8 @@ in
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.17" { inherit profileName; };
|
||||
futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.17" { inherit profileName; };
|
||||
garage_rpc = rustPackages."unknown".garage_rpc."0.5.0" { inherit profileName; };
|
||||
garage_util = rustPackages."unknown".garage_util."0.5.0" { inherit profileName; };
|
||||
garage_rpc = rustPackages."unknown".garage_rpc."0.5.1" { inherit profileName; };
|
||||
garage_util = rustPackages."unknown".garage_util."0.5.1" { inherit profileName; };
|
||||
hexdump = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; };
|
||||
log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; };
|
||||
rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.4" { inherit profileName; };
|
||||
@@ -749,9 +757,9 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"unknown".garage_util."0.5.0" = overridableMkRustCrate (profileName: rec {
|
||||
"unknown".garage_util."0.5.1" = overridableMkRustCrate (profileName: rec {
|
||||
name = "garage_util";
|
||||
version = "0.5.0";
|
||||
version = "0.5.1";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/util");
|
||||
dependencies = {
|
||||
@@ -776,18 +784,18 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"unknown".garage_web."0.5.0" = overridableMkRustCrate (profileName: rec {
|
||||
"unknown".garage_web."0.5.1" = overridableMkRustCrate (profileName: rec {
|
||||
name = "garage_web";
|
||||
version = "0.5.0";
|
||||
version = "0.5.1";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/web");
|
||||
dependencies = {
|
||||
err_derive = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.0" { profileName = "__noProfile"; };
|
||||
futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.17" { inherit profileName; };
|
||||
garage_api = rustPackages."unknown".garage_api."0.5.0" { inherit profileName; };
|
||||
garage_model = rustPackages."unknown".garage_model."0.5.0" { inherit profileName; };
|
||||
garage_table = rustPackages."unknown".garage_table."0.5.0" { inherit profileName; };
|
||||
garage_util = rustPackages."unknown".garage_util."0.5.0" { inherit profileName; };
|
||||
garage_api = rustPackages."unknown".garage_api."0.5.1" { inherit profileName; };
|
||||
garage_model = rustPackages."unknown".garage_model."0.5.1" { inherit profileName; };
|
||||
garage_table = rustPackages."unknown".garage_table."0.5.1" { inherit profileName; };
|
||||
garage_util = rustPackages."unknown".garage_util."0.5.1" { inherit profileName; };
|
||||
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.5" { inherit profileName; };
|
||||
hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.13" { inherit profileName; };
|
||||
log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; };
|
||||
@@ -1055,6 +1063,16 @@ in
|
||||
];
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".jobserver."0.1.24" = overridableMkRustCrate (profileName: rec {
|
||||
name = "jobserver";
|
||||
version = "0.1.24";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa"; };
|
||||
dependencies = {
|
||||
${ if hostPlatform.isUnix then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.103" { inherit profileName; };
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".kuska-handshake."0.2.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "kuska-handshake";
|
||||
version = "0.2.0";
|
||||
@@ -2351,4 +2369,44 @@ in
|
||||
];
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".zstd."0.9.0+zstd.1.5.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "zstd";
|
||||
version = "0.9.0+zstd.1.5.0";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "07749a5dc2cb6b36661290245e350f15ec3bbb304e493db54a1d354480522ccd"; };
|
||||
dependencies = {
|
||||
zstd_safe = rustPackages."registry+https://github.com/rust-lang/crates.io-index".zstd-safe."4.1.1+zstd.1.5.0" { inherit profileName; };
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".zstd-safe."4.1.1+zstd.1.5.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "zstd-safe";
|
||||
version = "4.1.1+zstd.1.5.0";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "c91c90f2c593b003603e5e0493c837088df4469da25aafff8bce42ba48caf079"; };
|
||||
features = builtins.concatLists [
|
||||
[ "std" ]
|
||||
];
|
||||
dependencies = {
|
||||
libc = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.103" { inherit profileName; };
|
||||
zstd_sys = rustPackages."registry+https://github.com/rust-lang/crates.io-index".zstd-sys."1.6.1+zstd.1.5.0" { inherit profileName; };
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".zstd-sys."1.6.1+zstd.1.5.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "zstd-sys";
|
||||
version = "1.6.1+zstd.1.5.0";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "615120c7a2431d16cf1cf979e7fc31ba7a5b5e5707b29c8a99e5dbf8a8392a33"; };
|
||||
features = builtins.concatLists [
|
||||
[ "std" ]
|
||||
];
|
||||
dependencies = {
|
||||
libc = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.103" { inherit profileName; };
|
||||
};
|
||||
buildDependencies = {
|
||||
cc = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.0.71" { profileName = "__noProfile"; };
|
||||
};
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
@@ -31,6 +31,7 @@
|
||||
|
||||
- [Design](./design/index.md)
|
||||
- [Goals and use Cases](./design/goals.md)
|
||||
- [Benchmarks](./design/benchmarks.md)
|
||||
- [Related work](./design/related_work.md)
|
||||
- [Internals](./design/internals.md)
|
||||
|
||||
|
||||
@@ -109,7 +109,7 @@ Do not change the `use_path_style` and `legacy_auth` entries, other configuratio
|
||||
Peertube proposes a clever integration of S3 by directly exposing its endpoint instead of proxifying requests through the application.
|
||||
In other words, Peertube is only responsible of the "control plane" and offload the "data plane" to Garage.
|
||||
In return, this system is a bit harder to configure, especially with Garage that supports less feature than other older S3 backends.
|
||||
We show that it is still possible to configure Garage with Peertube, allowing you to spread the load and the bandiwdth usage on the Garage cluster.
|
||||
We show that it is still possible to configure Garage with Peertube, allowing you to spread the load and the bandwidth usage on the Garage cluster.
|
||||
|
||||
### Enable path-style access by patching Peertube
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# Connect it to...
|
||||
# Integrations
|
||||
|
||||
Garage implements the Amazon S3 protocol, which makes it compatible with many existing software programs.
|
||||
|
||||
|
||||
@@ -74,6 +74,8 @@ data_dir = "/var/lib/garage/data"
|
||||
|
||||
replication_mode = "3"
|
||||
|
||||
compression_level = 2
|
||||
|
||||
rpc_bind_addr = "[::]:3901"
|
||||
rpc_public_addr = "<this node's public IP>:3901"
|
||||
rpc_secret = "<RPC secret>"
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
# Benchmarks
|
||||
|
||||
With Garage, we wanted to build a software defined storage service that follow the [KISS principle](https://en.wikipedia.org/wiki/KISS_principle),
|
||||
that is suitable for geo-distributed deployments and more generally that would work well for community hosting (like a Mastodon instance).
|
||||
|
||||
In our benchmarks, we aim to quantify how Garage performs on these goals compared to the other available solutions.
|
||||
|
||||
## Geo-distribution
|
||||
|
||||
The main challenge in a geo-distributed setup is latency between nodes of the cluster.
|
||||
The more a user request will require intra-cluster requests to complete, the more its latency will increase.
|
||||
This is especially true for sequential requests: requests that must wait the result of another request to be sent.
|
||||
We designed Garage without consensus algorithms (eg. Paxos or Raft) to minimize the number of sequential and parallel requests.
|
||||
|
||||
This serie of benchmarks quantifies the impact of this design choice.
|
||||
|
||||
### On a simple simulated network
|
||||
|
||||
We start with a controlled environment, all the instances are running on the same (powerful enough) machine.
|
||||
|
||||
To control the network latency, we simulate the network with [mknet](https://git.deuxfleurs.fr/trinity-1686a/mknet) (a tool we developped, based on `tc` and the linux network stack).
|
||||
To mesure S3 endpoints latency, we use our own tool [s3lat](https://git.deuxfleurs.fr/quentin/s3lat/) to observe only the intra-cluster latency and not some contention on the nodes (CPU, RAM, disk I/O, network bandwidth, etc.).
|
||||
Compared to other benchmark tools, S3Lat sends only one (small) request at the same time and measures its latency.
|
||||
We selected 5 standard endpoints that are often in the critical path: ListBuckets, ListObjects, GetObject, PutObject and RemoveObject.
|
||||
|
||||
In this first benchmark, we consider 5 instances that are located in a different place each. To simulate the distance, we configure mknet with a RTT between each node of 100 ms +/- 20 ms of jitter. We get the following graph, where the colored bars represent the mean latency while the error bars the minimum and maximum one:
|
||||
|
||||
|
||||
|
||||
Compared to garage, minio latency drastically increases on 3 endpoints: GetObject, PutObject, RemoveObject.
|
||||
|
||||
We suppose that these requests on minio make transactions over Raft, involving 4 sequential requests: 1) sending the message to the leader, 2) having the leader dispatch it to the other nodes, 3) waiting for the confirmation of followers and finally 4) commiting it. With our current configuration, one Raft transaction will take around 400 ms. GetObject seems to correlate to 1 transaction while PutObject and RemoveObject seems to correlate to 2 or 3. Reviewing minio code would be required to confirm this hypothesis.
|
||||
|
||||
Conversely, garage uses an architecture similar to DynamoDB and never require global cluster coordination to answer a request.
|
||||
Instead, garage can always contact the right node in charge of the requested data, and can answer in as low as one request in the case of GetObject and PutObject. We also observed that Garage latency, while often lower to minio, is more dispersed: garage is still in beta and has not received any performance optimization yet.
|
||||
|
||||
As a conclusion, Garage performs well in such setup while minio will be hard to use, especially for interactive use cases.
|
||||
|
||||
### On a complex simulated network
|
||||
|
||||
This time we consider a more heterogeneous network with 6 servers spread in 3 datacenter, giving us 2 servers per datacenters.
|
||||
We consider that intra-DC communications are now very cheap with a latency of 0.5ms and without any jitter.
|
||||
The inter-DC remains costly with the same value as before (100ms +/- 20ms of jitter).
|
||||
We plot a similar graph as before:
|
||||
|
||||
|
||||
|
||||
This new graph is very similar to the one before, neither minio or garage seems to benefit from this new topology, but they also do not suffer from it.
|
||||
|
||||
Considering garage, this is expected: nodes in the same DC are put in the same zone, and then data are spread on different zones for data resiliency and availaibility.
|
||||
Then, in the default mode, requesting data requires to query at least 2 zones to be sure that we have the most up to date information.
|
||||
These requests will involve at least one inter-DC communication.
|
||||
In other words, we prioritize data availability and synchronization over raw performances.
|
||||
|
||||
Minio's case is a bit different as by default a minio cluster is not location aware, so we can't explain its performances through location awareness.
|
||||
*We know that minio has a multi site mode but it is definitely not a first class citizen: data are asynchronously replicated from one minio cluster to another.*
|
||||
We suppose that, due to the consensus, for many of its requests minio will wait for a response of the majority of the server, also involving inter-DC communications.
|
||||
|
||||
As a conclusion, our new topology did not influence garage or minio performances, confirming that in presence of latency, garage is the best fit.
|
||||
|
||||
### On a real world deployment
|
||||
|
||||
*TODO*
|
||||
|
||||
|
||||
## Performance stability
|
||||
|
||||
A storage cluster will encounter different scenario over its life, many of them will not be predictable.
|
||||
In this context, we argue that, more than peak performances, we should seek predictable and stable performances to ensure data availability.
|
||||
|
||||
### Reference
|
||||
|
||||
*TODO*
|
||||
|
||||
### On a degraded cluster
|
||||
|
||||
*TODO*
|
||||
|
||||
### At scale
|
||||
|
||||
*TODO*
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 129 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 124 KiB |
@@ -22,7 +22,7 @@ Garage is an **open-source** distributed **storage service** you can **self-host
|
||||
</p>
|
||||
|
||||
<p align="center" style="text-align:center; margin-bottom: 5rem;">
|
||||
<a href="/design/use_cases.html">⮞ learn more about use cases ⮜</a>
|
||||
<a href="/design/goals.html#use-cases">⮞ learn more about use cases ⮜</a>
|
||||
</p>
|
||||
|
||||
Garage implements the **[Amazon S3 API](https://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html)** and thus is already **compatible** with many applications:
|
||||
|
||||
@@ -10,6 +10,8 @@ block_size = 1048576
|
||||
|
||||
replication_mode = "3"
|
||||
|
||||
compression_level = 1
|
||||
|
||||
rpc_secret = "4425f5c26c5e11581d3223904324dcb5b5d5dfb14e5e7f35e38c595424f5f1e6"
|
||||
rpc_bind_addr = "[::]:3901"
|
||||
rpc_public_addr = "[fc00:1::1]:3901"
|
||||
@@ -98,6 +100,30 @@ Never run a Garage cluster where that is not the case.**
|
||||
Changing the `replication_mode` of a cluster might work (make sure to shut down all nodes
|
||||
and changing it everywhere at the time), but is not officially supported.
|
||||
|
||||
### `compression_level`
|
||||
|
||||
Zstd compression level to use for storing blocks.
|
||||
|
||||
Values between `1` (faster compression) and `19` (smaller file) are standard compression
|
||||
levels for zstd. From `20` to `22`, compression levels are referred as "ultra" and must be
|
||||
used with extra care as it will use lot of memory. A value of `0` will let zstd choose a
|
||||
default value (currently `3`). Finally, zstd has also compression designed to be faster
|
||||
than default compression levels, they range from `-1` (smaller file) to `-99` (faster
|
||||
compression).
|
||||
|
||||
If you do not specify a `compression_level` entry, garage will set it to `1` for you. With
|
||||
this parameters, zstd consumes low amount of cpu and should work faster than line speed in
|
||||
most situations, while saving some space and intra-cluster
|
||||
bandwidth.
|
||||
|
||||
If you want to totally deactivate zstd in garage, you can pass the special value `'none'`. No
|
||||
zstd related code will be called, your chunks will be stored on disk without any processing.
|
||||
|
||||
Compression is done synchronously, setting a value too high will add latency to write queries.
|
||||
|
||||
This value can be different between nodes, compression is done by the node which receive the
|
||||
API call.
|
||||
|
||||
#### `rpc_secret`
|
||||
|
||||
Garage uses a secret key that is shared between all nodes of the cluster
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
Implemented:
|
||||
|
||||
- path-style URLs (`garage.tld/bucket/key`)
|
||||
- vhost-style URLs (`bucket.garage.tld/key`)
|
||||
- putting and getting objects in buckets
|
||||
- multipart uploads
|
||||
- listing objects
|
||||
@@ -12,9 +13,8 @@ Implemented:
|
||||
|
||||
Not implemented:
|
||||
|
||||
- vhost-style URLs (`bucket.garage.tld/key`)
|
||||
- object-level ACL
|
||||
- object versioning
|
||||
- [object versioning](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/166)
|
||||
- encryption
|
||||
- most `x-amz-` headers
|
||||
|
||||
@@ -31,10 +31,12 @@ All APIs that are not mentionned are not implemented and will return a 400 bad r
|
||||
| CreateBucket | Unsupported, stub (see below) |
|
||||
| CreateMultipartUpload | Implemented |
|
||||
| DeleteBucket | Unsupported (see below) |
|
||||
| DeleteBucketWebsite | Implemented |
|
||||
| DeleteObject | Implemented |
|
||||
| DeleteObjects | Implemented |
|
||||
| GetBucketLocation | Implemented |
|
||||
| GetBucketVersioning | Stub (see below) |
|
||||
| GetBucketWebsite | Unsupported |
|
||||
| GetObject | Implemented |
|
||||
| HeadBucket | Implemented |
|
||||
| HeadObject | Implemented |
|
||||
@@ -42,6 +44,7 @@ All APIs that are not mentionned are not implemented and will return a 400 bad r
|
||||
| ListObjects | Implemented, bugs? (see below) |
|
||||
| ListObjectsV2 | Implemented |
|
||||
| PutObject | Implemented |
|
||||
| PutBucketWebsite | Partially implemented (see below)|
|
||||
| UploadPart | Implemented |
|
||||
|
||||
|
||||
@@ -55,3 +58,6 @@ All APIs that are not mentionned are not implemented and will return a 400 bad r
|
||||
|
||||
- **ListObjects:** Implemented, but there isn't a very good specification of what `encoding-type=url` covers so there might be some encoding bugs. In our implementation the url-encoded fields are in the same in ListObjects as they are in ListObjectsV2.
|
||||
|
||||
- **PutBucketWebsite:** Implemented, but only store if website is enabled, not more complexe informations.
|
||||
|
||||
- **GetBucketWebsite:** Not implemented yet, will be when PubBucketWebsite store more informations.
|
||||
|
||||
@@ -9,8 +9,8 @@ your motivations for doing so in the PR message.
|
||||
| **S-tier** (high priority) | |
|
||||
| | HeadBucket |
|
||||
| | GetBucketLocation |
|
||||
| | *CreateBucket* |
|
||||
| | *DeleteBucket* |
|
||||
| | [*CreateBucket*](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/97) |
|
||||
| | [*DeleteBucket*](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/97) |
|
||||
| | ListBuckets |
|
||||
| | ListObjects |
|
||||
| | ListObjectsV2 |
|
||||
@@ -24,16 +24,16 @@ your motivations for doing so in the PR message.
|
||||
| | CompleteMultipartUpload |
|
||||
| | AbortMultipartUpload |
|
||||
| | UploadPart |
|
||||
| | *ListMultipartUploads* |
|
||||
| | *ListParts* |
|
||||
| | [*ListMultipartUploads*](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/103) |
|
||||
| | [*ListParts*](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/103) |
|
||||
| **A-tier** (will implement) | |
|
||||
| | *GetBucketCors* |
|
||||
| | *PutBucketCors* |
|
||||
| | *DeleteBucketCors* |
|
||||
| | *UploadPartCopy* |
|
||||
| | *GetBucketWebsite* |
|
||||
| | *PutBucketWebsite* |
|
||||
| | *DeleteBucketWebsite* |
|
||||
| | [*GetBucketCors*](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/138) |
|
||||
| | [*PutBucketCors*](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/138) |
|
||||
| | [*DeleteBucketCors*](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/138) |
|
||||
| | [*UploadPartCopy*](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/160) |
|
||||
| | [*GetBucketWebsite*](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/77) |
|
||||
| | [*PutBucketWebsite*](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/77) |
|
||||
| | DeleteBucketWebsite |
|
||||
| ~~~~~~~~~~~~~~~~~~~~~~~~~~ | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
|
||||
| **B-tier** | |
|
||||
| | GetBucketAcl |
|
||||
@@ -50,12 +50,14 @@ your motivations for doing so in the PR message.
|
||||
| | ListObjectVersions |
|
||||
| | GetObjectAcl |
|
||||
| | PutObjectAcl |
|
||||
| | GetBucketLifecycleConfiguration |
|
||||
| | PutBucketLifecycleConfiguration |
|
||||
| | DeleteBucketLifecycle |
|
||||
| **garbage-tier** | |
|
||||
| | DeleteBucketEncryption |
|
||||
| | DeleteBucketAnalyticsConfiguration |
|
||||
| | DeleteBucketIntelligentTieringConfiguration |
|
||||
| | DeleteBucketInventoryConfiguration |
|
||||
| | DeleteBucketLifecycle |
|
||||
| | DeleteBucketMetricsConfiguration |
|
||||
| | DeleteBucketOwnershipControls |
|
||||
| | DeleteBucketPolicy |
|
||||
@@ -68,7 +70,6 @@ your motivations for doing so in the PR message.
|
||||
| | GetBucketEncryption |
|
||||
| | GetBucketIntelligentTieringConfiguration |
|
||||
| | GetBucketInventoryConfiguration |
|
||||
| | GetBucketLifecycleConfiguration |
|
||||
| | GetBucketLogging |
|
||||
| | GetBucketMetricsConfiguration |
|
||||
| | GetBucketNotificationConfiguration |
|
||||
@@ -90,7 +91,6 @@ your motivations for doing so in the PR message.
|
||||
| | PutBucketEncryption |
|
||||
| | PutBucketIntelligentTieringConfiguration |
|
||||
| | PutBucketInventoryConfiguration |
|
||||
| | PutBucketLifecycleConfiguration |
|
||||
| | PutBucketLogging |
|
||||
| | PutBucketMetricsConfiguration |
|
||||
| | PutBucketNotificationConfiguration |
|
||||
|
||||
+22
-17
@@ -30,6 +30,11 @@ dd if=/dev/urandom of=/tmp/garage.1.rnd bs=1k count=2 # No multipart, inline sto
|
||||
dd if=/dev/urandom of=/tmp/garage.2.rnd bs=1M count=5 # No multipart but file will be chunked
|
||||
dd if=/dev/urandom of=/tmp/garage.3.rnd bs=1M count=10 # by default, AWS starts using multipart at 8MB
|
||||
|
||||
# data of lower entropy, to test compression
|
||||
dd if=/dev/urandom bs=1k count=2 | base64 -w0 > /tmp/garage.1.b64
|
||||
dd if=/dev/urandom bs=1M count=5 | base64 -w0 > /tmp/garage.2.b64
|
||||
dd if=/dev/urandom bs=1M count=10 | base64 -w0 > /tmp/garage.3.b64
|
||||
|
||||
echo "🧪 S3 API testing..."
|
||||
|
||||
# AWS
|
||||
@@ -37,11 +42,11 @@ if [ -z "$SKIP_AWS" ]; then
|
||||
echo "🛠️ Testing with awscli"
|
||||
source ${SCRIPT_FOLDER}/dev-env-aws.sh
|
||||
aws s3 ls
|
||||
for idx in $(seq 1 3); do
|
||||
aws s3 cp "/tmp/garage.$idx.rnd" "s3://eprouvette/&+-é\"/garage.$idx.aws"
|
||||
for idx in {1..3}.{rnd,b64}; do
|
||||
aws s3 cp "/tmp/garage.$idx" "s3://eprouvette/&+-é\"/garage.$idx.aws"
|
||||
aws s3 ls s3://eprouvette
|
||||
aws s3 cp "s3://eprouvette/&+-é\"/garage.$idx.aws" "/tmp/garage.$idx.dl"
|
||||
diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
|
||||
diff /tmp/garage.$idx /tmp/garage.$idx.dl
|
||||
rm /tmp/garage.$idx.dl
|
||||
aws s3 rm "s3://eprouvette/&+-é\"/garage.$idx.aws"
|
||||
done
|
||||
@@ -52,11 +57,11 @@ if [ -z "$SKIP_S3CMD" ]; then
|
||||
echo "🛠️ Testing with s3cmd"
|
||||
source ${SCRIPT_FOLDER}/dev-env-s3cmd.sh
|
||||
s3cmd ls
|
||||
for idx in $(seq 1 3); do
|
||||
s3cmd put "/tmp/garage.$idx.rnd" "s3://eprouvette/&+-é\"/garage.$idx.s3cmd"
|
||||
for idx in {1..3}.{rnd,b64}; do
|
||||
s3cmd put "/tmp/garage.$idx" "s3://eprouvette/&+-é\"/garage.$idx.s3cmd"
|
||||
s3cmd ls s3://eprouvette
|
||||
s3cmd get "s3://eprouvette/&+-é\"/garage.$idx.s3cmd" "/tmp/garage.$idx.dl"
|
||||
diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
|
||||
diff /tmp/garage.$idx /tmp/garage.$idx.dl
|
||||
rm /tmp/garage.$idx.dl
|
||||
s3cmd rm "s3://eprouvette/&+-é\"/garage.$idx.s3cmd"
|
||||
done
|
||||
@@ -67,11 +72,11 @@ if [ -z "$SKIP_MC" ]; then
|
||||
echo "🛠️ Testing with mc (minio client)"
|
||||
source ${SCRIPT_FOLDER}/dev-env-mc.sh
|
||||
mc ls garage/
|
||||
for idx in $(seq 1 3); do
|
||||
mc cp "/tmp/garage.$idx.rnd" "garage/eprouvette/&+-é\"/garage.$idx.mc"
|
||||
for idx in {1..3}.{rnd,b64}; do
|
||||
mc cp "/tmp/garage.$idx" "garage/eprouvette/&+-é\"/garage.$idx.mc"
|
||||
mc ls garage/eprouvette
|
||||
mc cp "garage/eprouvette/&+-é\"/garage.$idx.mc" "/tmp/garage.$idx.dl"
|
||||
diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
|
||||
diff /tmp/garage.$idx /tmp/garage.$idx.dl
|
||||
rm /tmp/garage.$idx.dl
|
||||
mc rm "garage/eprouvette/&+-é\"/garage.$idx.mc"
|
||||
done
|
||||
@@ -82,13 +87,13 @@ if [ -z "$SKIP_RCLONE" ]; then
|
||||
echo "🛠️ Testing with rclone"
|
||||
source ${SCRIPT_FOLDER}/dev-env-rclone.sh
|
||||
rclone lsd garage:
|
||||
for idx in $(seq 1 3); do
|
||||
cp /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
|
||||
for idx in {1..3}.{rnd,b64}; do
|
||||
cp /tmp/garage.$idx /tmp/garage.$idx.dl
|
||||
rclone copy "/tmp/garage.$idx.dl" "garage:eprouvette/&+-é\"/"
|
||||
rm /tmp/garage.$idx.dl
|
||||
rclone ls garage:eprouvette
|
||||
rclone copy "garage:eprouvette/&+-é\"/garage.$idx.dl" "/tmp/"
|
||||
diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
|
||||
diff /tmp/garage.$idx /tmp/garage.$idx.dl
|
||||
rm /tmp/garage.$idx.dl
|
||||
rclone delete "garage:eprouvette/&+-é\"/garage.$idx.dl"
|
||||
done
|
||||
@@ -100,17 +105,17 @@ if [ -z "$SKIP_DUCK" ]; then
|
||||
source ${SCRIPT_FOLDER}/dev-env-duck.sh
|
||||
duck --list garage:/
|
||||
duck --mkdir "garage:/eprouvette/duck"
|
||||
for idx in $(seq 1 3); do
|
||||
duck --verbose --upload "garage:/eprouvette/duck/" "/tmp/garage.$idx.rnd"
|
||||
for idx in {1..3}.{rnd,b64}; do
|
||||
duck --verbose --upload "garage:/eprouvette/duck/" "/tmp/garage.$idx"
|
||||
duck --list garage:/eprouvette/duck/
|
||||
duck --download "garage:/eprouvette/duck/garage.$idx.rnd" "/tmp/garage.$idx.dl"
|
||||
diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
|
||||
duck --download "garage:/eprouvette/duck/garage.$idx" "/tmp/garage.$idx.dl"
|
||||
diff /tmp/garage.$idx /tmp/garage.$idx.dl
|
||||
rm /tmp/garage.$idx.dl
|
||||
duck --delete "garage:/eprouvette/duck/garage.$idx.dk"
|
||||
done
|
||||
fi
|
||||
|
||||
rm /tmp/garage.{1,2,3}.rnd
|
||||
rm /tmp/garage.{1..3}.{rnd,b64}
|
||||
|
||||
if [ -z "$SKIP_AWS" ]; then
|
||||
echo "🧪 Website Testing"
|
||||
|
||||
+4
-4
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_api"
|
||||
version = "0.5.0"
|
||||
version = "0.5.1"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
@@ -14,9 +14,9 @@ path = "lib.rs"
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
garage_model = { version = "0.5.0", path = "../model" }
|
||||
garage_table = { version = "0.5.0", path = "../table" }
|
||||
garage_util = { version = "0.5.0", path = "../util" }
|
||||
garage_model = { version = "0.5.1", path = "../model" }
|
||||
garage_table = { version = "0.5.1", path = "../table" }
|
||||
garage_util = { version = "0.5.1", path = "../util" }
|
||||
|
||||
base64 = "0.13"
|
||||
bytes = "1.0"
|
||||
|
||||
@@ -22,6 +22,7 @@ use crate::s3_get::*;
|
||||
use crate::s3_list::*;
|
||||
use crate::s3_put::*;
|
||||
use crate::s3_router::{Authorization, Endpoint};
|
||||
use crate::s3_website::*;
|
||||
|
||||
/// Run the S3 API server
|
||||
pub async fn run_api_server(
|
||||
@@ -254,6 +255,10 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
|
||||
Endpoint::DeleteObjects { bucket } => {
|
||||
handle_delete_objects(garage, &bucket, req, content_sha256).await
|
||||
}
|
||||
Endpoint::PutBucketWebsite { bucket } => {
|
||||
handle_put_website(garage, bucket, req, content_sha256).await
|
||||
}
|
||||
Endpoint::DeleteBucketWebsite { bucket } => handle_delete_website(garage, bucket).await,
|
||||
endpoint => Err(Error::NotImplemented(endpoint.name().to_owned())),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,4 +20,5 @@ pub mod s3_get;
|
||||
mod s3_list;
|
||||
mod s3_put;
|
||||
mod s3_router;
|
||||
mod s3_website;
|
||||
mod s3_xml;
|
||||
|
||||
@@ -511,11 +511,7 @@ impl Endpoint {
|
||||
let path = uri.path().trim_start_matches('/');
|
||||
let query = uri.query();
|
||||
if bucket.is_none() && path.is_empty() {
|
||||
if query.is_none() {
|
||||
return Ok(Self::ListBuckets);
|
||||
} else {
|
||||
return Err(Error::BadRequest("Invalid ListBuckets query".to_owned()));
|
||||
}
|
||||
return Ok(Self::ListBuckets);
|
||||
}
|
||||
|
||||
let (bucket, key) = if let Some(bucket) = bucket {
|
||||
|
||||
@@ -0,0 +1,308 @@
|
||||
use quick_xml::de::from_reader;
|
||||
use std::sync::Arc;
|
||||
|
||||
use hyper::{Body, Request, Response, StatusCode};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::error::*;
|
||||
use crate::s3_xml::{xmlns_tag, IntValue, Value};
|
||||
use crate::signature::verify_signed_content;
|
||||
use garage_model::bucket_table::BucketState;
|
||||
use garage_model::garage::Garage;
|
||||
use garage_table::*;
|
||||
use garage_util::data::Hash;
|
||||
|
||||
pub async fn handle_delete_website(
|
||||
garage: Arc<Garage>,
|
||||
bucket: String,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let mut bucket = garage
|
||||
.bucket_table
|
||||
.get(&EmptyKey, &bucket)
|
||||
.await?
|
||||
.ok_or(Error::NotFound)?;
|
||||
|
||||
if let BucketState::Present(state) = bucket.state.get_mut() {
|
||||
state.website.update(false);
|
||||
garage.bucket_table.insert(&bucket).await?;
|
||||
}
|
||||
|
||||
Ok(Response::builder()
|
||||
.status(StatusCode::NO_CONTENT)
|
||||
.body(Body::from(vec![]))
|
||||
.unwrap())
|
||||
}
|
||||
|
||||
pub async fn handle_put_website(
|
||||
garage: Arc<Garage>,
|
||||
bucket: String,
|
||||
req: Request<Body>,
|
||||
content_sha256: Option<Hash>,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let body = hyper::body::to_bytes(req.into_body()).await?;
|
||||
verify_signed_content(content_sha256, &body[..])?;
|
||||
|
||||
let mut bucket = garage
|
||||
.bucket_table
|
||||
.get(&EmptyKey, &bucket)
|
||||
.await?
|
||||
.ok_or(Error::NotFound)?;
|
||||
|
||||
let conf: WebsiteConfiguration = from_reader(&body as &[u8])?;
|
||||
conf.validate()?;
|
||||
|
||||
if let BucketState::Present(state) = bucket.state.get_mut() {
|
||||
state.website.update(true);
|
||||
garage.bucket_table.insert(&bucket).await?;
|
||||
}
|
||||
|
||||
Ok(Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.body(Body::from(vec![]))
|
||||
.unwrap())
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct WebsiteConfiguration {
|
||||
#[serde(serialize_with = "xmlns_tag", skip_deserializing)]
|
||||
pub xmlns: (),
|
||||
#[serde(rename = "ErrorDocument")]
|
||||
pub error_document: Option<Key>,
|
||||
#[serde(rename = "IndexDocument")]
|
||||
pub index_document: Option<Suffix>,
|
||||
#[serde(rename = "RedirectAllRequestsTo")]
|
||||
pub redirect_all_requests_to: Option<Target>,
|
||||
#[serde(rename = "RoutingRules")]
|
||||
pub routing_rules: Option<Vec<RoutingRule>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct RoutingRule {
|
||||
#[serde(rename = "RoutingRule")]
|
||||
pub inner: RoutingRuleInner,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct RoutingRuleInner {
|
||||
#[serde(rename = "Condition")]
|
||||
pub condition: Option<Condition>,
|
||||
#[serde(rename = "Redirect")]
|
||||
pub redirect: Redirect,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct Key {
|
||||
#[serde(rename = "Key")]
|
||||
pub key: Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct Suffix {
|
||||
#[serde(rename = "Suffix")]
|
||||
pub suffix: Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct Target {
|
||||
#[serde(rename = "HostName")]
|
||||
pub hostname: Value,
|
||||
#[serde(rename = "Protocol")]
|
||||
pub protocol: Option<Value>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct Condition {
|
||||
#[serde(rename = "HttpErrorCodeReturnedEquals")]
|
||||
pub http_error_code: Option<IntValue>,
|
||||
#[serde(rename = "KeyPrefixEquals")]
|
||||
pub prefix: Option<Value>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct Redirect {
|
||||
#[serde(rename = "HostName")]
|
||||
pub hostname: Option<Value>,
|
||||
#[serde(rename = "Protocol")]
|
||||
pub protocol: Option<Value>,
|
||||
#[serde(rename = "HttpRedirectCode")]
|
||||
pub http_redirect_code: Option<IntValue>,
|
||||
#[serde(rename = "ReplaceKeyPrefixWith")]
|
||||
pub replace_prefix: Option<Value>,
|
||||
#[serde(rename = "ReplaceKeyWith")]
|
||||
pub replace_full: Option<Value>,
|
||||
}
|
||||
|
||||
impl WebsiteConfiguration {
|
||||
pub fn validate(&self) -> Result<(), Error> {
|
||||
if self.redirect_all_requests_to.is_some()
|
||||
&& (self.error_document.is_some()
|
||||
|| self.index_document.is_some()
|
||||
|| self.routing_rules.is_some())
|
||||
{
|
||||
return Err(Error::BadRequest(
|
||||
"Bad XML: can't have RedirectAllRequestsTo and other fields".to_owned(),
|
||||
));
|
||||
}
|
||||
if let Some(ref ed) = self.error_document {
|
||||
ed.validate()?;
|
||||
}
|
||||
if let Some(ref id) = self.index_document {
|
||||
id.validate()?;
|
||||
}
|
||||
if let Some(ref rart) = self.redirect_all_requests_to {
|
||||
rart.validate()?;
|
||||
}
|
||||
if let Some(ref rrs) = self.routing_rules {
|
||||
for rr in rrs {
|
||||
rr.inner.validate()?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Key {
|
||||
pub fn validate(&self) -> Result<(), Error> {
|
||||
if self.key.0.is_empty() {
|
||||
Err(Error::BadRequest(
|
||||
"Bad XML: error document specified but empty".to_owned(),
|
||||
))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Suffix {
|
||||
pub fn validate(&self) -> Result<(), Error> {
|
||||
if self.suffix.0.is_empty() | self.suffix.0.contains('/') {
|
||||
Err(Error::BadRequest(
|
||||
"Bad XML: index document is empty or contains /".to_owned(),
|
||||
))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Target {
|
||||
pub fn validate(&self) -> Result<(), Error> {
|
||||
if let Some(ref protocol) = self.protocol {
|
||||
if protocol.0 != "http" && protocol.0 != "https" {
|
||||
return Err(Error::BadRequest("Bad XML: invalid protocol".to_owned()));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl RoutingRuleInner {
|
||||
pub fn validate(&self) -> Result<(), Error> {
|
||||
let has_prefix = self
|
||||
.condition
|
||||
.as_ref()
|
||||
.map(|c| c.prefix.as_ref())
|
||||
.flatten()
|
||||
.is_some();
|
||||
self.redirect.validate(has_prefix)
|
||||
}
|
||||
}
|
||||
|
||||
impl Redirect {
|
||||
pub fn validate(&self, has_prefix: bool) -> Result<(), Error> {
|
||||
if self.replace_prefix.is_some() {
|
||||
if self.replace_full.is_some() {
|
||||
return Err(Error::BadRequest(
|
||||
"Bad XML: both ReplaceKeyPrefixWith and ReplaceKeyWith are set".to_owned(),
|
||||
));
|
||||
}
|
||||
if !has_prefix {
|
||||
return Err(Error::BadRequest(
|
||||
"Bad XML: ReplaceKeyPrefixWith is set, but KeyPrefixEquals isn't".to_owned(),
|
||||
));
|
||||
}
|
||||
}
|
||||
if let Some(ref protocol) = self.protocol {
|
||||
if protocol.0 != "http" && protocol.0 != "https" {
|
||||
return Err(Error::BadRequest("Bad XML: invalid protocol".to_owned()));
|
||||
}
|
||||
}
|
||||
// TODO there are probably more invalide cases, but which ones?
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
use quick_xml::de::from_str;
|
||||
|
||||
#[test]
|
||||
fn test_deserialize() {
|
||||
let message = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<WebsiteConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<ErrorDocument>
|
||||
<Key>my-error-doc</Key>
|
||||
</ErrorDocument>
|
||||
<IndexDocument>
|
||||
<Suffix>my-index</Suffix>
|
||||
</IndexDocument>
|
||||
<RedirectAllRequestsTo>
|
||||
<HostName>garage.tld</HostName>
|
||||
<Protocol>https</Protocol>
|
||||
</RedirectAllRequestsTo>
|
||||
<RoutingRules>
|
||||
<RoutingRule>
|
||||
<Condition>
|
||||
<HttpErrorCodeReturnedEquals>404</HttpErrorCodeReturnedEquals>
|
||||
<KeyPrefixEquals>prefix1</KeyPrefixEquals>
|
||||
</Condition>
|
||||
<Redirect>
|
||||
<HostName>gara.ge</HostName>
|
||||
<Protocol>http</Protocol>
|
||||
<HttpRedirectCode>303</HttpRedirectCode>
|
||||
<ReplaceKeyPrefixWith>prefix2</ReplaceKeyPrefixWith>
|
||||
<ReplaceKeyWith>fullkey</ReplaceKeyWith>
|
||||
</Redirect>
|
||||
</RoutingRule>
|
||||
</RoutingRules>
|
||||
</WebsiteConfiguration>"#;
|
||||
let conf: WebsiteConfiguration = from_str(message).unwrap();
|
||||
let ref_value = WebsiteConfiguration {
|
||||
xmlns: (),
|
||||
error_document: Some(Key {
|
||||
key: Value("my-error-doc".to_owned()),
|
||||
}),
|
||||
index_document: Some(Suffix {
|
||||
suffix: Value("my-index".to_owned()),
|
||||
}),
|
||||
redirect_all_requests_to: Some(Target {
|
||||
hostname: Value("garage.tld".to_owned()),
|
||||
protocol: Some(Value("https".to_owned())),
|
||||
}),
|
||||
routing_rules: Some(vec![RoutingRule {
|
||||
inner: RoutingRuleInner {
|
||||
condition: Some(Condition {
|
||||
http_error_code: Some(IntValue(404)),
|
||||
prefix: Some(Value("prefix1".to_owned())),
|
||||
}),
|
||||
redirect: Redirect {
|
||||
hostname: Some(Value("gara.ge".to_owned())),
|
||||
protocol: Some(Value("http".to_owned())),
|
||||
http_redirect_code: Some(IntValue(303)),
|
||||
replace_prefix: Some(Value("prefix2".to_owned())),
|
||||
replace_full: Some(Value("fullkey".to_owned())),
|
||||
},
|
||||
},
|
||||
}]),
|
||||
};
|
||||
assert_eq! {
|
||||
ref_value,
|
||||
conf
|
||||
}
|
||||
// TODO verify result is ok
|
||||
// TODO cycle back and verify if ok
|
||||
}
|
||||
}
|
||||
+4
-4
@@ -1,5 +1,5 @@
|
||||
use quick_xml::se::to_string;
|
||||
use serde::{Serialize, Serializer};
|
||||
use serde::{Deserialize, Serialize, Serializer};
|
||||
|
||||
use crate::Error as ApiError;
|
||||
|
||||
@@ -9,14 +9,14 @@ pub fn to_xml_with_header<T: Serialize>(x: &T) -> Result<String, ApiError> {
|
||||
Ok(xml)
|
||||
}
|
||||
|
||||
fn xmlns_tag<S: Serializer>(_v: &(), s: S) -> Result<S::Ok, S::Error> {
|
||||
pub fn xmlns_tag<S: Serializer>(_v: &(), s: S) -> Result<S::Ok, S::Error> {
|
||||
s.serialize_str("http://s3.amazonaws.com/doc/2006-03-01/")
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct Value(#[serde(rename = "$value")] pub String);
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct IntValue(#[serde(rename = "$value")] pub i64);
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage"
|
||||
version = "0.5.0"
|
||||
version = "0.5.1"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
@@ -15,12 +15,12 @@ path = "main.rs"
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
garage_api = { version = "0.5.0", path = "../api" }
|
||||
garage_model = { version = "0.5.0", path = "../model" }
|
||||
garage_rpc = { version = "0.5.0", path = "../rpc" }
|
||||
garage_table = { version = "0.5.0", path = "../table" }
|
||||
garage_util = { version = "0.5.0", path = "../util" }
|
||||
garage_web = { version = "0.5.0", path = "../web" }
|
||||
garage_api = { version = "0.5.1", path = "../api" }
|
||||
garage_model = { version = "0.5.1", path = "../model" }
|
||||
garage_rpc = { version = "0.5.1", path = "../rpc" }
|
||||
garage_table = { version = "0.5.1", path = "../table" }
|
||||
garage_util = { version = "0.5.1", path = "../util" }
|
||||
garage_web = { version = "0.5.1", path = "../web" }
|
||||
|
||||
bytes = "1.0"
|
||||
git-version = "0.3.4"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_model"
|
||||
version = "0.5.0"
|
||||
version = "0.5.1"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
@@ -14,15 +14,16 @@ path = "lib.rs"
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
garage_rpc = { version = "0.5.0", path = "../rpc" }
|
||||
garage_table = { version = "0.5.0", path = "../table" }
|
||||
garage_util = { version = "0.5.0", path = "../util" }
|
||||
garage_rpc = { version = "0.5.1", path = "../rpc" }
|
||||
garage_table = { version = "0.5.1", path = "../table" }
|
||||
garage_util = { version = "0.5.1", path = "../util" }
|
||||
|
||||
async-trait = "0.1.7"
|
||||
arc-swap = "1.0"
|
||||
hex = "0.4"
|
||||
log = "0.4"
|
||||
rand = "0.8"
|
||||
zstd = { version = "0.9", default-features = false }
|
||||
|
||||
sled = "0.34"
|
||||
|
||||
|
||||
+165
-36
@@ -11,6 +11,7 @@ use serde::{Deserialize, Serialize};
|
||||
use tokio::fs;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::sync::{watch, Mutex, Notify};
|
||||
use zstd::stream::{decode_all as zstd_decode, Encoder};
|
||||
|
||||
use garage_util::data::*;
|
||||
use garage_util::error::*;
|
||||
@@ -55,22 +56,81 @@ pub enum BlockRpc {
|
||||
GetBlock(Hash),
|
||||
/// Message to send a block of data, either because requested, of for first delivery of new
|
||||
/// block
|
||||
PutBlock(PutBlockMessage),
|
||||
PutBlock {
|
||||
hash: Hash,
|
||||
data: DataBlock,
|
||||
},
|
||||
/// Ask other node if they should have this block, but don't actually have it
|
||||
NeedBlockQuery(Hash),
|
||||
/// Response : whether the node do require that block
|
||||
NeedBlockReply(bool),
|
||||
}
|
||||
|
||||
/// Structure used to send a block
|
||||
/// A possibly compressed block of data
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct PutBlockMessage {
|
||||
/// Hash of the block
|
||||
pub hash: Hash,
|
||||
pub enum DataBlock {
|
||||
/// Uncompressed data
|
||||
Plain(#[serde(with = "serde_bytes")] Vec<u8>),
|
||||
/// Data compressed with zstd
|
||||
Compressed(#[serde(with = "serde_bytes")] Vec<u8>),
|
||||
}
|
||||
|
||||
/// Content of the block
|
||||
#[serde(with = "serde_bytes")]
|
||||
pub data: Vec<u8>,
|
||||
impl DataBlock {
|
||||
/// Query whether this block is compressed
|
||||
pub fn is_compressed(&self) -> bool {
|
||||
matches!(self, DataBlock::Compressed(_))
|
||||
}
|
||||
|
||||
/// Get the inner, possibly compressed buffer. You should probably use [`DataBlock::verify_get`]
|
||||
/// instead
|
||||
pub fn inner_buffer(&self) -> &[u8] {
|
||||
use DataBlock::*;
|
||||
let (Plain(ref res) | Compressed(ref res)) = self;
|
||||
res
|
||||
}
|
||||
|
||||
/// Get the buffer, possibly decompressing it, and verify it's integrity.
|
||||
/// For Plain block, data is compared to hash, for Compressed block, zstd checksumming system
|
||||
/// is used instead.
|
||||
pub fn verify_get(self, hash: Hash) -> Result<Vec<u8>, Error> {
|
||||
match self {
|
||||
DataBlock::Plain(data) => {
|
||||
if blake2sum(&data) == hash {
|
||||
Ok(data)
|
||||
} else {
|
||||
Err(Error::CorruptData(hash))
|
||||
}
|
||||
}
|
||||
DataBlock::Compressed(data) => {
|
||||
zstd_decode(&data[..]).map_err(|_| Error::CorruptData(hash))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Verify data integrity. Allocate less than [`DataBlock::verify_get`] and don't consume self, but
|
||||
/// does not return the buffer content.
|
||||
pub fn verify(&self, hash: Hash) -> Result<(), Error> {
|
||||
match self {
|
||||
DataBlock::Plain(data) => {
|
||||
if blake2sum(data) == hash {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(Error::CorruptData(hash))
|
||||
}
|
||||
}
|
||||
DataBlock::Compressed(data) => zstd::stream::copy_decode(&data[..], std::io::sink())
|
||||
.map_err(|_| Error::CorruptData(hash)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_buffer(data: Vec<u8>, level: Option<i32>) -> DataBlock {
|
||||
if let Some(level) = level {
|
||||
if let Ok(data) = zstd_encode(&data[..], level) {
|
||||
return DataBlock::Compressed(data);
|
||||
}
|
||||
}
|
||||
DataBlock::Plain(data)
|
||||
}
|
||||
}
|
||||
|
||||
impl Rpc for BlockRpc {
|
||||
@@ -138,10 +198,8 @@ impl BlockManager {
|
||||
block_manager
|
||||
}
|
||||
|
||||
// ---- Public interface ----
|
||||
|
||||
/// Ask nodes that might have a block for it
|
||||
pub async fn rpc_get_block(&self, hash: &Hash) -> Result<Vec<u8>, Error> {
|
||||
/// Ask nodes that might have a (possibly compressed) block for it
|
||||
async fn rpc_get_raw_block(&self, hash: &Hash) -> Result<DataBlock, Error> {
|
||||
let who = self.replication.read_nodes(hash);
|
||||
let resps = self
|
||||
.system
|
||||
@@ -158,8 +216,8 @@ impl BlockManager {
|
||||
.await?;
|
||||
|
||||
for resp in resps {
|
||||
if let BlockRpc::PutBlock(msg) = resp {
|
||||
return Ok(msg.data);
|
||||
if let BlockRpc::PutBlock { data, .. } = resp {
|
||||
return Ok(data);
|
||||
}
|
||||
}
|
||||
Err(Error::Message(format!(
|
||||
@@ -168,15 +226,30 @@ impl BlockManager {
|
||||
)))
|
||||
}
|
||||
|
||||
// ---- Public interface ----
|
||||
|
||||
/// Ask nodes that might have a block for it
|
||||
pub async fn rpc_get_block(&self, hash: &Hash) -> Result<Vec<u8>, Error> {
|
||||
self.rpc_get_raw_block(hash).await?.verify_get(*hash)
|
||||
}
|
||||
|
||||
/// Send block to nodes that should have it
|
||||
pub async fn rpc_put_block(&self, hash: Hash, data: Vec<u8>) -> Result<(), Error> {
|
||||
let who = self.replication.write_nodes(&hash);
|
||||
let compression_level = self
|
||||
.garage
|
||||
.load()
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.config
|
||||
.compression_level;
|
||||
let data = DataBlock::from_buffer(data, compression_level);
|
||||
self.system
|
||||
.rpc
|
||||
.try_call_many(
|
||||
&self.endpoint,
|
||||
&who[..],
|
||||
BlockRpc::PutBlock(PutBlockMessage { hash, data }),
|
||||
BlockRpc::PutBlock { hash, data },
|
||||
RequestStrategy::with_priority(PRIO_NORMAL)
|
||||
.with_quorum(self.replication.write_quorum())
|
||||
.with_timeout(BLOCK_RW_TIMEOUT),
|
||||
@@ -306,7 +379,7 @@ impl BlockManager {
|
||||
// ---- Reading and writing blocks locally ----
|
||||
|
||||
/// Write a block to disk
|
||||
async fn write_block(&self, hash: &Hash, data: &[u8]) -> Result<BlockRpc, Error> {
|
||||
async fn write_block(&self, hash: &Hash, data: &DataBlock) -> Result<BlockRpc, Error> {
|
||||
self.mutation_lock
|
||||
.lock()
|
||||
.await
|
||||
@@ -316,21 +389,31 @@ impl BlockManager {
|
||||
|
||||
/// Read block from disk, verifying it's integrity
|
||||
async fn read_block(&self, hash: &Hash) -> Result<BlockRpc, Error> {
|
||||
let path = self.block_path(hash);
|
||||
|
||||
let mut f = match fs::File::open(&path).await {
|
||||
Ok(f) => f,
|
||||
let mut path = self.block_path(hash);
|
||||
let compressed = match self.is_block_compressed(hash).await {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
// Not found but maybe we should have had it ??
|
||||
self.put_to_resync(hash, 2 * BLOCK_RW_TIMEOUT)?;
|
||||
return Err(Into::into(e));
|
||||
}
|
||||
};
|
||||
if compressed {
|
||||
path.set_extension("zst");
|
||||
}
|
||||
let mut f = fs::File::open(&path).await?;
|
||||
|
||||
let mut data = vec![];
|
||||
f.read_to_end(&mut data).await?;
|
||||
drop(f);
|
||||
|
||||
if blake2sum(&data[..]) != *hash {
|
||||
let data = if compressed {
|
||||
DataBlock::Compressed(data)
|
||||
} else {
|
||||
DataBlock::Plain(data)
|
||||
};
|
||||
|
||||
if data.verify(*hash).is_err() {
|
||||
self.mutation_lock
|
||||
.lock()
|
||||
.await
|
||||
@@ -340,7 +423,7 @@ impl BlockManager {
|
||||
return Err(Error::CorruptData(*hash));
|
||||
}
|
||||
|
||||
Ok(BlockRpc::PutBlock(PutBlockMessage { hash: *hash, data }))
|
||||
Ok(BlockRpc::PutBlock { hash: *hash, data })
|
||||
}
|
||||
|
||||
/// Check if this node should have a block, but don't actually have it
|
||||
@@ -362,13 +445,25 @@ impl BlockManager {
|
||||
path
|
||||
}
|
||||
|
||||
/// Utility: give the full path where a block should be found
|
||||
/// Utility: give the full path where a block should be found, minus extension if block is
|
||||
/// compressed
|
||||
fn block_path(&self, hash: &Hash) -> PathBuf {
|
||||
let mut path = self.block_dir(hash);
|
||||
path.push(hex::encode(hash.as_ref()));
|
||||
path
|
||||
}
|
||||
|
||||
/// Utility: check if block is stored compressed. Error if block is not stored
|
||||
async fn is_block_compressed(&self, hash: &Hash) -> Result<bool, Error> {
|
||||
let mut path = self.block_path(hash);
|
||||
path.set_extension("zst");
|
||||
if fs::metadata(&path).await.is_ok() {
|
||||
return Ok(true);
|
||||
}
|
||||
path.set_extension("");
|
||||
fs::metadata(&path).await.map(|_| false).map_err(Into::into)
|
||||
}
|
||||
|
||||
// ---- Resync loop ----
|
||||
|
||||
pub fn spawn_background_worker(self: Arc<Self>) {
|
||||
@@ -550,8 +645,8 @@ impl BlockManager {
|
||||
hash
|
||||
);
|
||||
|
||||
let block_data = self.rpc_get_block(hash).await?;
|
||||
self.write_block(hash, &block_data[..]).await?;
|
||||
let block_data = self.rpc_get_raw_block(hash).await?;
|
||||
self.write_block(hash, &block_data).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -602,6 +697,7 @@ impl BlockManager {
|
||||
};
|
||||
let ent_type = data_dir_ent.file_type().await?;
|
||||
|
||||
let name = name.strip_suffix(".zst").unwrap_or(&name);
|
||||
if name.len() == 2 && hex::decode(&name).is_ok() && ent_type.is_dir() {
|
||||
state = self
|
||||
.for_each_file_rec(&data_dir_ent.path(), state, f, must_exit)
|
||||
@@ -631,7 +727,7 @@ impl EndpointHandler<BlockRpc> for BlockManager {
|
||||
_from: NodeID,
|
||||
) -> Result<BlockRpc, Error> {
|
||||
match message {
|
||||
BlockRpc::PutBlock(m) => self.write_block(&m.hash, &m.data).await,
|
||||
BlockRpc::PutBlock { hash, data } => self.write_block(hash, data).await,
|
||||
BlockRpc::GetBlock(h) => self.read_block(h).await,
|
||||
BlockRpc::NeedBlockQuery(h) => self.need_block(h).await.map(BlockRpc::NeedBlockReply),
|
||||
_ => Err(Error::BadRpc("Unexpected RPC message".to_string())),
|
||||
@@ -650,9 +746,7 @@ impl BlockManagerLocked {
|
||||
hash: &Hash,
|
||||
mgr: &BlockManager,
|
||||
) -> Result<BlockStatus, Error> {
|
||||
let path = mgr.block_path(hash);
|
||||
|
||||
let exists = fs::metadata(&path).await.is_ok();
|
||||
let exists = mgr.is_block_compressed(hash).await.is_ok();
|
||||
let needed = mgr.get_block_rc(hash)?;
|
||||
|
||||
Ok(BlockStatus { exists, needed })
|
||||
@@ -661,16 +755,31 @@ impl BlockManagerLocked {
|
||||
async fn write_block(
|
||||
&self,
|
||||
hash: &Hash,
|
||||
data: &[u8],
|
||||
data: &DataBlock,
|
||||
mgr: &BlockManager,
|
||||
) -> Result<BlockRpc, Error> {
|
||||
let compressed = data.is_compressed();
|
||||
let data = data.inner_buffer();
|
||||
|
||||
let mut path = mgr.block_dir(hash);
|
||||
fs::create_dir_all(&path).await?;
|
||||
|
||||
path.push(hex::encode(hash));
|
||||
if fs::metadata(&path).await.is_ok() {
|
||||
return Ok(BlockRpc::Ok);
|
||||
}
|
||||
let to_delete = match (mgr.is_block_compressed(hash).await, compressed) {
|
||||
(Ok(true), _) => return Ok(BlockRpc::Ok),
|
||||
(Ok(false), false) => return Ok(BlockRpc::Ok),
|
||||
(Ok(false), true) => {
|
||||
let path_to_delete = path.clone();
|
||||
path.set_extension("zst");
|
||||
Some(path_to_delete)
|
||||
}
|
||||
(Err(_), compressed) => {
|
||||
if compressed {
|
||||
path.set_extension("zst");
|
||||
}
|
||||
None
|
||||
}
|
||||
};
|
||||
|
||||
let mut path2 = path.clone();
|
||||
path2.set_extension("tmp");
|
||||
@@ -679,6 +788,9 @@ impl BlockManagerLocked {
|
||||
drop(f);
|
||||
|
||||
fs::rename(path2, path).await?;
|
||||
if let Some(to_delete) = to_delete {
|
||||
fs::remove_file(to_delete).await?;
|
||||
}
|
||||
|
||||
Ok(BlockRpc::Ok)
|
||||
}
|
||||
@@ -688,9 +800,14 @@ impl BlockManagerLocked {
|
||||
"Block {:?} is corrupted. Renaming to .corrupted and resyncing.",
|
||||
hash
|
||||
);
|
||||
let path = mgr.block_path(hash);
|
||||
let mut path = mgr.block_path(hash);
|
||||
let mut path2 = path.clone();
|
||||
path2.set_extension("corrupted");
|
||||
if mgr.is_block_compressed(hash).await? {
|
||||
path.set_extension("zst");
|
||||
path2.set_extension("zst.corrupted");
|
||||
} else {
|
||||
path2.set_extension("corrupted");
|
||||
}
|
||||
fs::rename(path, path2).await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -699,7 +816,10 @@ impl BlockManagerLocked {
|
||||
let BlockStatus { exists, needed } = self.check_block_status(hash, mgr).await?;
|
||||
|
||||
if exists && needed.is_deletable() {
|
||||
let path = mgr.block_path(hash);
|
||||
let mut path = mgr.block_path(hash);
|
||||
if mgr.is_block_compressed(hash).await? {
|
||||
path.set_extension("zst");
|
||||
}
|
||||
fs::remove_file(path).await?;
|
||||
}
|
||||
Ok(())
|
||||
@@ -806,3 +926,12 @@ impl RcEntry {
|
||||
!self.is_deletable()
|
||||
}
|
||||
}
|
||||
|
||||
fn zstd_encode<R: std::io::Read>(mut source: R, level: i32) -> std::io::Result<Vec<u8>> {
|
||||
let mut result = Vec::<u8>::new();
|
||||
let mut encoder = Encoder::new(&mut result, level)?;
|
||||
encoder.include_checksum(true)?;
|
||||
std::io::copy(&mut source, &mut encoder)?;
|
||||
encoder.finish()?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_rpc"
|
||||
version = "0.5.0"
|
||||
version = "0.5.1"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
@@ -14,7 +14,7 @@ path = "lib.rs"
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
garage_util = { version = "0.5.0", path = "../util" }
|
||||
garage_util = { version = "0.5.1", path = "../util" }
|
||||
|
||||
arc-swap = "1.0"
|
||||
bytes = "1.0"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_table"
|
||||
version = "0.5.0"
|
||||
version = "0.5.1"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
@@ -14,8 +14,8 @@ path = "lib.rs"
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
garage_rpc = { version = "0.5.0", path = "../rpc" }
|
||||
garage_util = { version = "0.5.0", path = "../util" }
|
||||
garage_rpc = { version = "0.5.1", path = "../rpc" }
|
||||
garage_util = { version = "0.5.1", path = "../util" }
|
||||
|
||||
async-trait = "0.1.7"
|
||||
bytes = "1.0"
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_util"
|
||||
version = "0.5.0"
|
||||
version = "0.5.1"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
|
||||
@@ -30,6 +30,13 @@ pub struct Config {
|
||||
// (we can add more aliases for this later)
|
||||
pub replication_mode: String,
|
||||
|
||||
/// Zstd compression level used on data blocks
|
||||
#[serde(
|
||||
deserialize_with = "deserialize_compression",
|
||||
default = "default_compression"
|
||||
)]
|
||||
pub compression_level: Option<i32>,
|
||||
|
||||
/// RPC secret key: 32 bytes hex encoded
|
||||
pub rpc_secret: String,
|
||||
|
||||
@@ -123,3 +130,57 @@ where
|
||||
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
fn default_compression() -> Option<i32> {
|
||||
Some(1)
|
||||
}
|
||||
|
||||
fn deserialize_compression<'de, D>(deserializer: D) -> Result<Option<i32>, D::Error>
|
||||
where
|
||||
D: de::Deserializer<'de>,
|
||||
{
|
||||
use std::convert::TryFrom;
|
||||
|
||||
struct OptionVisitor;
|
||||
|
||||
impl<'de> serde::de::Visitor<'de> for OptionVisitor {
|
||||
type Value = Option<i32>;
|
||||
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
formatter.write_str("int or 'none'")
|
||||
}
|
||||
|
||||
fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
|
||||
where
|
||||
E: de::Error,
|
||||
{
|
||||
if value.eq_ignore_ascii_case("none") {
|
||||
Ok(None)
|
||||
} else {
|
||||
Err(E::custom(format!(
|
||||
"Invalid compression level: '{}', should be a number, or 'none'",
|
||||
value
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
fn visit_i64<E>(self, v: i64) -> Result<Self::Value, E>
|
||||
where
|
||||
E: de::Error,
|
||||
{
|
||||
i32::try_from(v)
|
||||
.map(Some)
|
||||
.map_err(|_| E::custom("Compression level out of bound".to_owned()))
|
||||
}
|
||||
|
||||
fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E>
|
||||
where
|
||||
E: de::Error,
|
||||
{
|
||||
i32::try_from(v)
|
||||
.map(Some)
|
||||
.map_err(|_| E::custom("Compression level out of bound".to_owned()))
|
||||
}
|
||||
}
|
||||
|
||||
deserializer.deserialize_any(OptionVisitor)
|
||||
}
|
||||
|
||||
@@ -77,11 +77,21 @@ where
|
||||
self.v = new_value;
|
||||
}
|
||||
|
||||
/// Get the timestamp currently associated with the value
|
||||
pub fn timestamp(&self) -> u64 {
|
||||
self.ts
|
||||
}
|
||||
|
||||
/// Get the CRDT value
|
||||
pub fn get(&self) -> &T {
|
||||
&self.v
|
||||
}
|
||||
|
||||
/// Take the CRDT value
|
||||
pub fn take(self) -> T {
|
||||
self.v
|
||||
}
|
||||
|
||||
/// Get a mutable reference to the CRDT's value
|
||||
///
|
||||
/// This is usefull to mutate the inside value without changing the LWW timestamp.
|
||||
|
||||
+5
-5
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_web"
|
||||
version = "0.5.0"
|
||||
version = "0.5.1"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
@@ -14,10 +14,10 @@ path = "lib.rs"
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
garage_api = { version = "0.5.0", path = "../api" }
|
||||
garage_model = { version = "0.5.0", path = "../model" }
|
||||
garage_util = { version = "0.5.0", path = "../util" }
|
||||
garage_table = { version = "0.5.0", path = "../table" }
|
||||
garage_api = { version = "0.5.1", path = "../api" }
|
||||
garage_model = { version = "0.5.1", path = "../model" }
|
||||
garage_util = { version = "0.5.1", path = "../util" }
|
||||
garage_table = { version = "0.5.1", path = "../table" }
|
||||
|
||||
err-derive = "0.3"
|
||||
log = "0.4"
|
||||
|
||||
Reference in New Issue
Block a user