mirror of
https://github.com/deuxfleurs-org/garage.git
synced 2026-08-14 08:03:11 +00:00
Compare commits
17 Commits
v0.5-beta1
...
v0.5.1
| Author | SHA1 | Date | |
|---|---|---|---|
 Alex Auvolat
|
398ddb1300 | ||
|
trinity-1686a
|
945b75dbf1 | ||
|
trinity-1686a
|
ca7b438f3f | ||
|
trinity-1686a
|
1eb972b1ac | ||
|
trinity-1686a
|
60d4459926 | ||
|
Quentin Dufour
|
3b3a1f275f | ||
|
Quentin Dufour
|
dba9af2968 | ||
|
Quentin Dufour
|
e9358054ac | ||
|
Quentin Dufour
|
f9e5520ffb | ||
|
Alex Auvolat
|
4b369347c0 | ||
|
Alex Auvolat
|
224c89ad6e | ||
|
Quentin Dufour
|
7c2037ba87 | ||
|
trinity-1686a
|
c4ac8835d3 | ||
|
Alex Auvolat
|
ccce75bc25 | ||
|
trinity-1686a
|
7f26ed55cd | ||
|
Quentin Dufour
|
8811bb08e6 | ||
|
Quentin Dufour
|
85b2e4ca29 |
Generated
+49
-7
@@ -106,6 +106,9 @@ name = "cc"
|
||||
version = "1.0.71"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "79c2681d6594606957bbb8631c4b90a7fcaaa72cdb714743a437b156d6a7eedd"
|
||||
dependencies = [
|
||||
"jobserver",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
@@ -379,7 +382,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "garage"
|
||||
version = "0.5.0"
|
||||
version = "0.5.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes 1.1.0",
|
||||
@@ -408,7 +411,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "garage_api"
|
||||
version = "0.5.0"
|
||||
version = "0.5.1"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"bytes 1.1.0",
|
||||
@@ -440,7 +443,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "garage_model"
|
||||
version = "0.5.0"
|
||||
version = "0.5.1"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -458,11 +461,12 @@ dependencies = [
|
||||
"serde_bytes",
|
||||
"sled",
|
||||
"tokio",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "garage_rpc"
|
||||
version = "0.5.0"
|
||||
version = "0.5.1"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -487,7 +491,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "garage_table"
|
||||
version = "0.5.0"
|
||||
version = "0.5.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes 1.1.0",
|
||||
@@ -507,7 +511,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "garage_util"
|
||||
version = "0.5.0"
|
||||
version = "0.5.1"
|
||||
dependencies = [
|
||||
"blake2",
|
||||
"chrono",
|
||||
@@ -531,7 +535,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "garage_web"
|
||||
version = "0.5.0"
|
||||
version = "0.5.1"
|
||||
dependencies = [
|
||||
"err-derive 0.3.0",
|
||||
"futures",
|
||||
@@ -752,6 +756,15 @@ version = "0.4.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
|
||||
|
||||
[[package]]
|
||||
name = "jobserver"
|
||||
version = "0.1.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "kuska-handshake"
|
||||
version = "0.2.0"
|
||||
@@ -1652,3 +1665,32 @@ name = "xxhash-rust"
|
||||
version = "0.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e575e15bedf6e57b5c2d763ffc6c3c760143466cbd09d762d539680ab5992ded"
|
||||
|
||||
[[package]]
|
||||
name = "zstd"
|
||||
version = "0.9.0+zstd.1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "07749a5dc2cb6b36661290245e350f15ec3bbb304e493db54a1d354480522ccd"
|
||||
dependencies = [
|
||||
"zstd-safe",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd-safe"
|
||||
version = "4.1.1+zstd.1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c91c90f2c593b003603e5e0493c837088df4469da25aafff8bce42ba48caf079"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"zstd-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd-sys"
|
||||
version = "1.6.1+zstd.1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "615120c7a2431d16cf1cf979e7fc31ba7a5b5e5707b29c8a99e5dbf8a8392a33"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
]
|
||||
|
||||
@@ -40,13 +40,13 @@ in
|
||||
{
|
||||
cargo2nixVersion = "0.9.0";
|
||||
workspace = {
|
||||
garage_util = rustPackages.unknown.garage_util."0.5.0";
|
||||
garage_rpc = rustPackages.unknown.garage_rpc."0.5.0";
|
||||
garage_table = rustPackages.unknown.garage_table."0.5.0";
|
||||
garage_model = rustPackages.unknown.garage_model."0.5.0";
|
||||
garage_api = rustPackages.unknown.garage_api."0.5.0";
|
||||
garage_web = rustPackages.unknown.garage_web."0.5.0";
|
||||
garage = rustPackages.unknown.garage."0.5.0";
|
||||
garage_util = rustPackages.unknown.garage_util."0.5.1";
|
||||
garage_rpc = rustPackages.unknown.garage_rpc."0.5.1";
|
||||
garage_table = rustPackages.unknown.garage_table."0.5.1";
|
||||
garage_model = rustPackages.unknown.garage_model."0.5.1";
|
||||
garage_api = rustPackages.unknown.garage_api."0.5.1";
|
||||
garage_web = rustPackages.unknown.garage_web."0.5.1";
|
||||
garage = rustPackages.unknown.garage."0.5.1";
|
||||
};
|
||||
"registry+https://github.com/rust-lang/crates.io-index".aho-corasick."0.7.18" = overridableMkRustCrate (profileName: rec {
|
||||
name = "aho-corasick";
|
||||
@@ -196,6 +196,13 @@ in
|
||||
version = "1.0.71";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "79c2681d6594606957bbb8631c4b90a7fcaaa72cdb714743a437b156d6a7eedd"; };
|
||||
features = builtins.concatLists [
|
||||
[ "jobserver" ]
|
||||
[ "parallel" ]
|
||||
];
|
||||
dependencies = {
|
||||
jobserver = rustPackages."registry+https://github.com/rust-lang/crates.io-index".jobserver."0.1.24" { inherit profileName; };
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" = overridableMkRustCrate (profileName: rec {
|
||||
@@ -246,7 +253,7 @@ in
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469"; };
|
||||
dependencies = {
|
||||
${ if hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" || hostPlatform.config == "aarch64-apple-darwin" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.103" { inherit profileName; };
|
||||
${ if hostPlatform.config == "aarch64-apple-darwin" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.103" { inherit profileName; };
|
||||
};
|
||||
});
|
||||
|
||||
@@ -606,9 +613,9 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"unknown".garage."0.5.0" = overridableMkRustCrate (profileName: rec {
|
||||
"unknown".garage."0.5.1" = overridableMkRustCrate (profileName: rec {
|
||||
name = "garage";
|
||||
version = "0.5.0";
|
||||
version = "0.5.1";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/garage");
|
||||
dependencies = {
|
||||
@@ -616,12 +623,12 @@ in
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.17" { inherit profileName; };
|
||||
futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.17" { inherit profileName; };
|
||||
garage_api = rustPackages."unknown".garage_api."0.5.0" { inherit profileName; };
|
||||
garage_model = rustPackages."unknown".garage_model."0.5.0" { inherit profileName; };
|
||||
garage_rpc = rustPackages."unknown".garage_rpc."0.5.0" { inherit profileName; };
|
||||
garage_table = rustPackages."unknown".garage_table."0.5.0" { inherit profileName; };
|
||||
garage_util = rustPackages."unknown".garage_util."0.5.0" { inherit profileName; };
|
||||
garage_web = rustPackages."unknown".garage_web."0.5.0" { inherit profileName; };
|
||||
garage_api = rustPackages."unknown".garage_api."0.5.1" { inherit profileName; };
|
||||
garage_model = rustPackages."unknown".garage_model."0.5.1" { inherit profileName; };
|
||||
garage_rpc = rustPackages."unknown".garage_rpc."0.5.1" { inherit profileName; };
|
||||
garage_table = rustPackages."unknown".garage_table."0.5.1" { inherit profileName; };
|
||||
garage_util = rustPackages."unknown".garage_util."0.5.1" { inherit profileName; };
|
||||
garage_web = rustPackages."unknown".garage_web."0.5.1" { inherit profileName; };
|
||||
git_version = rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; };
|
||||
hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
|
||||
sodiumoxide = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; };
|
||||
@@ -638,9 +645,9 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"unknown".garage_api."0.5.0" = overridableMkRustCrate (profileName: rec {
|
||||
"unknown".garage_api."0.5.1" = overridableMkRustCrate (profileName: rec {
|
||||
name = "garage_api";
|
||||
version = "0.5.0";
|
||||
version = "0.5.1";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/api");
|
||||
dependencies = {
|
||||
@@ -651,9 +658,9 @@ in
|
||||
err_derive = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.0" { profileName = "__noProfile"; };
|
||||
futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.17" { inherit profileName; };
|
||||
futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.17" { inherit profileName; };
|
||||
garage_model = rustPackages."unknown".garage_model."0.5.0" { inherit profileName; };
|
||||
garage_table = rustPackages."unknown".garage_table."0.5.0" { inherit profileName; };
|
||||
garage_util = rustPackages."unknown".garage_util."0.5.0" { inherit profileName; };
|
||||
garage_model = rustPackages."unknown".garage_model."0.5.1" { inherit profileName; };
|
||||
garage_table = rustPackages."unknown".garage_table."0.5.1" { inherit profileName; };
|
||||
garage_util = rustPackages."unknown".garage_util."0.5.1" { inherit profileName; };
|
||||
hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
|
||||
hmac = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.10.1" { inherit profileName; };
|
||||
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.5" { inherit profileName; };
|
||||
@@ -673,9 +680,9 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"unknown".garage_model."0.5.0" = overridableMkRustCrate (profileName: rec {
|
||||
"unknown".garage_model."0.5.1" = overridableMkRustCrate (profileName: rec {
|
||||
name = "garage_model";
|
||||
version = "0.5.0";
|
||||
version = "0.5.1";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/model");
|
||||
dependencies = {
|
||||
@@ -683,9 +690,9 @@ in
|
||||
async_trait = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.51" { profileName = "__noProfile"; };
|
||||
futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.17" { inherit profileName; };
|
||||
futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.17" { inherit profileName; };
|
||||
garage_rpc = rustPackages."unknown".garage_rpc."0.5.0" { inherit profileName; };
|
||||
garage_table = rustPackages."unknown".garage_table."0.5.0" { inherit profileName; };
|
||||
garage_util = rustPackages."unknown".garage_util."0.5.0" { inherit profileName; };
|
||||
garage_rpc = rustPackages."unknown".garage_rpc."0.5.1" { inherit profileName; };
|
||||
garage_table = rustPackages."unknown".garage_table."0.5.1" { inherit profileName; };
|
||||
garage_util = rustPackages."unknown".garage_util."0.5.1" { inherit profileName; };
|
||||
hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
|
||||
log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; };
|
||||
netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.0" { inherit profileName; };
|
||||
@@ -695,12 +702,13 @@ in
|
||||
serde_bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; };
|
||||
sled = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sled."0.34.7" { inherit profileName; };
|
||||
tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.12.0" { inherit profileName; };
|
||||
zstd = rustPackages."registry+https://github.com/rust-lang/crates.io-index".zstd."0.9.0+zstd.1.5.0" { inherit profileName; };
|
||||
};
|
||||
});
|
||||
|
||||
"unknown".garage_rpc."0.5.0" = overridableMkRustCrate (profileName: rec {
|
||||
"unknown".garage_rpc."0.5.1" = overridableMkRustCrate (profileName: rec {
|
||||
name = "garage_rpc";
|
||||
version = "0.5.0";
|
||||
version = "0.5.1";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/rpc");
|
||||
dependencies = {
|
||||
@@ -709,7 +717,7 @@ in
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.17" { inherit profileName; };
|
||||
futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.17" { inherit profileName; };
|
||||
garage_util = rustPackages."unknown".garage_util."0.5.0" { inherit profileName; };
|
||||
garage_util = rustPackages."unknown".garage_util."0.5.1" { inherit profileName; };
|
||||
gethostname = rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.2.1" { inherit profileName; };
|
||||
hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
|
||||
hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.13" { inherit profileName; };
|
||||
@@ -726,9 +734,9 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"unknown".garage_table."0.5.0" = overridableMkRustCrate (profileName: rec {
|
||||
"unknown".garage_table."0.5.1" = overridableMkRustCrate (profileName: rec {
|
||||
name = "garage_table";
|
||||
version = "0.5.0";
|
||||
version = "0.5.1";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/table");
|
||||
dependencies = {
|
||||
@@ -736,8 +744,8 @@ in
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.17" { inherit profileName; };
|
||||
futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.17" { inherit profileName; };
|
||||
garage_rpc = rustPackages."unknown".garage_rpc."0.5.0" { inherit profileName; };
|
||||
garage_util = rustPackages."unknown".garage_util."0.5.0" { inherit profileName; };
|
||||
garage_rpc = rustPackages."unknown".garage_rpc."0.5.1" { inherit profileName; };
|
||||
garage_util = rustPackages."unknown".garage_util."0.5.1" { inherit profileName; };
|
||||
hexdump = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; };
|
||||
log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; };
|
||||
rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.4" { inherit profileName; };
|
||||
@@ -749,9 +757,9 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"unknown".garage_util."0.5.0" = overridableMkRustCrate (profileName: rec {
|
||||
"unknown".garage_util."0.5.1" = overridableMkRustCrate (profileName: rec {
|
||||
name = "garage_util";
|
||||
version = "0.5.0";
|
||||
version = "0.5.1";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/util");
|
||||
dependencies = {
|
||||
@@ -776,18 +784,18 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"unknown".garage_web."0.5.0" = overridableMkRustCrate (profileName: rec {
|
||||
"unknown".garage_web."0.5.1" = overridableMkRustCrate (profileName: rec {
|
||||
name = "garage_web";
|
||||
version = "0.5.0";
|
||||
version = "0.5.1";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/web");
|
||||
dependencies = {
|
||||
err_derive = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.0" { profileName = "__noProfile"; };
|
||||
futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.17" { inherit profileName; };
|
||||
garage_api = rustPackages."unknown".garage_api."0.5.0" { inherit profileName; };
|
||||
garage_model = rustPackages."unknown".garage_model."0.5.0" { inherit profileName; };
|
||||
garage_table = rustPackages."unknown".garage_table."0.5.0" { inherit profileName; };
|
||||
garage_util = rustPackages."unknown".garage_util."0.5.0" { inherit profileName; };
|
||||
garage_api = rustPackages."unknown".garage_api."0.5.1" { inherit profileName; };
|
||||
garage_model = rustPackages."unknown".garage_model."0.5.1" { inherit profileName; };
|
||||
garage_table = rustPackages."unknown".garage_table."0.5.1" { inherit profileName; };
|
||||
garage_util = rustPackages."unknown".garage_util."0.5.1" { inherit profileName; };
|
||||
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.5" { inherit profileName; };
|
||||
hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.13" { inherit profileName; };
|
||||
log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; };
|
||||
@@ -1055,6 +1063,16 @@ in
|
||||
];
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".jobserver."0.1.24" = overridableMkRustCrate (profileName: rec {
|
||||
name = "jobserver";
|
||||
version = "0.1.24";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa"; };
|
||||
dependencies = {
|
||||
${ if hostPlatform.isUnix then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.103" { inherit profileName; };
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".kuska-handshake."0.2.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "kuska-handshake";
|
||||
version = "0.2.0";
|
||||
@@ -2351,4 +2369,44 @@ in
|
||||
];
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".zstd."0.9.0+zstd.1.5.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "zstd";
|
||||
version = "0.9.0+zstd.1.5.0";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "07749a5dc2cb6b36661290245e350f15ec3bbb304e493db54a1d354480522ccd"; };
|
||||
dependencies = {
|
||||
zstd_safe = rustPackages."registry+https://github.com/rust-lang/crates.io-index".zstd-safe."4.1.1+zstd.1.5.0" { inherit profileName; };
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".zstd-safe."4.1.1+zstd.1.5.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "zstd-safe";
|
||||
version = "4.1.1+zstd.1.5.0";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "c91c90f2c593b003603e5e0493c837088df4469da25aafff8bce42ba48caf079"; };
|
||||
features = builtins.concatLists [
|
||||
[ "std" ]
|
||||
];
|
||||
dependencies = {
|
||||
libc = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.103" { inherit profileName; };
|
||||
zstd_sys = rustPackages."registry+https://github.com/rust-lang/crates.io-index".zstd-sys."1.6.1+zstd.1.5.0" { inherit profileName; };
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".zstd-sys."1.6.1+zstd.1.5.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "zstd-sys";
|
||||
version = "1.6.1+zstd.1.5.0";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "615120c7a2431d16cf1cf979e7fc31ba7a5b5e5707b29c8a99e5dbf8a8392a33"; };
|
||||
features = builtins.concatLists [
|
||||
[ "std" ]
|
||||
];
|
||||
dependencies = {
|
||||
libc = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.103" { inherit profileName; };
|
||||
};
|
||||
buildDependencies = {
|
||||
cc = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.0.71" { profileName = "__noProfile"; };
|
||||
};
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
Testing
|
||||
-------
|
||||
|
||||
How are we going to test that our replication method works correctly?
|
||||
We will have to introduce lots of dummy data and then add/remove nodes many times.
|
||||
|
||||
|
||||
Attaining S3 compatibility
|
||||
--------------------------
|
||||
|
||||
- test multipart uploads
|
||||
- get ranges
|
||||
|
||||
- fix sync not working in some cases ? (when starting from empty?)
|
||||
|
||||
- api_server following the S3 semantics for head/get/put/list/delete: verify more that it works as intended
|
||||
- PUT requests: verify content-md5 if provided
|
||||
- possibly other necessary endpoints ?
|
||||
|
||||
|
||||
Lower priority
|
||||
--------------
|
||||
|
||||
- less a priority: hinted handoff
|
||||
- repair: re-propagate block ref table to rc
|
||||
- FIXME in rpc_server when garage shuts down and futures can be interrupted
|
||||
(tokio::spawn should be replaced by a new function background::spawn_joinable)
|
||||
@@ -8,7 +8,7 @@
|
||||
- [Multi-node deployment](./cookbook/real_world.md)
|
||||
- [Building from source](./cookbook/from_source.md)
|
||||
- [Integration with systemd](./cookbook/systemd.md)
|
||||
- [Gateways](./cookbook/gateways.md)
|
||||
- [Configuring a gateway node](./cookbook/gateways.md)
|
||||
- [Exposing buckets as websites](./cookbook/exposing_websites.md)
|
||||
- [Configuring a reverse proxy](./cookbook/reverse_proxy.md)
|
||||
- [Recovering from failures](./cookbook/recovering.md)
|
||||
@@ -30,9 +30,10 @@
|
||||
- [S3 compatibility status](./reference_manual/s3_compatibility.md)
|
||||
|
||||
- [Design](./design/index.md)
|
||||
- [Related Work](./design/related_work.md)
|
||||
- [Goals and use Cases](./design/goals.md)
|
||||
- [Benchmarks](./design/benchmarks.md)
|
||||
- [Related work](./design/related_work.md)
|
||||
- [Internals](./design/internals.md)
|
||||
- [Design draft](./design/design_draft.md)
|
||||
|
||||
- [Development](./development/index.md)
|
||||
- [Setup your environment](./development/devenv.md)
|
||||
@@ -41,5 +42,7 @@
|
||||
- [Miscellaneous notes](./development/miscellaneous_notes.md)
|
||||
|
||||
- [Working Documents](./working_documents/index.md)
|
||||
- [Load Balancing Data](./working_documents/load_balancing.md)
|
||||
- [S3 compatibility target](./working_documents/compatibility_target.md)
|
||||
- [Load balancing data](./working_documents/load_balancing.md)
|
||||
- [Migrating from 0.3 to 0.4](./working_documents/migration_04.md)
|
||||
- [Design draft](./working_documents/design_draft.md)
|
||||
|
||||
@@ -109,7 +109,7 @@ Do not change the `use_path_style` and `legacy_auth` entries, other configuratio
|
||||
Peertube proposes a clever integration of S3 by directly exposing its endpoint instead of proxifying requests through the application.
|
||||
In other words, Peertube is only responsible of the "control plane" and offload the "data plane" to Garage.
|
||||
In return, this system is a bit harder to configure, especially with Garage that supports less feature than other older S3 backends.
|
||||
We show that it is still possible to configure Garage with Peertube, allowing you to spread the load and the bandiwdth usage on the Garage cluster.
|
||||
We show that it is still possible to configure Garage with Peertube, allowing you to spread the load and the bandwidth usage on the Garage cluster.
|
||||
|
||||
### Enable path-style access by patching Peertube
|
||||
|
||||
|
||||
@@ -1,7 +1,19 @@
|
||||
# Connect it to
|
||||
# Integrations
|
||||
|
||||
To configure an S3 client to interact with Garage, you will need the following
|
||||
parameters:
|
||||
Garage implements the Amazon S3 protocol, which makes it compatible with many existing software programs.
|
||||
|
||||
In particular, you will find here instructions to connect it with:
|
||||
|
||||
- [web applications](./apps.md)
|
||||
- [website hosting](./websites.md)
|
||||
- [software repositories](./repositories.md)
|
||||
- [CLI tools](./cli.md)
|
||||
- [your own code](./code.md)
|
||||
|
||||
### Generic instructions
|
||||
|
||||
To configure S3-compatible software to interact with Garage,
|
||||
you will need the following parameters:
|
||||
|
||||
- An **API endpoint**: this corresponds to the HTTP or HTTPS address
|
||||
used to contact the Garage server. When runing Garage locally this will usually
|
||||
@@ -27,12 +39,3 @@ provided that you follow the following guidelines:
|
||||
If this is not configured explicitly, clients usually try to talk to region `us-east-1`.
|
||||
Garage should normally redirect your client to the correct region,
|
||||
but in case your client does not support this you might have to configure it manually.
|
||||
|
||||
We will now provide example configurations for the most common clients per category:
|
||||
|
||||
- [Apps](./apps.md)
|
||||
- [Websites](./websites.md)
|
||||
- [Repositories](./repositories.md)
|
||||
- [CLI tools](./cli.md)
|
||||
- [Your code](./code.md)
|
||||
|
||||
|
||||
@@ -6,11 +6,11 @@ Gateways allow you to expose Garage endpoints (S3 API and websites) without stor
|
||||
|
||||
You can configure Garage as a gateway on all nodes that will consume your S3 API, it will provide you the following benefits:
|
||||
|
||||
- **It removes 1 or 2 network RTT** Instead of (querying your reverse proxy then) querying a random node of the cluster that will forward your request to the nodes effectively storing the data, your local gateway will directly knows which node to query.
|
||||
- **It removes 1 or 2 network RTT.** Instead of (querying your reverse proxy then) querying a random node of the cluster that will forward your request to the nodes effectively storing the data, your local gateway will directly knows which node to query.
|
||||
|
||||
- **It ease server management** Instead of tracking in your reverse proxy and DNS what are the current Garage nodes, your gateway being part of the cluster keeps this information for you. In your software, you will always specify `http://localhost:3900`.
|
||||
- **It eases server management.** Instead of tracking in your reverse proxy and DNS what are the current Garage nodes, your gateway being part of the cluster keeps this information for you. In your software, you will always specify `http://localhost:3900`.
|
||||
|
||||
- **It simplifies security** Instead of having to maintain and renew a TLS certificate, you leverage the Secret Handshake protocol we use for our cluster. The S3 API protocol will be in plain text but limited to your local machine.
|
||||
- **It simplifies security.** Instead of having to maintain and renew a TLS certificate, you leverage the Secret Handshake protocol we use for our cluster. The S3 API protocol will be in plain text but limited to your local machine.
|
||||
|
||||
## Limitations
|
||||
|
||||
|
||||
@@ -4,22 +4,23 @@ A cookbook, when you cook, is a collection of recipes.
|
||||
Similarly, Garage's cookbook contains a collection of recipes that are known to works well!
|
||||
This chapter could also be referred as "Tutorials" or "Best practices".
|
||||
|
||||
- **[Deploying Garage](real_world.md):** This page will walk you through all of the necessary
|
||||
- **[Multi-node deployment](real_world.md):** This page will walk you through all of the necessary
|
||||
steps to deploy Garage in a real-world setting.
|
||||
|
||||
- **[Configuring S3 clients](clients.md):** This page will explain how to configure
|
||||
popular S3 clients to interact with a Garage server.
|
||||
|
||||
- **[Hosting a website](website.md):** This page explains how to use Garage
|
||||
to host a static website.
|
||||
|
||||
- **[Recovering from failures](recovering.md):** Garage's first selling point is resilience
|
||||
to hardware failures. This section explains how to recover from such a failure in the
|
||||
best possible way.
|
||||
|
||||
- **[Building from source](from_source.md):** This page explains how to build Garage from
|
||||
source in case a binary is not provided for your architecture, or if you want to
|
||||
hack with us!
|
||||
|
||||
- **[Starting with Systemd](from_source.md):** This page explains how to run Garage
|
||||
- **[Integration with Systemd](systemd.md):** This page explains how to run Garage
|
||||
as a Systemd service (instead of as a Docker container).
|
||||
|
||||
- **[Configuring a gateway node](gateways.md):** This page explains how to run a gateway node in a Garage cluster, i.e. a Garage node that doesn't store data but accelerates access to data present on the other nodes.
|
||||
|
||||
- **[Hosting a website](exposing_websites.md):** This page explains how to use Garage
|
||||
to host a static website.
|
||||
|
||||
- **[Configuring a reverse-proxy](reverse_proxy.md):** This page explains how to configure a reverse-proxy to add TLS support to your S3 api endpoint.
|
||||
|
||||
- **[Recovering from failures](recovering.md):** Garage's first selling point is resilience
|
||||
to hardware failures. This section explains how to recover from such a failure in the
|
||||
best possible way.
|
||||
|
||||
@@ -74,6 +74,8 @@ data_dir = "/var/lib/garage/data"
|
||||
|
||||
replication_mode = "3"
|
||||
|
||||
compression_level = 2
|
||||
|
||||
rpc_bind_addr = "[::]:3901"
|
||||
rpc_public_addr = "<this node's public IP>:3901"
|
||||
rpc_secret = "<RPC secret>"
|
||||
@@ -286,5 +288,5 @@ and is covered in the [quick start guide](../quick_start/index.md).
|
||||
Remember also that the CLI is self-documented thanks to the `--help` flag and
|
||||
the `help` subcommand (e.g. `garage help`, `garage key --help`).
|
||||
|
||||
Configuring an S3 client to interact with Garage is covered
|
||||
[in the next section](clients.md).
|
||||
Configuring S3-compatible applicatiosn to interact with Garage
|
||||
is covered in the [Integrations](/connect/index.html) section.
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
# Benchmarks
|
||||
|
||||
With Garage, we wanted to build a software defined storage service that follow the [KISS principle](https://en.wikipedia.org/wiki/KISS_principle),
|
||||
that is suitable for geo-distributed deployments and more generally that would work well for community hosting (like a Mastodon instance).
|
||||
|
||||
In our benchmarks, we aim to quantify how Garage performs on these goals compared to the other available solutions.
|
||||
|
||||
## Geo-distribution
|
||||
|
||||
The main challenge in a geo-distributed setup is latency between nodes of the cluster.
|
||||
The more a user request will require intra-cluster requests to complete, the more its latency will increase.
|
||||
This is especially true for sequential requests: requests that must wait the result of another request to be sent.
|
||||
We designed Garage without consensus algorithms (eg. Paxos or Raft) to minimize the number of sequential and parallel requests.
|
||||
|
||||
This serie of benchmarks quantifies the impact of this design choice.
|
||||
|
||||
### On a simple simulated network
|
||||
|
||||
We start with a controlled environment, all the instances are running on the same (powerful enough) machine.
|
||||
|
||||
To control the network latency, we simulate the network with [mknet](https://git.deuxfleurs.fr/trinity-1686a/mknet) (a tool we developped, based on `tc` and the linux network stack).
|
||||
To mesure S3 endpoints latency, we use our own tool [s3lat](https://git.deuxfleurs.fr/quentin/s3lat/) to observe only the intra-cluster latency and not some contention on the nodes (CPU, RAM, disk I/O, network bandwidth, etc.).
|
||||
Compared to other benchmark tools, S3Lat sends only one (small) request at the same time and measures its latency.
|
||||
We selected 5 standard endpoints that are often in the critical path: ListBuckets, ListObjects, GetObject, PutObject and RemoveObject.
|
||||
|
||||
In this first benchmark, we consider 5 instances that are located in a different place each. To simulate the distance, we configure mknet with a RTT between each node of 100 ms +/- 20 ms of jitter. We get the following graph, where the colored bars represent the mean latency while the error bars the minimum and maximum one:
|
||||
|
||||
|
||||
|
||||
Compared to garage, minio latency drastically increases on 3 endpoints: GetObject, PutObject, RemoveObject.
|
||||
|
||||
We suppose that these requests on minio make transactions over Raft, involving 4 sequential requests: 1) sending the message to the leader, 2) having the leader dispatch it to the other nodes, 3) waiting for the confirmation of followers and finally 4) commiting it. With our current configuration, one Raft transaction will take around 400 ms. GetObject seems to correlate to 1 transaction while PutObject and RemoveObject seems to correlate to 2 or 3. Reviewing minio code would be required to confirm this hypothesis.
|
||||
|
||||
Conversely, garage uses an architecture similar to DynamoDB and never require global cluster coordination to answer a request.
|
||||
Instead, garage can always contact the right node in charge of the requested data, and can answer in as low as one request in the case of GetObject and PutObject. We also observed that Garage latency, while often lower to minio, is more dispersed: garage is still in beta and has not received any performance optimization yet.
|
||||
|
||||
As a conclusion, Garage performs well in such setup while minio will be hard to use, especially for interactive use cases.
|
||||
|
||||
### On a complex simulated network
|
||||
|
||||
This time we consider a more heterogeneous network with 6 servers spread in 3 datacenter, giving us 2 servers per datacenters.
|
||||
We consider that intra-DC communications are now very cheap with a latency of 0.5ms and without any jitter.
|
||||
The inter-DC remains costly with the same value as before (100ms +/- 20ms of jitter).
|
||||
We plot a similar graph as before:
|
||||
|
||||
|
||||
|
||||
This new graph is very similar to the one before, neither minio or garage seems to benefit from this new topology, but they also do not suffer from it.
|
||||
|
||||
Considering garage, this is expected: nodes in the same DC are put in the same zone, and then data are spread on different zones for data resiliency and availaibility.
|
||||
Then, in the default mode, requesting data requires to query at least 2 zones to be sure that we have the most up to date information.
|
||||
These requests will involve at least one inter-DC communication.
|
||||
In other words, we prioritize data availability and synchronization over raw performances.
|
||||
|
||||
Minio's case is a bit different as by default a minio cluster is not location aware, so we can't explain its performances through location awareness.
|
||||
*We know that minio has a multi site mode but it is definitely not a first class citizen: data are asynchronously replicated from one minio cluster to another.*
|
||||
We suppose that, due to the consensus, for many of its requests minio will wait for a response of the majority of the server, also involving inter-DC communications.
|
||||
|
||||
As a conclusion, our new topology did not influence garage or minio performances, confirming that in presence of latency, garage is the best fit.
|
||||
|
||||
### On a real world deployment
|
||||
|
||||
*TODO*
|
||||
|
||||
|
||||
## Performance stability
|
||||
|
||||
A storage cluster will encounter different scenario over its life, many of them will not be predictable.
|
||||
In this context, we argue that, more than peak performances, we should seek predictable and stable performances to ensure data availability.
|
||||
|
||||
### Reference
|
||||
|
||||
*TODO*
|
||||
|
||||
### On a degraded cluster
|
||||
|
||||
*TODO*
|
||||
|
||||
### At scale
|
||||
|
||||
*TODO*
|
||||
@@ -0,0 +1,53 @@
|
||||
# Goals and use cases
|
||||
|
||||
## Goals and non-goals
|
||||
|
||||
Garage is a lightweight geo-distributed data store that implements the
|
||||
[Amazon S3](https://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html)
|
||||
object storage protocole. It enables applications to store large blobs such
|
||||
as pictures, video, images, documents, etc., in a redundant multi-node
|
||||
setting. S3 is versatile enough to also be used to publish a static
|
||||
website.
|
||||
|
||||
Garage is an opinionated object storage solutoin, we focus on the following **desirable properties**:
|
||||
|
||||
- **Self-contained & lightweight**: works everywhere and integrates well in existing environments to target [hyperconverged infrastructures](https://en.wikipedia.org/wiki/Hyper-converged_infrastructure).
|
||||
- **Highly resilient**: highly resilient to network failures, network latency, disk failures, sysadmin failures.
|
||||
- **Simple**: simple to understand, simple to operate, simple to debug.
|
||||
- **Internet enabled**: made for multi-sites (eg. datacenters, offices, households, etc.) interconnected through regular Internet connections.
|
||||
|
||||
We also noted that the pursuit of some other goals are detrimental to our initial goals.
|
||||
The following has been identified as **non-goals** (if these points matter to you, you should not use Garage):
|
||||
|
||||
- **Extreme performances**: high performances constrain a lot the design and the infrastructure; we seek performances through minimalism only.
|
||||
- **Feature extensiveness**: we do not plan to add additional features compared to the ones provided by the S3 API.
|
||||
- **Storage optimizations**: erasure coding or any other coding technique both increase the difficulty of placing data and synchronizing; we limit ourselves to duplication.
|
||||
- **POSIX/Filesystem compatibility**: we do not aim at being POSIX compatible or to emulate any kind of filesystem. Indeed, in a distributed environment, such synchronizations are translated in network messages that impose severe constraints on the deployment.
|
||||
|
||||
## Use-cases
|
||||
|
||||
*Are you also using Garage in your organization? [Open a PR](https://git.deuxfleurs.fr/Deuxfleurs/garage) to add your use case here!*
|
||||
|
||||
### Deuxfleurs
|
||||
|
||||
[Deuxfleurs](https://deuxfleurs.fr) is an experimental non-profit hosting
|
||||
organization that develops Garage. Deuxfleurs is focused on building highly
|
||||
available infrastructure through redundancy in multiple geographical
|
||||
locations. They use Garage themselves for the following tasks:
|
||||
|
||||
- Hosting of [main website](https://deuxfleurs.fr), [this website](https://garagehq.deuxfleurs.fr), as well as the personal website of many of the members of the organization
|
||||
|
||||
- As a [Matrix media backend](https://github.com/matrix-org/synapse-s3-storage-provider)
|
||||
|
||||
- To store personal data and shared documents through [Bagage](https://git.deuxfleurs.fr/Deuxfleurs/bagage), a homegrown WebDav-to-S3 proxy
|
||||
|
||||
- In the Drone continuous integration platform to store task logs
|
||||
|
||||
- As a Nix binary cache
|
||||
|
||||
- As a backup target using `rclone`
|
||||
|
||||
The Deuxfleurs Garage cluster is a multi-site cluster currently composed of
|
||||
4 nodes in 2 physical locations. In the future it will be expanded to at
|
||||
least 3 physical locations to fully exploit Garage's potential for high
|
||||
availability.
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 129 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 124 KiB |
@@ -1,5 +1,26 @@
|
||||
# Design
|
||||
|
||||
The design section helps you to see Garage from a "big picture" perspective.
|
||||
It will allow you to understand if Garage is a good fit for you,
|
||||
how to better use it, how to contribute to it, what can Garage could and could not do, etc.
|
||||
The design section helps you to see Garage from a "big picture"
|
||||
perspective. It will allow you to understand if Garage is a good fit for
|
||||
you, how to better use it, how to contribute to it, what can Garage could
|
||||
and could not do, etc.
|
||||
|
||||
- **[Goals and use cases](goals.md):** This page explains why Garage was concieved and what practical use cases it targets.
|
||||
|
||||
- **[Related work](related_work.md):** This pages presents the theoretical background on which Garage is built, and describes other software storage solutions and why they didn't work for us.
|
||||
|
||||
- **[Internals](internals.md):** This page enters into more details on how Garage manages data internally.
|
||||
|
||||
## Talks
|
||||
|
||||
We love to talk and hear about Garage, that's why we keep a log here:
|
||||
|
||||
- [(fr, 2021-11-13, video) Garage : Mille et une façons de stocker vos données](https://video.tedomum.net/w/moYKcv198dyMrT8hCS5jz9) and [slides (html)](https://rfid.deuxfleurs.fr/presentations/2021-11-13/garage/) - during [RFID#1](https://rfid.deuxfleurs.fr/programme/2021-11-13/) event
|
||||
|
||||
- [(en, 2021-04-28) Distributed object storage is centralised](https://git.deuxfleurs.fr/Deuxfleurs/garage/raw/commit/b1f60579a13d3c5eba7f74b1775c84639ea9b51a/doc/talks/2021-04-28_spirals-team/talk.pdf)
|
||||
|
||||
- [(fr, 2020-12-02) Garage : jouer dans la cour des grands quand on est un hébergeur associatif](https://git.deuxfleurs.fr/Deuxfleurs/garage/raw/commit/b1f60579a13d3c5eba7f74b1775c84639ea9b51a/doc/talks/2020-12-02_wide-team/talk.pdf)
|
||||
|
||||
*Did you write or talk about Garage? [Open a pull request](https://git.deuxfleurs.fr/Deuxfleurs/garage/) to add a link here!*
|
||||
|
||||
|
||||
|
||||
@@ -4,14 +4,17 @@
|
||||
|
||||
TODO: write this section
|
||||
|
||||
- The Dynamo ring
|
||||
- The Dynamo ring (see [this paper](https://dl.acm.org/doi/abs/10.1145/1323293.1294281) and [that paper](https://www.usenix.org/conference/nsdi16/technical-sessions/presentation/eisenbud))
|
||||
|
||||
- CRDTs
|
||||
- CRDTs (see [this paper](https://link.springer.com/chapter/10.1007/978-3-642-24550-3_29))
|
||||
|
||||
- Consistency model of Garage tables
|
||||
|
||||
See this presentation (in French) for some first information:
|
||||
<https://git.deuxfleurs.fr/Deuxfleurs/garage/src/branch/main/doc/talks/2020-12-02_wide-team/talk.pdf>
|
||||
In the meantime, you can find some information at the following links:
|
||||
|
||||
- [this presentation (in French)](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/branch/main/doc/talks/2020-12-02_wide-team/talk.pdf)
|
||||
|
||||
- [an old design draft](/working_documents/design_draft.md)
|
||||
|
||||
|
||||
## Garbage collection
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# Related Work
|
||||
# Related work
|
||||
|
||||
## Context
|
||||
|
||||
@@ -41,14 +41,35 @@ There were many attempts in research too. I am only thinking to [LBFS](https://p
|
||||
|
||||
## Existing software
|
||||
|
||||
**[Pithos](https://github.com/exoscale/pithos) :**
|
||||
**[MinIO](https://min.io/):** MinIO shares our *Self-contained & lightweight* goal but selected two of our non-goals: *Storage optimizations* through erasure coding and *POSIX/Filesystem compatibility* through strong consistency.
|
||||
However, by pursuing these two non-goals, MinIO do not reach our desirable properties.
|
||||
Firstly, it fails on the *Simple* property: due to the erasure coding, MinIO has severe limitations on how drives can be added or deleted from a cluster.
|
||||
Secondly, it fails on the *Internet enabled* property: due to its strong consistency, MinIO is latency sensitive.
|
||||
Furthermore, MinIO has no knowledge of "sites" and thus can not distribute data to minimize the failure of a given site.
|
||||
|
||||
**[Openstack Swift](https://docs.openstack.org/swift/latest/):**
|
||||
OpenStack Swift at least fails on the *Self-contained & lightweight* goal.
|
||||
Starting it requires around 8GB of RAM, which is too much especially in an hyperconverged infrastructure.
|
||||
We also do not classify Swift as *Simple*.
|
||||
|
||||
**[Ceph](https://ceph.io/ceph-storage/object-storage/):**
|
||||
This review holds for the whole Ceph stack, including the RADOS paper, Ceph Object Storage module, the RADOS Gateway, etc.
|
||||
At its core, Ceph has been designed to provide *POSIX/Filesystem compatibility* which requires strong consistency, which in turn
|
||||
makes Ceph latency-sensitive and fails our *Internet enabled* goal.
|
||||
Due to its industry oriented design, Ceph is also far from being *Simple* to operate and from being *Self-contained & lightweight* which makes it hard to integrate it in an hyperconverged infrastructure.
|
||||
In a certain way, Ceph and MinIO are closer together than they are from Garage or OpenStack Swift.
|
||||
|
||||
**[Pithos](https://github.com/exoscale/pithos):**
|
||||
Pithos has been abandonned and should probably not used yet, in the following we explain why we did not pick their design.
|
||||
Pithos was relying as a S3 proxy in front of Cassandra (and was working with Scylla DB too).
|
||||
From its designers' mouth, storing data in Cassandra has shown its limitations justifying the project abandonment.
|
||||
They built a closed-source version 2 that does not store blobs in the database (only metadata) but did not communicate further on it.
|
||||
We considered there v2's design but concluded that it does not fit both our *Self-contained & lightweight* and *Simple* properties. It makes the development, the deployment and the operations more complicated while reducing the flexibility.
|
||||
|
||||
**[IPFS](https://ipfs.io/) :**
|
||||
**[Riak CS](https://docs.riak.com/riak/cs/2.1.1/index.html):**
|
||||
*Not written yet*
|
||||
|
||||
**[IPFS](https://ipfs.io/):**
|
||||
*Not written yet*
|
||||
|
||||
## Specific research papers
|
||||
|
||||
@@ -2,3 +2,13 @@
|
||||
|
||||
Now that you are a Garage expert, you want to enhance it, you are in the right place!
|
||||
We discuss here how to hack on Garage, how we manage its development, etc.
|
||||
|
||||
## Rust API (docs.rs)
|
||||
If you encounter a specific bug in Garage or plan to patch it, you may jump directly to the source code's documentation!
|
||||
|
||||
- [garage\_api](https://docs.rs/garage_api/latest/garage_api/) - contains the S3 standard API endpoint
|
||||
- [garage\_model](https://docs.rs/garage_model/latest/garage_model/) - contains Garage's model built on the table abstraction
|
||||
- [garage\_rpc](https://docs.rs/garage_rpc/latest/garage_rpc/) - contains Garage's federation protocol
|
||||
- [garage\_table](https://docs.rs/garage_table/latest/garage_table/) - contains core Garage's CRDT datatypes
|
||||
- [garage\_util](https://docs.rs/garage_util/latest/garage_util/) - contains garage helpers
|
||||
- [garage\_web](https://docs.rs/garage_web/latest/garage_web/) - contains the S3 website endpoint
|
||||
|
||||
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 1.4 MiB |
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 310 KiB |
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 35 KiB |
+38
-69
@@ -12,91 +12,62 @@
|
||||
]
|
||||
</p>
|
||||
|
||||
```
|
||||
This very website is hosted using Garage. In other words: the doc is the PoC!
|
||||
```
|
||||
|
||||
# The Garage Geo-Distributed Data Store
|
||||
# Data resiliency for everyone
|
||||
|
||||
Garage is a lightweight geo-distributed data store that implements the
|
||||
[Amazon S3](https://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html)
|
||||
object storage protocole. It enables applications to store large blobs such
|
||||
as pictures, video, images, documents, etc., in a redundant multi-node
|
||||
setting. S3 is versatile enough to also be used to publish a static
|
||||
website.
|
||||
Garage is an **open-source** distributed **storage service** you can **self-host** to fullfill many needs:
|
||||
|
||||
Garage comes from the observation that despite the numerous existing
|
||||
implementation of object stores, many people have broken data management
|
||||
policies (backup/replication on a single site or none at all). To promote
|
||||
better data management policies, we focused on the following **desirable
|
||||
properties**:
|
||||
<p align="center" style="text-align:center; margin-bottom: 5rem;">
|
||||
<img alt="Summary of the possible usages with a related icon: host a website, store media and backup target" src="img/usage.svg" />
|
||||
</p>
|
||||
|
||||
- **Self-contained & lightweight**: works everywhere and integrates well in existing environments to target [hyperconverged infrastructures](https://en.wikipedia.org/wiki/Hyper-converged_infrastructure).
|
||||
- **Highly resilient**: highly resilient to network failures, network latency, disk failures, sysadmin failures.
|
||||
- **Simple**: simple to understand, simple to operate, simple to debug.
|
||||
- **Internet enabled**: made for multi-sites (eg. datacenters, offices, households, etc.) interconnected through regular Internet connections.
|
||||
<p align="center" style="text-align:center; margin-bottom: 5rem;">
|
||||
<a href="/design/goals.html#use-cases">⮞ learn more about use cases ⮜</a>
|
||||
</p>
|
||||
|
||||
We also noted that the pursuit of some other goals are detrimental to our initial goals.
|
||||
The following has been identified as **non-goals** (if these points matter to you, you should not use Garage):
|
||||
Garage implements the **[Amazon S3 API](https://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html)** and thus is already **compatible** with many applications:
|
||||
|
||||
- **Extreme performances**: high performances constrain a lot the design and the infrastructure; we seek performances through minimalism only.
|
||||
- **Feature extensiveness**: complete implementation of the S3 API or any other API to make Garage a drop-in replacement is not targeted as it could lead to decisions impacting our desirable properties.
|
||||
- **Storage optimizations**: erasure coding or any other coding technique both increase the difficulty of placing data and synchronizing; we limit ourselves to duplication.
|
||||
- **POSIX/Filesystem compatibility**: we do not aim at being POSIX compatible or to emulate any kind of filesystem. Indeed, in a distributed environment, such synchronizations are translated in network messages that impose severe constraints on the deployment.
|
||||
<p align="center" style="text-align:center; margin-bottom: 8rem;">
|
||||
<img alt="Garage is already compatible with Nextcloud, Mastodon, Matrix Synapse, Cyberduck, RClone and Peertube" src="img/software.svg" />
|
||||
</p>
|
||||
|
||||
## Use Cases
|
||||
<p align="center" style="text-align:center; margin-bottom: 5rem;">
|
||||
<a href="/connect/index.html">⮞ learn more about integrations ⮜</a>
|
||||
</p>
|
||||
|
||||
**[Deuxfleurs](https://deuxfleurs.fr):** Garage is used by Deuxfleurs which
|
||||
is a non-profit hosting organization. Especially, it is used to host their
|
||||
main website, this documentation and some of its members' blogs.
|
||||
Deuxfleurs also uses Garage as their [Matrix's media
|
||||
backend](https://github.com/matrix-org/synapse-s3-storage-provider).
|
||||
Deuxfleurs also uses it in its continuous integration platform to store
|
||||
Drone's job logs and a Nix binary cache.
|
||||
|
||||
*Are you using Garage? [Open a pull request](https://git.deuxfleurs.fr/Deuxfleurs/garage/) to add your organization here!*
|
||||
Garage provides **data resiliency** by **replicating** data 3x over **distant** servers:
|
||||
|
||||
## Comparison to existing software
|
||||
<p align="center" style="text-align:center; margin-bottom: 5rem;">
|
||||
<img alt="An example deployment on a map with servers in 5 zones: UK, France, Belgium, Germany and Switzerland. Each chunk of data is replicated in 3 of these 5 zones." src="img/map.svg" />
|
||||
</p>
|
||||
|
||||
**[MinIO](https://min.io/):** MinIO shares our *Self-contained & lightweight* goal but selected two of our non-goals: *Storage optimizations* through erasure coding and *POSIX/Filesystem compatibility* through strong consistency.
|
||||
However, by pursuing these two non-goals, MinIO do not reach our desirable properties.
|
||||
Firstly, it fails on the *Simple* property: due to the erasure coding, MinIO has severe limitations on how drives can be added or deleted from a cluster.
|
||||
Secondly, it fails on the *Internet enabled* property: due to its strong consistency, MinIO is latency sensitive.
|
||||
Furthermore, MinIO has no knowledge of "sites" and thus can not distribute data to minimize the failure of a given site.
|
||||
<p align="center" style="text-align:center; margin-bottom: 5rem;">
|
||||
<a href="/design/index.html">⮞ learn more about our design ⮜</a>
|
||||
</p>
|
||||
|
||||
**[Openstack Swift](https://docs.openstack.org/swift/latest/):**
|
||||
OpenStack Swift at least fails on the *Self-contained & lightweight* goal.
|
||||
Starting it requires around 8GB of RAM, which is too much especially in an hyperconverged infrastructure.
|
||||
We also do not classify Swift as *Simple*.
|
||||
Did you notice that *this website* is hosted and served by Garage?
|
||||
|
||||
**[Ceph](https://ceph.io/ceph-storage/object-storage/):**
|
||||
This review holds for the whole Ceph stack, including the RADOS paper, Ceph Object Storage module, the RADOS Gateway, etc.
|
||||
At its core, Ceph has been designed to provide *POSIX/Filesystem compatibility* which requires strong consistency, which in turn
|
||||
makes Ceph latency-sensitive and fails our *Internet enabled* goal.
|
||||
Due to its industry oriented design, Ceph is also far from being *Simple* to operate and from being *Self-contained & lightweight* which makes it hard to integrate it in an hyperconverged infrastructure.
|
||||
In a certain way, Ceph and MinIO are closer together than they are from Garage or OpenStack Swift.
|
||||
## Keeping requirements low
|
||||
|
||||
*More comparisons are available in our [Related Work](design/related_work.md) chapter.*
|
||||
We worked hard to keep requirements as low as possible as we target the largest possible public.
|
||||
|
||||
## Other Resources
|
||||
* **CPU:** any x86\_64 CPU from the last 10 years, ARMv7 or ARMv8.
|
||||
* **RAM:** 1GB
|
||||
* **Disk Space:** at least 16GB
|
||||
* **Network:** 200ms or less, 50 Mbps or more
|
||||
* **Heterogeneous hardware:** build a cluster with whatever second-hand machines are available
|
||||
|
||||
This website is not the only source of information about Garage!
|
||||
We reference here other places on the Internet where you can learn more about Garage.
|
||||
*For the network, as we do not use consensus algorithms like Paxos or Raft, Garage is not as latency sensitive.*
|
||||
*Thanks to Rust and its zero-cost abstractions, we keep CPU and memory low.*
|
||||
|
||||
### Rust API (docs.rs)
|
||||
## Built on the shoulder of giants
|
||||
|
||||
If you encounter a specific bug in Garage or plan to patch it, you may jump directly to the source code's documentation!
|
||||
- [Dynamo: Amazon’s Highly Available Key-value Store ](https://dl.acm.org/doi/abs/10.1145/1323293.1294281) by DeCandia et al.
|
||||
- [Conflict-Free Replicated Data Types](https://link.springer.com/chapter/10.1007/978-3-642-24550-3_29) by Shapiro et al.
|
||||
- [Maglev: A Fast and Reliable Software Network Load Balancer](https://www.usenix.org/conference/nsdi16/technical-sessions/presentation/eisenbud) by Eisenbud et al.
|
||||
|
||||
- [garage\_api](https://docs.rs/garage_api/latest/garage_api/) - contains the S3 standard API endpoint
|
||||
- [garage\_model](https://docs.rs/garage_model/latest/garage_model/) - contains Garage's model built on the table abstraction
|
||||
- [garage\_rpc](https://docs.rs/garage_rpc/latest/garage_rpc/) - contains Garage's federation protocol
|
||||
- [garage\_table](https://docs.rs/garage_table/latest/garage_table/) - contains core Garage's CRDT datatypes
|
||||
- [garage\_util](https://docs.rs/garage_util/latest/garage_util/) - contains garage helpers
|
||||
- [garage\_web](https://docs.rs/garage_web/latest/garage_web/) - contains the S3 website endpoint
|
||||
|
||||
### Talks
|
||||
|
||||
We love to talk and hear about Garage, that's why we keep a log here:
|
||||
## Talks
|
||||
|
||||
- [(fr, 2021-11-13, video) Garage : Mille et une façons de stocker vos données](https://video.tedomum.net/w/moYKcv198dyMrT8hCS5jz9) and [slides (html)](https://rfid.deuxfleurs.fr/presentations/2021-11-13/garage/) - during [RFID#1](https://rfid.deuxfleurs.fr/programme/2021-11-13/) event
|
||||
|
||||
@@ -104,8 +75,6 @@ We love to talk and hear about Garage, that's why we keep a log here:
|
||||
|
||||
- [(fr, 2020-12-02, pdf) Garage : jouer dans la cour des grands quand on est un hébergeur associatif](https://git.deuxfleurs.fr/Deuxfleurs/garage/raw/commit/b1f60579a13d3c5eba7f74b1775c84639ea9b51a/doc/talks/2020-12-02_wide-team/talk.pdf)
|
||||
|
||||
*Did you write or talk about Garage? [Open a pull request](https://git.deuxfleurs.fr/Deuxfleurs/garage/) to add a link here!*
|
||||
|
||||
## Community
|
||||
|
||||
If you want to discuss with us, you can join our Matrix channel at [#garage:deuxfleurs.fr](https://matrix.to/#/#garage:deuxfleurs.fr).
|
||||
@@ -116,7 +85,7 @@ Our code repository and issue tracker, which is the place where you should repor
|
||||
Garage's source code, is released under the [AGPL v3 License](https://www.gnu.org/licenses/agpl-3.0.en.html).
|
||||
Please note that if you patch Garage and then use it to provide any service over a network, you must share your code!
|
||||
|
||||
# Funding
|
||||
# Sponsors and funding
|
||||
|
||||
The Deuxfleurs association has received a grant from [NGI POINTER](https://pointer.ngi.eu/), to fund 3 people working on Garage full-time for a year: from October 2021 to September 2022.
|
||||
|
||||
|
||||
@@ -10,6 +10,8 @@ block_size = 1048576
|
||||
|
||||
replication_mode = "3"
|
||||
|
||||
compression_level = 1
|
||||
|
||||
rpc_secret = "4425f5c26c5e11581d3223904324dcb5b5d5dfb14e5e7f35e38c595424f5f1e6"
|
||||
rpc_bind_addr = "[::]:3901"
|
||||
rpc_public_addr = "[fc00:1::1]:3901"
|
||||
@@ -98,6 +100,30 @@ Never run a Garage cluster where that is not the case.**
|
||||
Changing the `replication_mode` of a cluster might work (make sure to shut down all nodes
|
||||
and changing it everywhere at the time), but is not officially supported.
|
||||
|
||||
### `compression_level`
|
||||
|
||||
Zstd compression level to use for storing blocks.
|
||||
|
||||
Values between `1` (faster compression) and `19` (smaller file) are standard compression
|
||||
levels for zstd. From `20` to `22`, compression levels are referred as "ultra" and must be
|
||||
used with extra care as it will use lot of memory. A value of `0` will let zstd choose a
|
||||
default value (currently `3`). Finally, zstd has also compression designed to be faster
|
||||
than default compression levels, they range from `-1` (smaller file) to `-99` (faster
|
||||
compression).
|
||||
|
||||
If you do not specify a `compression_level` entry, garage will set it to `1` for you. With
|
||||
this parameters, zstd consumes low amount of cpu and should work faster than line speed in
|
||||
most situations, while saving some space and intra-cluster
|
||||
bandwidth.
|
||||
|
||||
If you want to totally deactivate zstd in garage, you can pass the special value `'none'`. No
|
||||
zstd related code will be called, your chunks will be stored on disk without any processing.
|
||||
|
||||
Compression is done synchronously, setting a value too high will add latency to write queries.
|
||||
|
||||
This value can be different between nodes, compression is done by the node which receive the
|
||||
API call.
|
||||
|
||||
#### `rpc_secret`
|
||||
|
||||
Garage uses a secret key that is shared between all nodes of the cluster
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
Implemented:
|
||||
|
||||
- path-style URLs (`garage.tld/bucket/key`)
|
||||
- vhost-style URLs (`bucket.garage.tld/key`)
|
||||
- putting and getting objects in buckets
|
||||
- multipart uploads
|
||||
- listing objects
|
||||
@@ -12,9 +13,8 @@ Implemented:
|
||||
|
||||
Not implemented:
|
||||
|
||||
- vhost-style URLs (`bucket.garage.tld/key`)
|
||||
- object-level ACL
|
||||
- object versioning
|
||||
- [object versioning](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/166)
|
||||
- encryption
|
||||
- most `x-amz-` headers
|
||||
|
||||
@@ -31,10 +31,12 @@ All APIs that are not mentionned are not implemented and will return a 400 bad r
|
||||
| CreateBucket | Unsupported, stub (see below) |
|
||||
| CreateMultipartUpload | Implemented |
|
||||
| DeleteBucket | Unsupported (see below) |
|
||||
| DeleteBucketWebsite | Implemented |
|
||||
| DeleteObject | Implemented |
|
||||
| DeleteObjects | Implemented |
|
||||
| GetBucketLocation | Implemented |
|
||||
| GetBucketVersioning | Stub (see below) |
|
||||
| GetBucketWebsite | Unsupported |
|
||||
| GetObject | Implemented |
|
||||
| HeadBucket | Implemented |
|
||||
| HeadObject | Implemented |
|
||||
@@ -42,6 +44,7 @@ All APIs that are not mentionned are not implemented and will return a 400 bad r
|
||||
| ListObjects | Implemented, bugs? (see below) |
|
||||
| ListObjectsV2 | Implemented |
|
||||
| PutObject | Implemented |
|
||||
| PutBucketWebsite | Partially implemented (see below)|
|
||||
| UploadPart | Implemented |
|
||||
|
||||
|
||||
@@ -55,3 +58,6 @@ All APIs that are not mentionned are not implemented and will return a 400 bad r
|
||||
|
||||
- **ListObjects:** Implemented, but there isn't a very good specification of what `encoding-type=url` covers so there might be some encoding bugs. In our implementation the url-encoded fields are in the same in ListObjects as they are in ListObjectsV2.
|
||||
|
||||
- **PutBucketWebsite:** Implemented, but only store if website is enabled, not more complexe informations.
|
||||
|
||||
- **GetBucketWebsite:** Not implemented yet, will be when PubBucketWebsite store more informations.
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
# S3 compatibility target
|
||||
|
||||
If there is a specific S3 functionnality you have a need for, feel free to open
|
||||
a PR to put the corresponding endpoints higher in the list. Please explain
|
||||
your motivations for doing so in the PR message.
|
||||
|
||||
| Priority | Endpoints |
|
||||
| -------------------------- | --------- |
|
||||
| **S-tier** (high priority) | |
|
||||
| | HeadBucket |
|
||||
| | GetBucketLocation |
|
||||
| | [*CreateBucket*](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/97) |
|
||||
| | [*DeleteBucket*](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/97) |
|
||||
| | ListBuckets |
|
||||
| | ListObjects |
|
||||
| | ListObjectsV2 |
|
||||
| | HeadObject |
|
||||
| | GetObject |
|
||||
| | PutObject |
|
||||
| | CopyObject |
|
||||
| | DeleteObject |
|
||||
| | DeleteObjects |
|
||||
| | CreateMultipartUpload |
|
||||
| | CompleteMultipartUpload |
|
||||
| | AbortMultipartUpload |
|
||||
| | UploadPart |
|
||||
| | [*ListMultipartUploads*](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/103) |
|
||||
| | [*ListParts*](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/103) |
|
||||
| **A-tier** (will implement) | |
|
||||
| | [*GetBucketCors*](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/138) |
|
||||
| | [*PutBucketCors*](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/138) |
|
||||
| | [*DeleteBucketCors*](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/138) |
|
||||
| | [*UploadPartCopy*](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/160) |
|
||||
| | [*GetBucketWebsite*](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/77) |
|
||||
| | [*PutBucketWebsite*](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/77) |
|
||||
| | DeleteBucketWebsite |
|
||||
| ~~~~~~~~~~~~~~~~~~~~~~~~~~ | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
|
||||
| **B-tier** | |
|
||||
| | GetBucketAcl |
|
||||
| | PutBucketAcl |
|
||||
| | GetObjectLockConfiguration |
|
||||
| | PutObjectLockConfiguration |
|
||||
| | GetObjectRetention |
|
||||
| | PutObjectRetention |
|
||||
| | GetObjectLegalHold |
|
||||
| | PutObjectLegalHold |
|
||||
| **C-tier** | |
|
||||
| | GetBucketVersioning |
|
||||
| | PutBucketVersioning |
|
||||
| | ListObjectVersions |
|
||||
| | GetObjectAcl |
|
||||
| | PutObjectAcl |
|
||||
| | GetBucketLifecycleConfiguration |
|
||||
| | PutBucketLifecycleConfiguration |
|
||||
| | DeleteBucketLifecycle |
|
||||
| **garbage-tier** | |
|
||||
| | DeleteBucketEncryption |
|
||||
| | DeleteBucketAnalyticsConfiguration |
|
||||
| | DeleteBucketIntelligentTieringConfiguration |
|
||||
| | DeleteBucketInventoryConfiguration |
|
||||
| | DeleteBucketMetricsConfiguration |
|
||||
| | DeleteBucketOwnershipControls |
|
||||
| | DeleteBucketPolicy |
|
||||
| | DeleteBucketReplication |
|
||||
| | DeleteBucketTagging |
|
||||
| | DeleteObjectTagging |
|
||||
| | DeletePublicAccessBlock |
|
||||
| | GetBucketAccelerateConfiguration |
|
||||
| | GetBucketAnalyticsConfiguration |
|
||||
| | GetBucketEncryption |
|
||||
| | GetBucketIntelligentTieringConfiguration |
|
||||
| | GetBucketInventoryConfiguration |
|
||||
| | GetBucketLogging |
|
||||
| | GetBucketMetricsConfiguration |
|
||||
| | GetBucketNotificationConfiguration |
|
||||
| | GetBucketOwnershipControls |
|
||||
| | GetBucketPolicy |
|
||||
| | GetBucketPolicyStatus |
|
||||
| | GetBucketReplication |
|
||||
| | GetBucketRequestPayment |
|
||||
| | GetBucketTagging |
|
||||
| | GetObjectTagging |
|
||||
| | GetObjectTorrent |
|
||||
| | GetPublicAccessBlock |
|
||||
| | ListBucketAnalyticsConfigurations |
|
||||
| | ListBucketIntelligentTieringConfigurations |
|
||||
| | ListBucketInventoryConfigurations |
|
||||
| | ListBucketMetricsConfigurations |
|
||||
| | PutBucketAccelerateConfiguration |
|
||||
| | PutBucketAnalyticsConfiguration |
|
||||
| | PutBucketEncryption |
|
||||
| | PutBucketIntelligentTieringConfiguration |
|
||||
| | PutBucketInventoryConfiguration |
|
||||
| | PutBucketLogging |
|
||||
| | PutBucketMetricsConfiguration |
|
||||
| | PutBucketNotificationConfiguration |
|
||||
| | PutBucketOwnershipControls |
|
||||
| | PutBucketPolicy |
|
||||
| | PutBucketReplication |
|
||||
| | PutBucketRequestPayment |
|
||||
| | PutBucketTagging |
|
||||
| | PutObjectTagging |
|
||||
| | PutPublicAccessBlock |
|
||||
| | RestoreObject |
|
||||
| | SelectObjectContent |
|
||||
-83
@@ -1,83 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -xe
|
||||
|
||||
cd $(dirname $0)
|
||||
|
||||
mkdir -p pki
|
||||
cd pki
|
||||
|
||||
# Create a certificate authority that both the client side and the server side of
|
||||
# the RPC protocol will use to authenticate the other side.
|
||||
if [ ! -f garage-ca.key ]; then
|
||||
echo "Generating Garage CA keys..."
|
||||
openssl genpkey -algorithm ED25519 -out garage-ca.key
|
||||
openssl req -x509 -new -nodes -key garage-ca.key -sha256 -days 3650 -out garage-ca.crt -subj "/C=FR/O=Garage"
|
||||
fi
|
||||
|
||||
|
||||
# Generate a certificate that can be used either as a server certificate
|
||||
# or a client certificate. This is what the RPC client and server will use
|
||||
# to prove that they are authenticated by the CA.
|
||||
if [ ! -f garage.crt ]; then
|
||||
echo "Generating Garage agent keys..."
|
||||
if [ ! -f garage.key ]; then
|
||||
openssl genpkey -algorithm ED25519 -out garage.key
|
||||
fi
|
||||
openssl req -new -sha256 -key garage.key -subj "/C=FR/O=Garage/CN=garage" \
|
||||
-out garage.csr
|
||||
openssl req -in garage.csr -noout -text
|
||||
openssl x509 -req -in garage.csr \
|
||||
-extensions v3_req \
|
||||
-extfile <(cat <<EOF
|
||||
[req]
|
||||
distinguished_name = req_distinguished_name
|
||||
req_extensions = v3_req
|
||||
prompt = no
|
||||
|
||||
[req_distinguished_name]
|
||||
C = FR
|
||||
O = Garage
|
||||
CN = garage
|
||||
|
||||
[v3_req]
|
||||
keyUsage = keyEncipherment, dataEncipherment
|
||||
extendedKeyUsage = serverAuth, clientAuth
|
||||
subjectAltName = @alt_names
|
||||
[alt_names]
|
||||
DNS.1 = garage
|
||||
EOF
|
||||
) \
|
||||
-CA garage-ca.crt -CAkey garage-ca.key -CAcreateserial \
|
||||
-out garage.crt -days 365
|
||||
fi
|
||||
|
||||
# Client-only certificate used for the CLI
|
||||
if [ ! -f garage-client.crt ]; then
|
||||
echo "Generating Garage client keys..."
|
||||
if [ ! -f garage-client.key ]; then
|
||||
openssl genpkey -algorithm ED25519 -out garage-client.key
|
||||
fi
|
||||
openssl req -new -sha256 -key garage-client.key -subj "/C=FR/O=Garage" \
|
||||
-out garage-client.csr
|
||||
openssl req -in garage-client.csr -noout -text
|
||||
openssl x509 -req -in garage-client.csr \
|
||||
-extensions v3_req \
|
||||
-extfile <(cat <<EOF
|
||||
[req]
|
||||
distinguished_name = req_distinguished_name
|
||||
req_extensions = v3_req
|
||||
prompt = no
|
||||
|
||||
[req_distinguished_name]
|
||||
C = FR
|
||||
O = Garage
|
||||
|
||||
[v3_req]
|
||||
keyUsage = keyEncipherment, dataEncipherment
|
||||
extendedKeyUsage = clientAuth
|
||||
EOF
|
||||
) \
|
||||
-CA garage-ca.crt -CAkey garage-ca.key -CAcreateserial \
|
||||
-out garage-client.crt -days 365
|
||||
fi
|
||||
@@ -47,6 +47,10 @@ EOF
|
||||
|
||||
echo -en "$LABEL configuration written to $CONF_PATH\n"
|
||||
|
||||
(garage -c /tmp/config.$count.toml server 2>&1|while read r; do echo -en "$LABEL $r\n"; done) &
|
||||
done
|
||||
# >>>>>>>>>>>>>>>> END FOR LOOP ON NODES
|
||||
|
||||
if [ -z "$SKIP_HTTPS" ]; then
|
||||
echo -en "$LABEL Starting dummy HTTPS reverse proxy\n"
|
||||
mkdir -p /tmp/garagessl
|
||||
@@ -62,10 +66,6 @@ if [ -z "$SKIP_HTTPS" ]; then
|
||||
socat openssl-listen:4443,reuseaddr,fork,cert=/tmp/garagessl/test.pem,verify=0 tcp4-connect:localhost:3911 &
|
||||
fi
|
||||
|
||||
(garage -c /tmp/config.$count.toml server 2>&1|while read r; do echo -en "$LABEL $r\n"; done) &
|
||||
done
|
||||
# >>>>>>>>>>>>>>>> END FOR LOOP ON NODES
|
||||
|
||||
sleep 3
|
||||
# Establish connections between nodes
|
||||
for count in $(seq 1 3); do
|
||||
|
||||
+37
-17
@@ -30,6 +30,11 @@ dd if=/dev/urandom of=/tmp/garage.1.rnd bs=1k count=2 # No multipart, inline sto
|
||||
dd if=/dev/urandom of=/tmp/garage.2.rnd bs=1M count=5 # No multipart but file will be chunked
|
||||
dd if=/dev/urandom of=/tmp/garage.3.rnd bs=1M count=10 # by default, AWS starts using multipart at 8MB
|
||||
|
||||
# data of lower entropy, to test compression
|
||||
dd if=/dev/urandom bs=1k count=2 | base64 -w0 > /tmp/garage.1.b64
|
||||
dd if=/dev/urandom bs=1M count=5 | base64 -w0 > /tmp/garage.2.b64
|
||||
dd if=/dev/urandom bs=1M count=10 | base64 -w0 > /tmp/garage.3.b64
|
||||
|
||||
echo "🧪 S3 API testing..."
|
||||
|
||||
# AWS
|
||||
@@ -37,11 +42,11 @@ if [ -z "$SKIP_AWS" ]; then
|
||||
echo "🛠️ Testing with awscli"
|
||||
source ${SCRIPT_FOLDER}/dev-env-aws.sh
|
||||
aws s3 ls
|
||||
for idx in $(seq 1 3); do
|
||||
aws s3 cp "/tmp/garage.$idx.rnd" "s3://eprouvette/&+-é\"/garage.$idx.aws"
|
||||
for idx in {1..3}.{rnd,b64}; do
|
||||
aws s3 cp "/tmp/garage.$idx" "s3://eprouvette/&+-é\"/garage.$idx.aws"
|
||||
aws s3 ls s3://eprouvette
|
||||
aws s3 cp "s3://eprouvette/&+-é\"/garage.$idx.aws" "/tmp/garage.$idx.dl"
|
||||
diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
|
||||
diff /tmp/garage.$idx /tmp/garage.$idx.dl
|
||||
rm /tmp/garage.$idx.dl
|
||||
aws s3 rm "s3://eprouvette/&+-é\"/garage.$idx.aws"
|
||||
done
|
||||
@@ -52,11 +57,11 @@ if [ -z "$SKIP_S3CMD" ]; then
|
||||
echo "🛠️ Testing with s3cmd"
|
||||
source ${SCRIPT_FOLDER}/dev-env-s3cmd.sh
|
||||
s3cmd ls
|
||||
for idx in $(seq 1 3); do
|
||||
s3cmd put "/tmp/garage.$idx.rnd" "s3://eprouvette/&+-é\"/garage.$idx.s3cmd"
|
||||
for idx in {1..3}.{rnd,b64}; do
|
||||
s3cmd put "/tmp/garage.$idx" "s3://eprouvette/&+-é\"/garage.$idx.s3cmd"
|
||||
s3cmd ls s3://eprouvette
|
||||
s3cmd get "s3://eprouvette/&+-é\"/garage.$idx.s3cmd" "/tmp/garage.$idx.dl"
|
||||
diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
|
||||
diff /tmp/garage.$idx /tmp/garage.$idx.dl
|
||||
rm /tmp/garage.$idx.dl
|
||||
s3cmd rm "s3://eprouvette/&+-é\"/garage.$idx.s3cmd"
|
||||
done
|
||||
@@ -67,11 +72,11 @@ if [ -z "$SKIP_MC" ]; then
|
||||
echo "🛠️ Testing with mc (minio client)"
|
||||
source ${SCRIPT_FOLDER}/dev-env-mc.sh
|
||||
mc ls garage/
|
||||
for idx in $(seq 1 3); do
|
||||
mc cp "/tmp/garage.$idx.rnd" "garage/eprouvette/&+-é\"/garage.$idx.mc"
|
||||
for idx in {1..3}.{rnd,b64}; do
|
||||
mc cp "/tmp/garage.$idx" "garage/eprouvette/&+-é\"/garage.$idx.mc"
|
||||
mc ls garage/eprouvette
|
||||
mc cp "garage/eprouvette/&+-é\"/garage.$idx.mc" "/tmp/garage.$idx.dl"
|
||||
diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
|
||||
diff /tmp/garage.$idx /tmp/garage.$idx.dl
|
||||
rm /tmp/garage.$idx.dl
|
||||
mc rm "garage/eprouvette/&+-é\"/garage.$idx.mc"
|
||||
done
|
||||
@@ -82,13 +87,13 @@ if [ -z "$SKIP_RCLONE" ]; then
|
||||
echo "🛠️ Testing with rclone"
|
||||
source ${SCRIPT_FOLDER}/dev-env-rclone.sh
|
||||
rclone lsd garage:
|
||||
for idx in $(seq 1 3); do
|
||||
cp /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
|
||||
for idx in {1..3}.{rnd,b64}; do
|
||||
cp /tmp/garage.$idx /tmp/garage.$idx.dl
|
||||
rclone copy "/tmp/garage.$idx.dl" "garage:eprouvette/&+-é\"/"
|
||||
rm /tmp/garage.$idx.dl
|
||||
rclone ls garage:eprouvette
|
||||
rclone copy "garage:eprouvette/&+-é\"/garage.$idx.dl" "/tmp/"
|
||||
diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
|
||||
diff /tmp/garage.$idx /tmp/garage.$idx.dl
|
||||
rm /tmp/garage.$idx.dl
|
||||
rclone delete "garage:eprouvette/&+-é\"/garage.$idx.dl"
|
||||
done
|
||||
@@ -100,17 +105,17 @@ if [ -z "$SKIP_DUCK" ]; then
|
||||
source ${SCRIPT_FOLDER}/dev-env-duck.sh
|
||||
duck --list garage:/
|
||||
duck --mkdir "garage:/eprouvette/duck"
|
||||
for idx in $(seq 1 3); do
|
||||
duck --verbose --upload "garage:/eprouvette/duck/" "/tmp/garage.$idx.rnd"
|
||||
for idx in {1..3}.{rnd,b64}; do
|
||||
duck --verbose --upload "garage:/eprouvette/duck/" "/tmp/garage.$idx"
|
||||
duck --list garage:/eprouvette/duck/
|
||||
duck --download "garage:/eprouvette/duck/garage.$idx.rnd" "/tmp/garage.$idx.dl"
|
||||
diff /tmp/garage.$idx.rnd /tmp/garage.$idx.dl
|
||||
duck --download "garage:/eprouvette/duck/garage.$idx" "/tmp/garage.$idx.dl"
|
||||
diff /tmp/garage.$idx /tmp/garage.$idx.dl
|
||||
rm /tmp/garage.$idx.dl
|
||||
duck --delete "garage:/eprouvette/duck/garage.$idx.dk"
|
||||
done
|
||||
fi
|
||||
|
||||
rm /tmp/garage.{1,2,3}.rnd
|
||||
rm /tmp/garage.{1..3}.{rnd,b64}
|
||||
|
||||
if [ -z "$SKIP_AWS" ]; then
|
||||
echo "🧪 Website Testing"
|
||||
@@ -125,6 +130,21 @@ if [ -z "$SKIP_AWS" ]; then
|
||||
rm /tmp/garage-index.html
|
||||
fi
|
||||
|
||||
if [ -z "$SKIP_AWS" ]; then
|
||||
echo "🪣 Test bucket logic "
|
||||
AWS_ACCESS_KEY_ID=`cat /tmp/garage.s3 |cut -d' ' -f1`
|
||||
[ $(aws s3 ls | wc -l) == 1 ]
|
||||
garage -c /tmp/config.1.toml bucket create seau
|
||||
garage -c /tmp/config.1.toml bucket allow --read seau --key $AWS_ACCESS_KEY_ID
|
||||
[ $(aws s3 ls | wc -l) == 2 ]
|
||||
garage -c /tmp/config.1.toml bucket deny --read seau --key $AWS_ACCESS_KEY_ID
|
||||
[ $(aws s3 ls | wc -l) == 1 ]
|
||||
garage -c /tmp/config.1.toml bucket allow --read seau --key $AWS_ACCESS_KEY_ID
|
||||
[ $(aws s3 ls | wc -l) == 2 ]
|
||||
garage -c /tmp/config.1.toml bucket delete --yes seau
|
||||
[ $(aws s3 ls | wc -l) == 1 ]
|
||||
fi
|
||||
|
||||
echo "🏁 Teardown"
|
||||
AWS_ACCESS_KEY_ID=`cat /tmp/garage.s3 |cut -d' ' -f1`
|
||||
AWS_SECRET_ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
|
||||
|
||||
+4
-4
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_api"
|
||||
version = "0.5.0"
|
||||
version = "0.5.1"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
@@ -14,9 +14,9 @@ path = "lib.rs"
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
garage_model = { version = "0.5.0", path = "../model" }
|
||||
garage_table = { version = "0.5.0", path = "../table" }
|
||||
garage_util = { version = "0.5.0", path = "../util" }
|
||||
garage_model = { version = "0.5.1", path = "../model" }
|
||||
garage_table = { version = "0.5.1", path = "../table" }
|
||||
garage_util = { version = "0.5.1", path = "../util" }
|
||||
|
||||
base64 = "0.13"
|
||||
bytes = "1.0"
|
||||
|
||||
+154
-154
@@ -1,4 +1,3 @@
|
||||
use std::collections::HashMap;
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -6,7 +5,7 @@ use futures::future::Future;
|
||||
use hyper::header;
|
||||
use hyper::server::conn::AddrStream;
|
||||
use hyper::service::{make_service_fn, service_fn};
|
||||
use hyper::{Body, Method, Request, Response, Server};
|
||||
use hyper::{Body, Request, Response, Server};
|
||||
|
||||
use garage_util::error::Error as GarageError;
|
||||
|
||||
@@ -22,6 +21,8 @@ use crate::s3_delete::*;
|
||||
use crate::s3_get::*;
|
||||
use crate::s3_list::*;
|
||||
use crate::s3_put::*;
|
||||
use crate::s3_router::{Authorization, Endpoint};
|
||||
use crate::s3_website::*;
|
||||
|
||||
/// Run the S3 API server
|
||||
pub async fn run_api_server(
|
||||
@@ -65,10 +66,15 @@ async fn handler(
|
||||
}
|
||||
Err(e) => {
|
||||
let body: Body = Body::from(e.aws_xml(&garage.config.s3_api.s3_region, uri.path()));
|
||||
let http_error = Response::builder()
|
||||
let mut http_error_builder = Response::builder()
|
||||
.status(e.http_status_code())
|
||||
.header("Content-Type", "application/xml")
|
||||
.body(body)?;
|
||||
.header("Content-Type", "application/xml");
|
||||
|
||||
if let Some(header_map) = http_error_builder.headers_mut() {
|
||||
e.add_headers(header_map)
|
||||
}
|
||||
|
||||
let http_error = http_error_builder.body(body)?;
|
||||
|
||||
if e.http_status_code().is_server_error() {
|
||||
warn!("Response: error {}, {}", e.http_status_code(), e);
|
||||
@@ -81,8 +87,6 @@ async fn handler(
|
||||
}
|
||||
|
||||
async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Response<Body>, Error> {
|
||||
let path = req.uri().path().to_string();
|
||||
let path = percent_encoding::percent_decode_str(&path).decode_utf8()?;
|
||||
let (api_key, content_sha256) = check_signature(&garage, &req).await?;
|
||||
|
||||
let authority = req
|
||||
@@ -100,166 +104,162 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
|
||||
.as_ref()
|
||||
.and_then(|root_domain| host_to_bucket(&host, root_domain));
|
||||
|
||||
if path == "/" && bucket.is_none() {
|
||||
return handle_list_buckets(&api_key);
|
||||
}
|
||||
|
||||
let (bucket, key) = parse_bucket_key(&path, bucket)?;
|
||||
let allowed = match req.method() {
|
||||
&Method::HEAD | &Method::GET => api_key.allow_read(bucket),
|
||||
_ => api_key.allow_write(bucket),
|
||||
let endpoint = Endpoint::from_request(&req, bucket.map(ToOwned::to_owned))?;
|
||||
let allowed = match endpoint.authorization_type() {
|
||||
Authorization::None => true,
|
||||
Authorization::Read(bucket) => api_key.allow_read(bucket),
|
||||
Authorization::Write(bucket) => api_key.allow_write(bucket),
|
||||
};
|
||||
|
||||
if !allowed {
|
||||
return Err(Error::Forbidden(
|
||||
"Operation is not allowed for this key.".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let mut params = HashMap::new();
|
||||
if let Some(query) = req.uri().query() {
|
||||
let query_pairs = url::form_urlencoded::parse(query.as_bytes());
|
||||
for (key, val) in query_pairs {
|
||||
params.insert(key.to_lowercase(), val.to_string());
|
||||
match endpoint {
|
||||
Endpoint::ListBuckets => handle_list_buckets(&api_key),
|
||||
Endpoint::HeadObject { bucket, key, .. } => handle_head(garage, &req, &bucket, &key).await,
|
||||
Endpoint::GetObject { bucket, key, .. } => handle_get(garage, &req, &bucket, &key).await,
|
||||
Endpoint::UploadPart {
|
||||
bucket,
|
||||
key,
|
||||
part_number,
|
||||
upload_id,
|
||||
} => {
|
||||
handle_put_part(
|
||||
garage,
|
||||
req,
|
||||
&bucket,
|
||||
&key,
|
||||
part_number,
|
||||
&upload_id,
|
||||
content_sha256,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(key) = key {
|
||||
match *req.method() {
|
||||
Method::HEAD => {
|
||||
// HeadObject query
|
||||
Ok(handle_head(garage, &req, bucket, key).await?)
|
||||
Endpoint::CopyObject { bucket, key } => {
|
||||
let copy_source = req.headers().get("x-amz-copy-source").unwrap().to_str()?;
|
||||
let copy_source = percent_encoding::percent_decode_str(copy_source).decode_utf8()?;
|
||||
let (source_bucket, source_key) = parse_bucket_key(©_source, None)?;
|
||||
if !api_key.allow_read(source_bucket) {
|
||||
return Err(Error::Forbidden(format!(
|
||||
"Reading from bucket {} not allowed for this key",
|
||||
source_bucket
|
||||
)));
|
||||
}
|
||||
Method::GET => {
|
||||
// GetObject query
|
||||
Ok(handle_get(garage, &req, bucket, key).await?)
|
||||
}
|
||||
Method::PUT => {
|
||||
if params.contains_key(&"partnumber".to_string())
|
||||
&& params.contains_key(&"uploadid".to_string())
|
||||
{
|
||||
// UploadPart query
|
||||
let part_number = params.get("partnumber").unwrap();
|
||||
let upload_id = params.get("uploadid").unwrap();
|
||||
Ok(handle_put_part(
|
||||
garage,
|
||||
req,
|
||||
let source_key = source_key.ok_or_bad_request("No source key specified")?;
|
||||
handle_copy(garage, &req, &bucket, &key, source_bucket, source_key).await
|
||||
}
|
||||
Endpoint::PutObject { bucket, key } => {
|
||||
handle_put(garage, req, &bucket, &key, content_sha256).await
|
||||
}
|
||||
Endpoint::AbortMultipartUpload {
|
||||
bucket,
|
||||
key,
|
||||
upload_id,
|
||||
} => handle_abort_multipart_upload(garage, &bucket, &key, &upload_id).await,
|
||||
Endpoint::DeleteObject { bucket, key, .. } => handle_delete(garage, &bucket, &key).await,
|
||||
Endpoint::CreateMultipartUpload { bucket, key } => {
|
||||
handle_create_multipart_upload(garage, &req, &bucket, &key).await
|
||||
}
|
||||
Endpoint::CompleteMultipartUpload {
|
||||
bucket,
|
||||
key,
|
||||
upload_id,
|
||||
} => {
|
||||
handle_complete_multipart_upload(garage, req, &bucket, &key, &upload_id, content_sha256)
|
||||
.await
|
||||
}
|
||||
Endpoint::CreateBucket { bucket } => {
|
||||
debug!(
|
||||
"Body: {}",
|
||||
std::str::from_utf8(&hyper::body::to_bytes(req.into_body()).await?)
|
||||
.unwrap_or("<invalid utf8>")
|
||||
);
|
||||
let empty_body: Body = Body::from(vec![]);
|
||||
let response = Response::builder()
|
||||
.header("Location", format!("/{}", bucket))
|
||||
.body(empty_body)
|
||||
.unwrap();
|
||||
Ok(response)
|
||||
}
|
||||
Endpoint::HeadBucket { .. } => {
|
||||
let empty_body: Body = Body::from(vec![]);
|
||||
let response = Response::builder().body(empty_body).unwrap();
|
||||
Ok(response)
|
||||
}
|
||||
Endpoint::DeleteBucket { .. } => Err(Error::Forbidden(
|
||||
"Cannot delete buckets using S3 api, please talk to Garage directly".into(),
|
||||
)),
|
||||
Endpoint::GetBucketLocation { .. } => handle_get_bucket_location(garage),
|
||||
Endpoint::GetBucketVersioning { .. } => handle_get_bucket_versioning(),
|
||||
Endpoint::ListObjects {
|
||||
bucket,
|
||||
delimiter,
|
||||
encoding_type,
|
||||
marker,
|
||||
max_keys,
|
||||
prefix,
|
||||
} => {
|
||||
handle_list(
|
||||
garage,
|
||||
&ListObjectsQuery {
|
||||
is_v2: false,
|
||||
bucket,
|
||||
delimiter: delimiter.map(|d| d.to_string()),
|
||||
max_keys: max_keys.unwrap_or(1000),
|
||||
prefix: prefix.unwrap_or_default(),
|
||||
marker,
|
||||
continuation_token: None,
|
||||
start_after: None,
|
||||
urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false),
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
Endpoint::ListObjectsV2 {
|
||||
bucket,
|
||||
delimiter,
|
||||
encoding_type,
|
||||
max_keys,
|
||||
prefix,
|
||||
continuation_token,
|
||||
start_after,
|
||||
list_type,
|
||||
..
|
||||
} => {
|
||||
if list_type == "2" {
|
||||
handle_list(
|
||||
garage,
|
||||
&ListObjectsQuery {
|
||||
is_v2: true,
|
||||
bucket,
|
||||
key,
|
||||
part_number,
|
||||
upload_id,
|
||||
content_sha256,
|
||||
)
|
||||
.await?)
|
||||
} else if req.headers().contains_key("x-amz-copy-source") {
|
||||
// CopyObject query
|
||||
let copy_source = req.headers().get("x-amz-copy-source").unwrap().to_str()?;
|
||||
let copy_source =
|
||||
percent_encoding::percent_decode_str(copy_source).decode_utf8()?;
|
||||
let (source_bucket, source_key) = parse_bucket_key(©_source, None)?;
|
||||
if !api_key.allow_read(source_bucket) {
|
||||
return Err(Error::Forbidden(format!(
|
||||
"Reading from bucket {} not allowed for this key",
|
||||
source_bucket
|
||||
)));
|
||||
}
|
||||
let source_key = source_key.ok_or_bad_request("No source key specified")?;
|
||||
Ok(handle_copy(garage, &req, bucket, key, source_bucket, source_key).await?)
|
||||
} else {
|
||||
// PutObject query
|
||||
Ok(handle_put(garage, req, bucket, key, content_sha256).await?)
|
||||
}
|
||||
delimiter: delimiter.map(|d| d.to_string()),
|
||||
max_keys: max_keys.unwrap_or(1000),
|
||||
prefix: prefix.unwrap_or_default(),
|
||||
marker: None,
|
||||
continuation_token,
|
||||
start_after,
|
||||
urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false),
|
||||
},
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
Err(Error::BadRequest(format!(
|
||||
"Invalid endpoint: list-type={}",
|
||||
list_type
|
||||
)))
|
||||
}
|
||||
Method::DELETE => {
|
||||
if params.contains_key(&"uploadid".to_string()) {
|
||||
// AbortMultipartUpload query
|
||||
let upload_id = params.get("uploadid").unwrap();
|
||||
Ok(handle_abort_multipart_upload(garage, bucket, key, upload_id).await?)
|
||||
} else {
|
||||
// DeleteObject query
|
||||
Ok(handle_delete(garage, bucket, key).await?)
|
||||
}
|
||||
}
|
||||
Method::POST => {
|
||||
if params.contains_key(&"uploads".to_string()) {
|
||||
// CreateMultipartUpload call
|
||||
Ok(handle_create_multipart_upload(garage, &req, bucket, key).await?)
|
||||
} else if params.contains_key(&"uploadid".to_string()) {
|
||||
// CompleteMultipartUpload call
|
||||
let upload_id = params.get("uploadid").unwrap();
|
||||
Ok(handle_complete_multipart_upload(
|
||||
garage,
|
||||
req,
|
||||
bucket,
|
||||
key,
|
||||
upload_id,
|
||||
content_sha256,
|
||||
)
|
||||
.await?)
|
||||
} else {
|
||||
Err(Error::BadRequest(
|
||||
"Not a CreateMultipartUpload call, what is it?".to_string(),
|
||||
))
|
||||
}
|
||||
}
|
||||
_ => Err(Error::BadRequest("Invalid method".to_string())),
|
||||
}
|
||||
} else {
|
||||
match *req.method() {
|
||||
Method::PUT => {
|
||||
// CreateBucket
|
||||
// If we're here, the bucket already exists, so just answer ok
|
||||
debug!(
|
||||
"Body: {}",
|
||||
std::str::from_utf8(&hyper::body::to_bytes(req.into_body()).await?)
|
||||
.unwrap_or("<invalid utf8>")
|
||||
);
|
||||
let empty_body: Body = Body::from(vec![]);
|
||||
let response = Response::builder()
|
||||
.header("Location", format!("/{}", bucket))
|
||||
.body(empty_body)
|
||||
.unwrap();
|
||||
Ok(response)
|
||||
}
|
||||
Method::HEAD => {
|
||||
// HeadBucket
|
||||
let empty_body: Body = Body::from(vec![]);
|
||||
let response = Response::builder().body(empty_body).unwrap();
|
||||
Ok(response)
|
||||
}
|
||||
Method::DELETE => {
|
||||
// DeleteBucket query
|
||||
Err(Error::Forbidden(
|
||||
"Cannot delete buckets using S3 api, please talk to Garage directly".into(),
|
||||
))
|
||||
}
|
||||
Method::GET => {
|
||||
if params.contains_key("location") {
|
||||
// GetBucketLocation call
|
||||
Ok(handle_get_bucket_location(garage)?)
|
||||
} else if params.contains_key("versioning") {
|
||||
// GetBucketVersioning
|
||||
Ok(handle_get_bucket_versioning()?)
|
||||
} else {
|
||||
// ListObjects or ListObjectsV2 query
|
||||
let q = parse_list_objects_query(bucket, ¶ms)?;
|
||||
Ok(handle_list(garage, &q).await?)
|
||||
}
|
||||
}
|
||||
Method::POST => {
|
||||
if params.contains_key(&"delete".to_string()) {
|
||||
// DeleteObjects
|
||||
Ok(handle_delete_objects(garage, bucket, req, content_sha256).await?)
|
||||
} else {
|
||||
debug!(
|
||||
"Body: {}",
|
||||
std::str::from_utf8(&hyper::body::to_bytes(req.into_body()).await?)
|
||||
.unwrap_or("<invalid utf8>")
|
||||
);
|
||||
Err(Error::BadRequest("Unsupported call".to_string()))
|
||||
}
|
||||
}
|
||||
_ => Err(Error::BadRequest("Invalid method".to_string())),
|
||||
Endpoint::DeleteObjects { bucket } => {
|
||||
handle_delete_objects(garage, &bucket, req, content_sha256).await
|
||||
}
|
||||
Endpoint::PutBucketWebsite { bucket } => {
|
||||
handle_put_website(garage, bucket, req, content_sha256).await
|
||||
}
|
||||
Endpoint::DeleteBucketWebsite { bucket } => handle_delete_website(garage, bucket).await,
|
||||
endpoint => Err(Error::NotImplemented(endpoint.name().to_owned())),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+27
-2
@@ -1,5 +1,8 @@
|
||||
use std::convert::TryInto;
|
||||
|
||||
use err_derive::Error;
|
||||
use hyper::StatusCode;
|
||||
use hyper::header::HeaderValue;
|
||||
use hyper::{HeaderMap, StatusCode};
|
||||
|
||||
use garage_util::error::Error as GarageError;
|
||||
|
||||
@@ -57,11 +60,15 @@ pub enum Error {
|
||||
|
||||
/// The client sent a range header with invalid value
|
||||
#[error(display = "Invalid HTTP range: {:?}", _0)]
|
||||
InvalidRange(#[error(from)] http_range::HttpRangeParseError),
|
||||
InvalidRange(#[error(from)] (http_range::HttpRangeParseError, u64)),
|
||||
|
||||
/// The client sent an invalid request
|
||||
#[error(display = "Bad request: {}", _0)]
|
||||
BadRequest(String),
|
||||
|
||||
/// The client sent a request for an action not supported by garage
|
||||
#[error(display = "Unimplemented action: {}", _0)]
|
||||
NotImplemented(String),
|
||||
}
|
||||
|
||||
impl From<roxmltree::Error> for Error {
|
||||
@@ -90,6 +97,8 @@ impl Error {
|
||||
Error::InternalError(_) | Error::Hyper(_) | Error::Http(_) => {
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
}
|
||||
Error::InvalidRange(_) => StatusCode::RANGE_NOT_SATISFIABLE,
|
||||
Error::NotImplemented(_) => StatusCode::NOT_IMPLEMENTED,
|
||||
_ => StatusCode::BAD_REQUEST,
|
||||
}
|
||||
}
|
||||
@@ -127,6 +136,22 @@ impl Error {
|
||||
.into()
|
||||
})
|
||||
}
|
||||
|
||||
pub fn add_headers(&self, header_map: &mut HeaderMap<HeaderValue>) {
|
||||
use hyper::header;
|
||||
#[allow(clippy::single_match)]
|
||||
match self {
|
||||
Error::InvalidRange((_, len)) => {
|
||||
header_map.append(
|
||||
header::CONTENT_RANGE,
|
||||
format!("bytes */{}", len)
|
||||
.try_into()
|
||||
.expect("header value only contain ascii"),
|
||||
);
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Trait to map error to the Bad Request error code
|
||||
|
||||
+1
-1
@@ -26,7 +26,7 @@ pub fn host_to_bucket<'a>(host: &'a str, root: &str) -> Option<&'a str> {
|
||||
/// The HTTP host contains both a host and a port.
|
||||
/// Extracting the port is more complex than just finding the colon (:) symbol due to IPv6
|
||||
/// We do not use the collect pattern as there is no way in std rust to collect over a stack allocated value
|
||||
/// check here: https://docs.rs/collect_slice/1.2.0/collect_slice/
|
||||
/// check here: <https://docs.rs/collect_slice/1.2.0/collect_slice/>
|
||||
pub fn authority_to_host(authority: &str) -> Result<String, Error> {
|
||||
let mut iter = authority.chars().enumerate();
|
||||
let (_, first_char) = iter
|
||||
|
||||
@@ -19,4 +19,6 @@ mod s3_delete;
|
||||
pub mod s3_get;
|
||||
mod s3_list;
|
||||
mod s3_put;
|
||||
mod s3_router;
|
||||
mod s3_website;
|
||||
mod s3_xml;
|
||||
|
||||
@@ -45,6 +45,7 @@ pub fn handle_list_buckets(api_key: &Key) -> Result<Response<Body>, Error> {
|
||||
.authorized_buckets
|
||||
.items()
|
||||
.iter()
|
||||
.filter(|(_, _, perms)| perms.allow_read || perms.allow_write)
|
||||
.map(|(name, ts, _)| s3_xml::Bucket {
|
||||
creation_date: s3_xml::Value(msec_to_rfc3339(*ts)),
|
||||
name: s3_xml::Value(name.to_string()),
|
||||
|
||||
+5
-8
@@ -156,11 +156,12 @@ pub async fn handle_get(
|
||||
let range = match req.headers().get("range") {
|
||||
Some(range) => {
|
||||
let range_str = range.to_str()?;
|
||||
let mut ranges = http_range::HttpRange::parse(range_str, last_v_meta.size)?;
|
||||
let mut ranges = http_range::HttpRange::parse(range_str, last_v_meta.size)
|
||||
.map_err(|e| (e, last_v_meta.size))?;
|
||||
if ranges.len() > 1 {
|
||||
return Err(Error::BadRequest(
|
||||
"Multiple ranges not supported".to_string(),
|
||||
));
|
||||
// garage does not support multi-range requests yet, so we respond with the entire
|
||||
// object when multiple ranges are requested
|
||||
None
|
||||
} else {
|
||||
ranges.pop()
|
||||
}
|
||||
@@ -235,10 +236,6 @@ async fn handle_get_range(
|
||||
begin: u64,
|
||||
end: u64,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
if end > version_meta.size {
|
||||
return Err(Error::BadRequest("Range not included in file".to_string()));
|
||||
}
|
||||
|
||||
let resp_builder = object_headers(version, version_meta)
|
||||
.header("Content-Length", format!("{}", end - begin))
|
||||
.header(
|
||||
|
||||
+1
-27
@@ -1,4 +1,4 @@
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::sync::Arc;
|
||||
|
||||
use hyper::{Body, Response};
|
||||
@@ -35,32 +35,6 @@ struct ListResultInfo {
|
||||
etag: String,
|
||||
}
|
||||
|
||||
pub fn parse_list_objects_query(
|
||||
bucket: &str,
|
||||
params: &HashMap<String, String>,
|
||||
) -> Result<ListObjectsQuery, Error> {
|
||||
Ok(ListObjectsQuery {
|
||||
is_v2: params.get("list-type").map(|x| x == "2").unwrap_or(false),
|
||||
bucket: bucket.to_string(),
|
||||
delimiter: params.get("delimiter").filter(|x| !x.is_empty()).cloned(),
|
||||
max_keys: params
|
||||
.get("max-keys")
|
||||
.map(|x| {
|
||||
x.parse::<usize>()
|
||||
.ok_or_bad_request("Invalid value for max-keys")
|
||||
})
|
||||
.unwrap_or(Ok(1000))?,
|
||||
prefix: params.get("prefix").cloned().unwrap_or_default(),
|
||||
marker: params.get("marker").cloned(),
|
||||
continuation_token: params.get("continuation-token").cloned(),
|
||||
start_after: params.get("start-after").cloned(),
|
||||
urlencode_resp: params
|
||||
.get("encoding-type")
|
||||
.map(|x| x == "url")
|
||||
.unwrap_or(false),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn handle_list(
|
||||
garage: Arc<Garage>,
|
||||
query: &ListObjectsQuery,
|
||||
|
||||
+1
-6
@@ -354,15 +354,10 @@ pub async fn handle_put_part(
|
||||
req: Request<Body>,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
part_number_str: &str,
|
||||
part_number: u64,
|
||||
upload_id: &str,
|
||||
content_sha256: Option<Hash>,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
// Check parameters
|
||||
let part_number = part_number_str
|
||||
.parse::<u64>()
|
||||
.ok_or_bad_request("Invalid part number")?;
|
||||
|
||||
let version_uuid = decode_upload_id(upload_id)?;
|
||||
|
||||
let content_md5 = match req.headers().get("content-md5") {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,308 @@
|
||||
use quick_xml::de::from_reader;
|
||||
use std::sync::Arc;
|
||||
|
||||
use hyper::{Body, Request, Response, StatusCode};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::error::*;
|
||||
use crate::s3_xml::{xmlns_tag, IntValue, Value};
|
||||
use crate::signature::verify_signed_content;
|
||||
use garage_model::bucket_table::BucketState;
|
||||
use garage_model::garage::Garage;
|
||||
use garage_table::*;
|
||||
use garage_util::data::Hash;
|
||||
|
||||
pub async fn handle_delete_website(
|
||||
garage: Arc<Garage>,
|
||||
bucket: String,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let mut bucket = garage
|
||||
.bucket_table
|
||||
.get(&EmptyKey, &bucket)
|
||||
.await?
|
||||
.ok_or(Error::NotFound)?;
|
||||
|
||||
if let BucketState::Present(state) = bucket.state.get_mut() {
|
||||
state.website.update(false);
|
||||
garage.bucket_table.insert(&bucket).await?;
|
||||
}
|
||||
|
||||
Ok(Response::builder()
|
||||
.status(StatusCode::NO_CONTENT)
|
||||
.body(Body::from(vec![]))
|
||||
.unwrap())
|
||||
}
|
||||
|
||||
pub async fn handle_put_website(
|
||||
garage: Arc<Garage>,
|
||||
bucket: String,
|
||||
req: Request<Body>,
|
||||
content_sha256: Option<Hash>,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let body = hyper::body::to_bytes(req.into_body()).await?;
|
||||
verify_signed_content(content_sha256, &body[..])?;
|
||||
|
||||
let mut bucket = garage
|
||||
.bucket_table
|
||||
.get(&EmptyKey, &bucket)
|
||||
.await?
|
||||
.ok_or(Error::NotFound)?;
|
||||
|
||||
let conf: WebsiteConfiguration = from_reader(&body as &[u8])?;
|
||||
conf.validate()?;
|
||||
|
||||
if let BucketState::Present(state) = bucket.state.get_mut() {
|
||||
state.website.update(true);
|
||||
garage.bucket_table.insert(&bucket).await?;
|
||||
}
|
||||
|
||||
Ok(Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.body(Body::from(vec![]))
|
||||
.unwrap())
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct WebsiteConfiguration {
|
||||
#[serde(serialize_with = "xmlns_tag", skip_deserializing)]
|
||||
pub xmlns: (),
|
||||
#[serde(rename = "ErrorDocument")]
|
||||
pub error_document: Option<Key>,
|
||||
#[serde(rename = "IndexDocument")]
|
||||
pub index_document: Option<Suffix>,
|
||||
#[serde(rename = "RedirectAllRequestsTo")]
|
||||
pub redirect_all_requests_to: Option<Target>,
|
||||
#[serde(rename = "RoutingRules")]
|
||||
pub routing_rules: Option<Vec<RoutingRule>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct RoutingRule {
|
||||
#[serde(rename = "RoutingRule")]
|
||||
pub inner: RoutingRuleInner,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct RoutingRuleInner {
|
||||
#[serde(rename = "Condition")]
|
||||
pub condition: Option<Condition>,
|
||||
#[serde(rename = "Redirect")]
|
||||
pub redirect: Redirect,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct Key {
|
||||
#[serde(rename = "Key")]
|
||||
pub key: Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct Suffix {
|
||||
#[serde(rename = "Suffix")]
|
||||
pub suffix: Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct Target {
|
||||
#[serde(rename = "HostName")]
|
||||
pub hostname: Value,
|
||||
#[serde(rename = "Protocol")]
|
||||
pub protocol: Option<Value>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct Condition {
|
||||
#[serde(rename = "HttpErrorCodeReturnedEquals")]
|
||||
pub http_error_code: Option<IntValue>,
|
||||
#[serde(rename = "KeyPrefixEquals")]
|
||||
pub prefix: Option<Value>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct Redirect {
|
||||
#[serde(rename = "HostName")]
|
||||
pub hostname: Option<Value>,
|
||||
#[serde(rename = "Protocol")]
|
||||
pub protocol: Option<Value>,
|
||||
#[serde(rename = "HttpRedirectCode")]
|
||||
pub http_redirect_code: Option<IntValue>,
|
||||
#[serde(rename = "ReplaceKeyPrefixWith")]
|
||||
pub replace_prefix: Option<Value>,
|
||||
#[serde(rename = "ReplaceKeyWith")]
|
||||
pub replace_full: Option<Value>,
|
||||
}
|
||||
|
||||
impl WebsiteConfiguration {
|
||||
pub fn validate(&self) -> Result<(), Error> {
|
||||
if self.redirect_all_requests_to.is_some()
|
||||
&& (self.error_document.is_some()
|
||||
|| self.index_document.is_some()
|
||||
|| self.routing_rules.is_some())
|
||||
{
|
||||
return Err(Error::BadRequest(
|
||||
"Bad XML: can't have RedirectAllRequestsTo and other fields".to_owned(),
|
||||
));
|
||||
}
|
||||
if let Some(ref ed) = self.error_document {
|
||||
ed.validate()?;
|
||||
}
|
||||
if let Some(ref id) = self.index_document {
|
||||
id.validate()?;
|
||||
}
|
||||
if let Some(ref rart) = self.redirect_all_requests_to {
|
||||
rart.validate()?;
|
||||
}
|
||||
if let Some(ref rrs) = self.routing_rules {
|
||||
for rr in rrs {
|
||||
rr.inner.validate()?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Key {
|
||||
pub fn validate(&self) -> Result<(), Error> {
|
||||
if self.key.0.is_empty() {
|
||||
Err(Error::BadRequest(
|
||||
"Bad XML: error document specified but empty".to_owned(),
|
||||
))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Suffix {
|
||||
pub fn validate(&self) -> Result<(), Error> {
|
||||
if self.suffix.0.is_empty() | self.suffix.0.contains('/') {
|
||||
Err(Error::BadRequest(
|
||||
"Bad XML: index document is empty or contains /".to_owned(),
|
||||
))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Target {
|
||||
pub fn validate(&self) -> Result<(), Error> {
|
||||
if let Some(ref protocol) = self.protocol {
|
||||
if protocol.0 != "http" && protocol.0 != "https" {
|
||||
return Err(Error::BadRequest("Bad XML: invalid protocol".to_owned()));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl RoutingRuleInner {
|
||||
pub fn validate(&self) -> Result<(), Error> {
|
||||
let has_prefix = self
|
||||
.condition
|
||||
.as_ref()
|
||||
.map(|c| c.prefix.as_ref())
|
||||
.flatten()
|
||||
.is_some();
|
||||
self.redirect.validate(has_prefix)
|
||||
}
|
||||
}
|
||||
|
||||
impl Redirect {
|
||||
pub fn validate(&self, has_prefix: bool) -> Result<(), Error> {
|
||||
if self.replace_prefix.is_some() {
|
||||
if self.replace_full.is_some() {
|
||||
return Err(Error::BadRequest(
|
||||
"Bad XML: both ReplaceKeyPrefixWith and ReplaceKeyWith are set".to_owned(),
|
||||
));
|
||||
}
|
||||
if !has_prefix {
|
||||
return Err(Error::BadRequest(
|
||||
"Bad XML: ReplaceKeyPrefixWith is set, but KeyPrefixEquals isn't".to_owned(),
|
||||
));
|
||||
}
|
||||
}
|
||||
if let Some(ref protocol) = self.protocol {
|
||||
if protocol.0 != "http" && protocol.0 != "https" {
|
||||
return Err(Error::BadRequest("Bad XML: invalid protocol".to_owned()));
|
||||
}
|
||||
}
|
||||
// TODO there are probably more invalide cases, but which ones?
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
use quick_xml::de::from_str;
|
||||
|
||||
#[test]
|
||||
fn test_deserialize() {
|
||||
let message = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<WebsiteConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<ErrorDocument>
|
||||
<Key>my-error-doc</Key>
|
||||
</ErrorDocument>
|
||||
<IndexDocument>
|
||||
<Suffix>my-index</Suffix>
|
||||
</IndexDocument>
|
||||
<RedirectAllRequestsTo>
|
||||
<HostName>garage.tld</HostName>
|
||||
<Protocol>https</Protocol>
|
||||
</RedirectAllRequestsTo>
|
||||
<RoutingRules>
|
||||
<RoutingRule>
|
||||
<Condition>
|
||||
<HttpErrorCodeReturnedEquals>404</HttpErrorCodeReturnedEquals>
|
||||
<KeyPrefixEquals>prefix1</KeyPrefixEquals>
|
||||
</Condition>
|
||||
<Redirect>
|
||||
<HostName>gara.ge</HostName>
|
||||
<Protocol>http</Protocol>
|
||||
<HttpRedirectCode>303</HttpRedirectCode>
|
||||
<ReplaceKeyPrefixWith>prefix2</ReplaceKeyPrefixWith>
|
||||
<ReplaceKeyWith>fullkey</ReplaceKeyWith>
|
||||
</Redirect>
|
||||
</RoutingRule>
|
||||
</RoutingRules>
|
||||
</WebsiteConfiguration>"#;
|
||||
let conf: WebsiteConfiguration = from_str(message).unwrap();
|
||||
let ref_value = WebsiteConfiguration {
|
||||
xmlns: (),
|
||||
error_document: Some(Key {
|
||||
key: Value("my-error-doc".to_owned()),
|
||||
}),
|
||||
index_document: Some(Suffix {
|
||||
suffix: Value("my-index".to_owned()),
|
||||
}),
|
||||
redirect_all_requests_to: Some(Target {
|
||||
hostname: Value("garage.tld".to_owned()),
|
||||
protocol: Some(Value("https".to_owned())),
|
||||
}),
|
||||
routing_rules: Some(vec![RoutingRule {
|
||||
inner: RoutingRuleInner {
|
||||
condition: Some(Condition {
|
||||
http_error_code: Some(IntValue(404)),
|
||||
prefix: Some(Value("prefix1".to_owned())),
|
||||
}),
|
||||
redirect: Redirect {
|
||||
hostname: Some(Value("gara.ge".to_owned())),
|
||||
protocol: Some(Value("http".to_owned())),
|
||||
http_redirect_code: Some(IntValue(303)),
|
||||
replace_prefix: Some(Value("prefix2".to_owned())),
|
||||
replace_full: Some(Value("fullkey".to_owned())),
|
||||
},
|
||||
},
|
||||
}]),
|
||||
};
|
||||
assert_eq! {
|
||||
ref_value,
|
||||
conf
|
||||
}
|
||||
// TODO verify result is ok
|
||||
// TODO cycle back and verify if ok
|
||||
}
|
||||
}
|
||||
+4
-4
@@ -1,5 +1,5 @@
|
||||
use quick_xml::se::to_string;
|
||||
use serde::{Serialize, Serializer};
|
||||
use serde::{Deserialize, Serialize, Serializer};
|
||||
|
||||
use crate::Error as ApiError;
|
||||
|
||||
@@ -9,14 +9,14 @@ pub fn to_xml_with_header<T: Serialize>(x: &T) -> Result<String, ApiError> {
|
||||
Ok(xml)
|
||||
}
|
||||
|
||||
fn xmlns_tag<S: Serializer>(_v: &(), s: S) -> Result<S::Ok, S::Error> {
|
||||
pub fn xmlns_tag<S: Serializer>(_v: &(), s: S) -> Result<S::Ok, S::Error> {
|
||||
s.serialize_str("http://s3.amazonaws.com/doc/2006-03-01/")
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct Value(#[serde(rename = "$value")] pub String);
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct IntValue(#[serde(rename = "$value")] pub i64);
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq)]
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage"
|
||||
version = "0.5.0"
|
||||
version = "0.5.1"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
@@ -15,12 +15,12 @@ path = "main.rs"
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
garage_api = { version = "0.5.0", path = "../api" }
|
||||
garage_model = { version = "0.5.0", path = "../model" }
|
||||
garage_rpc = { version = "0.5.0", path = "../rpc" }
|
||||
garage_table = { version = "0.5.0", path = "../table" }
|
||||
garage_util = { version = "0.5.0", path = "../util" }
|
||||
garage_web = { version = "0.5.0", path = "../web" }
|
||||
garage_api = { version = "0.5.1", path = "../api" }
|
||||
garage_model = { version = "0.5.1", path = "../model" }
|
||||
garage_rpc = { version = "0.5.1", path = "../rpc" }
|
||||
garage_table = { version = "0.5.1", path = "../table" }
|
||||
garage_util = { version = "0.5.1", path = "../util" }
|
||||
garage_web = { version = "0.5.1", path = "../web" }
|
||||
|
||||
bytes = "1.0"
|
||||
git-version = "0.3.4"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_model"
|
||||
version = "0.5.0"
|
||||
version = "0.5.1"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
@@ -14,15 +14,16 @@ path = "lib.rs"
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
garage_rpc = { version = "0.5.0", path = "../rpc" }
|
||||
garage_table = { version = "0.5.0", path = "../table" }
|
||||
garage_util = { version = "0.5.0", path = "../util" }
|
||||
garage_rpc = { version = "0.5.1", path = "../rpc" }
|
||||
garage_table = { version = "0.5.1", path = "../table" }
|
||||
garage_util = { version = "0.5.1", path = "../util" }
|
||||
|
||||
async-trait = "0.1.7"
|
||||
arc-swap = "1.0"
|
||||
hex = "0.4"
|
||||
log = "0.4"
|
||||
rand = "0.8"
|
||||
zstd = { version = "0.9", default-features = false }
|
||||
|
||||
sled = "0.34"
|
||||
|
||||
|
||||
+165
-36
@@ -11,6 +11,7 @@ use serde::{Deserialize, Serialize};
|
||||
use tokio::fs;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::sync::{watch, Mutex, Notify};
|
||||
use zstd::stream::{decode_all as zstd_decode, Encoder};
|
||||
|
||||
use garage_util::data::*;
|
||||
use garage_util::error::*;
|
||||
@@ -55,22 +56,81 @@ pub enum BlockRpc {
|
||||
GetBlock(Hash),
|
||||
/// Message to send a block of data, either because requested, of for first delivery of new
|
||||
/// block
|
||||
PutBlock(PutBlockMessage),
|
||||
PutBlock {
|
||||
hash: Hash,
|
||||
data: DataBlock,
|
||||
},
|
||||
/// Ask other node if they should have this block, but don't actually have it
|
||||
NeedBlockQuery(Hash),
|
||||
/// Response : whether the node do require that block
|
||||
NeedBlockReply(bool),
|
||||
}
|
||||
|
||||
/// Structure used to send a block
|
||||
/// A possibly compressed block of data
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct PutBlockMessage {
|
||||
/// Hash of the block
|
||||
pub hash: Hash,
|
||||
pub enum DataBlock {
|
||||
/// Uncompressed data
|
||||
Plain(#[serde(with = "serde_bytes")] Vec<u8>),
|
||||
/// Data compressed with zstd
|
||||
Compressed(#[serde(with = "serde_bytes")] Vec<u8>),
|
||||
}
|
||||
|
||||
/// Content of the block
|
||||
#[serde(with = "serde_bytes")]
|
||||
pub data: Vec<u8>,
|
||||
impl DataBlock {
|
||||
/// Query whether this block is compressed
|
||||
pub fn is_compressed(&self) -> bool {
|
||||
matches!(self, DataBlock::Compressed(_))
|
||||
}
|
||||
|
||||
/// Get the inner, possibly compressed buffer. You should probably use [`DataBlock::verify_get`]
|
||||
/// instead
|
||||
pub fn inner_buffer(&self) -> &[u8] {
|
||||
use DataBlock::*;
|
||||
let (Plain(ref res) | Compressed(ref res)) = self;
|
||||
res
|
||||
}
|
||||
|
||||
/// Get the buffer, possibly decompressing it, and verify it's integrity.
|
||||
/// For Plain block, data is compared to hash, for Compressed block, zstd checksumming system
|
||||
/// is used instead.
|
||||
pub fn verify_get(self, hash: Hash) -> Result<Vec<u8>, Error> {
|
||||
match self {
|
||||
DataBlock::Plain(data) => {
|
||||
if blake2sum(&data) == hash {
|
||||
Ok(data)
|
||||
} else {
|
||||
Err(Error::CorruptData(hash))
|
||||
}
|
||||
}
|
||||
DataBlock::Compressed(data) => {
|
||||
zstd_decode(&data[..]).map_err(|_| Error::CorruptData(hash))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Verify data integrity. Allocate less than [`DataBlock::verify_get`] and don't consume self, but
|
||||
/// does not return the buffer content.
|
||||
pub fn verify(&self, hash: Hash) -> Result<(), Error> {
|
||||
match self {
|
||||
DataBlock::Plain(data) => {
|
||||
if blake2sum(data) == hash {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(Error::CorruptData(hash))
|
||||
}
|
||||
}
|
||||
DataBlock::Compressed(data) => zstd::stream::copy_decode(&data[..], std::io::sink())
|
||||
.map_err(|_| Error::CorruptData(hash)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_buffer(data: Vec<u8>, level: Option<i32>) -> DataBlock {
|
||||
if let Some(level) = level {
|
||||
if let Ok(data) = zstd_encode(&data[..], level) {
|
||||
return DataBlock::Compressed(data);
|
||||
}
|
||||
}
|
||||
DataBlock::Plain(data)
|
||||
}
|
||||
}
|
||||
|
||||
impl Rpc for BlockRpc {
|
||||
@@ -138,10 +198,8 @@ impl BlockManager {
|
||||
block_manager
|
||||
}
|
||||
|
||||
// ---- Public interface ----
|
||||
|
||||
/// Ask nodes that might have a block for it
|
||||
pub async fn rpc_get_block(&self, hash: &Hash) -> Result<Vec<u8>, Error> {
|
||||
/// Ask nodes that might have a (possibly compressed) block for it
|
||||
async fn rpc_get_raw_block(&self, hash: &Hash) -> Result<DataBlock, Error> {
|
||||
let who = self.replication.read_nodes(hash);
|
||||
let resps = self
|
||||
.system
|
||||
@@ -158,8 +216,8 @@ impl BlockManager {
|
||||
.await?;
|
||||
|
||||
for resp in resps {
|
||||
if let BlockRpc::PutBlock(msg) = resp {
|
||||
return Ok(msg.data);
|
||||
if let BlockRpc::PutBlock { data, .. } = resp {
|
||||
return Ok(data);
|
||||
}
|
||||
}
|
||||
Err(Error::Message(format!(
|
||||
@@ -168,15 +226,30 @@ impl BlockManager {
|
||||
)))
|
||||
}
|
||||
|
||||
// ---- Public interface ----
|
||||
|
||||
/// Ask nodes that might have a block for it
|
||||
pub async fn rpc_get_block(&self, hash: &Hash) -> Result<Vec<u8>, Error> {
|
||||
self.rpc_get_raw_block(hash).await?.verify_get(*hash)
|
||||
}
|
||||
|
||||
/// Send block to nodes that should have it
|
||||
pub async fn rpc_put_block(&self, hash: Hash, data: Vec<u8>) -> Result<(), Error> {
|
||||
let who = self.replication.write_nodes(&hash);
|
||||
let compression_level = self
|
||||
.garage
|
||||
.load()
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.config
|
||||
.compression_level;
|
||||
let data = DataBlock::from_buffer(data, compression_level);
|
||||
self.system
|
||||
.rpc
|
||||
.try_call_many(
|
||||
&self.endpoint,
|
||||
&who[..],
|
||||
BlockRpc::PutBlock(PutBlockMessage { hash, data }),
|
||||
BlockRpc::PutBlock { hash, data },
|
||||
RequestStrategy::with_priority(PRIO_NORMAL)
|
||||
.with_quorum(self.replication.write_quorum())
|
||||
.with_timeout(BLOCK_RW_TIMEOUT),
|
||||
@@ -306,7 +379,7 @@ impl BlockManager {
|
||||
// ---- Reading and writing blocks locally ----
|
||||
|
||||
/// Write a block to disk
|
||||
async fn write_block(&self, hash: &Hash, data: &[u8]) -> Result<BlockRpc, Error> {
|
||||
async fn write_block(&self, hash: &Hash, data: &DataBlock) -> Result<BlockRpc, Error> {
|
||||
self.mutation_lock
|
||||
.lock()
|
||||
.await
|
||||
@@ -316,21 +389,31 @@ impl BlockManager {
|
||||
|
||||
/// Read block from disk, verifying it's integrity
|
||||
async fn read_block(&self, hash: &Hash) -> Result<BlockRpc, Error> {
|
||||
let path = self.block_path(hash);
|
||||
|
||||
let mut f = match fs::File::open(&path).await {
|
||||
Ok(f) => f,
|
||||
let mut path = self.block_path(hash);
|
||||
let compressed = match self.is_block_compressed(hash).await {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
// Not found but maybe we should have had it ??
|
||||
self.put_to_resync(hash, 2 * BLOCK_RW_TIMEOUT)?;
|
||||
return Err(Into::into(e));
|
||||
}
|
||||
};
|
||||
if compressed {
|
||||
path.set_extension("zst");
|
||||
}
|
||||
let mut f = fs::File::open(&path).await?;
|
||||
|
||||
let mut data = vec![];
|
||||
f.read_to_end(&mut data).await?;
|
||||
drop(f);
|
||||
|
||||
if blake2sum(&data[..]) != *hash {
|
||||
let data = if compressed {
|
||||
DataBlock::Compressed(data)
|
||||
} else {
|
||||
DataBlock::Plain(data)
|
||||
};
|
||||
|
||||
if data.verify(*hash).is_err() {
|
||||
self.mutation_lock
|
||||
.lock()
|
||||
.await
|
||||
@@ -340,7 +423,7 @@ impl BlockManager {
|
||||
return Err(Error::CorruptData(*hash));
|
||||
}
|
||||
|
||||
Ok(BlockRpc::PutBlock(PutBlockMessage { hash: *hash, data }))
|
||||
Ok(BlockRpc::PutBlock { hash: *hash, data })
|
||||
}
|
||||
|
||||
/// Check if this node should have a block, but don't actually have it
|
||||
@@ -362,13 +445,25 @@ impl BlockManager {
|
||||
path
|
||||
}
|
||||
|
||||
/// Utility: give the full path where a block should be found
|
||||
/// Utility: give the full path where a block should be found, minus extension if block is
|
||||
/// compressed
|
||||
fn block_path(&self, hash: &Hash) -> PathBuf {
|
||||
let mut path = self.block_dir(hash);
|
||||
path.push(hex::encode(hash.as_ref()));
|
||||
path
|
||||
}
|
||||
|
||||
/// Utility: check if block is stored compressed. Error if block is not stored
|
||||
async fn is_block_compressed(&self, hash: &Hash) -> Result<bool, Error> {
|
||||
let mut path = self.block_path(hash);
|
||||
path.set_extension("zst");
|
||||
if fs::metadata(&path).await.is_ok() {
|
||||
return Ok(true);
|
||||
}
|
||||
path.set_extension("");
|
||||
fs::metadata(&path).await.map(|_| false).map_err(Into::into)
|
||||
}
|
||||
|
||||
// ---- Resync loop ----
|
||||
|
||||
pub fn spawn_background_worker(self: Arc<Self>) {
|
||||
@@ -550,8 +645,8 @@ impl BlockManager {
|
||||
hash
|
||||
);
|
||||
|
||||
let block_data = self.rpc_get_block(hash).await?;
|
||||
self.write_block(hash, &block_data[..]).await?;
|
||||
let block_data = self.rpc_get_raw_block(hash).await?;
|
||||
self.write_block(hash, &block_data).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -602,6 +697,7 @@ impl BlockManager {
|
||||
};
|
||||
let ent_type = data_dir_ent.file_type().await?;
|
||||
|
||||
let name = name.strip_suffix(".zst").unwrap_or(&name);
|
||||
if name.len() == 2 && hex::decode(&name).is_ok() && ent_type.is_dir() {
|
||||
state = self
|
||||
.for_each_file_rec(&data_dir_ent.path(), state, f, must_exit)
|
||||
@@ -631,7 +727,7 @@ impl EndpointHandler<BlockRpc> for BlockManager {
|
||||
_from: NodeID,
|
||||
) -> Result<BlockRpc, Error> {
|
||||
match message {
|
||||
BlockRpc::PutBlock(m) => self.write_block(&m.hash, &m.data).await,
|
||||
BlockRpc::PutBlock { hash, data } => self.write_block(hash, data).await,
|
||||
BlockRpc::GetBlock(h) => self.read_block(h).await,
|
||||
BlockRpc::NeedBlockQuery(h) => self.need_block(h).await.map(BlockRpc::NeedBlockReply),
|
||||
_ => Err(Error::BadRpc("Unexpected RPC message".to_string())),
|
||||
@@ -650,9 +746,7 @@ impl BlockManagerLocked {
|
||||
hash: &Hash,
|
||||
mgr: &BlockManager,
|
||||
) -> Result<BlockStatus, Error> {
|
||||
let path = mgr.block_path(hash);
|
||||
|
||||
let exists = fs::metadata(&path).await.is_ok();
|
||||
let exists = mgr.is_block_compressed(hash).await.is_ok();
|
||||
let needed = mgr.get_block_rc(hash)?;
|
||||
|
||||
Ok(BlockStatus { exists, needed })
|
||||
@@ -661,16 +755,31 @@ impl BlockManagerLocked {
|
||||
async fn write_block(
|
||||
&self,
|
||||
hash: &Hash,
|
||||
data: &[u8],
|
||||
data: &DataBlock,
|
||||
mgr: &BlockManager,
|
||||
) -> Result<BlockRpc, Error> {
|
||||
let compressed = data.is_compressed();
|
||||
let data = data.inner_buffer();
|
||||
|
||||
let mut path = mgr.block_dir(hash);
|
||||
fs::create_dir_all(&path).await?;
|
||||
|
||||
path.push(hex::encode(hash));
|
||||
if fs::metadata(&path).await.is_ok() {
|
||||
return Ok(BlockRpc::Ok);
|
||||
}
|
||||
let to_delete = match (mgr.is_block_compressed(hash).await, compressed) {
|
||||
(Ok(true), _) => return Ok(BlockRpc::Ok),
|
||||
(Ok(false), false) => return Ok(BlockRpc::Ok),
|
||||
(Ok(false), true) => {
|
||||
let path_to_delete = path.clone();
|
||||
path.set_extension("zst");
|
||||
Some(path_to_delete)
|
||||
}
|
||||
(Err(_), compressed) => {
|
||||
if compressed {
|
||||
path.set_extension("zst");
|
||||
}
|
||||
None
|
||||
}
|
||||
};
|
||||
|
||||
let mut path2 = path.clone();
|
||||
path2.set_extension("tmp");
|
||||
@@ -679,6 +788,9 @@ impl BlockManagerLocked {
|
||||
drop(f);
|
||||
|
||||
fs::rename(path2, path).await?;
|
||||
if let Some(to_delete) = to_delete {
|
||||
fs::remove_file(to_delete).await?;
|
||||
}
|
||||
|
||||
Ok(BlockRpc::Ok)
|
||||
}
|
||||
@@ -688,9 +800,14 @@ impl BlockManagerLocked {
|
||||
"Block {:?} is corrupted. Renaming to .corrupted and resyncing.",
|
||||
hash
|
||||
);
|
||||
let path = mgr.block_path(hash);
|
||||
let mut path = mgr.block_path(hash);
|
||||
let mut path2 = path.clone();
|
||||
path2.set_extension("corrupted");
|
||||
if mgr.is_block_compressed(hash).await? {
|
||||
path.set_extension("zst");
|
||||
path2.set_extension("zst.corrupted");
|
||||
} else {
|
||||
path2.set_extension("corrupted");
|
||||
}
|
||||
fs::rename(path, path2).await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -699,7 +816,10 @@ impl BlockManagerLocked {
|
||||
let BlockStatus { exists, needed } = self.check_block_status(hash, mgr).await?;
|
||||
|
||||
if exists && needed.is_deletable() {
|
||||
let path = mgr.block_path(hash);
|
||||
let mut path = mgr.block_path(hash);
|
||||
if mgr.is_block_compressed(hash).await? {
|
||||
path.set_extension("zst");
|
||||
}
|
||||
fs::remove_file(path).await?;
|
||||
}
|
||||
Ok(())
|
||||
@@ -806,3 +926,12 @@ impl RcEntry {
|
||||
!self.is_deletable()
|
||||
}
|
||||
}
|
||||
|
||||
fn zstd_encode<R: std::io::Read>(mut source: R, level: i32) -> std::io::Result<Vec<u8>> {
|
||||
let mut result = Vec::<u8>::new();
|
||||
let mut encoder = Encoder::new(&mut result, level)?;
|
||||
encoder.include_checksum(true)?;
|
||||
std::io::copy(&mut source, &mut encoder)?;
|
||||
encoder.finish()?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_rpc"
|
||||
version = "0.5.0"
|
||||
version = "0.5.1"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
@@ -14,7 +14,7 @@ path = "lib.rs"
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
garage_util = { version = "0.5.0", path = "../util" }
|
||||
garage_util = { version = "0.5.1", path = "../util" }
|
||||
|
||||
arc-swap = "1.0"
|
||||
bytes = "1.0"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_table"
|
||||
version = "0.5.0"
|
||||
version = "0.5.1"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
@@ -14,8 +14,8 @@ path = "lib.rs"
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
garage_rpc = { version = "0.5.0", path = "../rpc" }
|
||||
garage_util = { version = "0.5.0", path = "../util" }
|
||||
garage_rpc = { version = "0.5.1", path = "../rpc" }
|
||||
garage_util = { version = "0.5.1", path = "../util" }
|
||||
|
||||
async-trait = "0.1.7"
|
||||
bytes = "1.0"
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_util"
|
||||
version = "0.5.0"
|
||||
version = "0.5.1"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
|
||||
@@ -30,6 +30,13 @@ pub struct Config {
|
||||
// (we can add more aliases for this later)
|
||||
pub replication_mode: String,
|
||||
|
||||
/// Zstd compression level used on data blocks
|
||||
#[serde(
|
||||
deserialize_with = "deserialize_compression",
|
||||
default = "default_compression"
|
||||
)]
|
||||
pub compression_level: Option<i32>,
|
||||
|
||||
/// RPC secret key: 32 bytes hex encoded
|
||||
pub rpc_secret: String,
|
||||
|
||||
@@ -123,3 +130,57 @@ where
|
||||
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
fn default_compression() -> Option<i32> {
|
||||
Some(1)
|
||||
}
|
||||
|
||||
fn deserialize_compression<'de, D>(deserializer: D) -> Result<Option<i32>, D::Error>
|
||||
where
|
||||
D: de::Deserializer<'de>,
|
||||
{
|
||||
use std::convert::TryFrom;
|
||||
|
||||
struct OptionVisitor;
|
||||
|
||||
impl<'de> serde::de::Visitor<'de> for OptionVisitor {
|
||||
type Value = Option<i32>;
|
||||
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
formatter.write_str("int or 'none'")
|
||||
}
|
||||
|
||||
fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
|
||||
where
|
||||
E: de::Error,
|
||||
{
|
||||
if value.eq_ignore_ascii_case("none") {
|
||||
Ok(None)
|
||||
} else {
|
||||
Err(E::custom(format!(
|
||||
"Invalid compression level: '{}', should be a number, or 'none'",
|
||||
value
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
fn visit_i64<E>(self, v: i64) -> Result<Self::Value, E>
|
||||
where
|
||||
E: de::Error,
|
||||
{
|
||||
i32::try_from(v)
|
||||
.map(Some)
|
||||
.map_err(|_| E::custom("Compression level out of bound".to_owned()))
|
||||
}
|
||||
|
||||
fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E>
|
||||
where
|
||||
E: de::Error,
|
||||
{
|
||||
i32::try_from(v)
|
||||
.map(Some)
|
||||
.map_err(|_| E::custom("Compression level out of bound".to_owned()))
|
||||
}
|
||||
}
|
||||
|
||||
deserializer.deserialize_any(OptionVisitor)
|
||||
}
|
||||
|
||||
@@ -77,11 +77,21 @@ where
|
||||
self.v = new_value;
|
||||
}
|
||||
|
||||
/// Get the timestamp currently associated with the value
|
||||
pub fn timestamp(&self) -> u64 {
|
||||
self.ts
|
||||
}
|
||||
|
||||
/// Get the CRDT value
|
||||
pub fn get(&self) -> &T {
|
||||
&self.v
|
||||
}
|
||||
|
||||
/// Take the CRDT value
|
||||
pub fn take(self) -> T {
|
||||
self.v
|
||||
}
|
||||
|
||||
/// Get a mutable reference to the CRDT's value
|
||||
///
|
||||
/// This is usefull to mutate the inside value without changing the LWW timestamp.
|
||||
|
||||
+5
-5
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "garage_web"
|
||||
version = "0.5.0"
|
||||
version = "0.5.1"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
@@ -14,10 +14,10 @@ path = "lib.rs"
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
garage_api = { version = "0.5.0", path = "../api" }
|
||||
garage_model = { version = "0.5.0", path = "../model" }
|
||||
garage_util = { version = "0.5.0", path = "../util" }
|
||||
garage_table = { version = "0.5.0", path = "../table" }
|
||||
garage_api = { version = "0.5.1", path = "../api" }
|
||||
garage_model = { version = "0.5.1", path = "../model" }
|
||||
garage_util = { version = "0.5.1", path = "../util" }
|
||||
garage_table = { version = "0.5.1", path = "../table" }
|
||||
|
||||
err-derive = "0.3"
|
||||
log = "0.4"
|
||||
|
||||
+10
-1
@@ -1,5 +1,6 @@
|
||||
use err_derive::Error;
|
||||
use hyper::StatusCode;
|
||||
use hyper::header::HeaderValue;
|
||||
use hyper::{HeaderMap, StatusCode};
|
||||
|
||||
use garage_util::error::Error as GarageError;
|
||||
|
||||
@@ -47,4 +48,12 @@ impl Error {
|
||||
_ => StatusCode::BAD_REQUEST,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add_headers(&self, header_map: &mut HeaderMap<HeaderValue>) {
|
||||
#[allow(clippy::single_match)]
|
||||
match self {
|
||||
Error::ApiError(e) => e.add_headers(header_map),
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -62,6 +62,7 @@ fn error_to_res(e: Error) -> Response<Body> {
|
||||
let body: Body = Body::from(format!("{}\n", e));
|
||||
let mut http_error = Response::new(body);
|
||||
*http_error.status_mut() = e.http_status_code();
|
||||
e.add_headers(http_error.headers_mut());
|
||||
http_error
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user