Compare commits

...

24 Commits

Author SHA1 Message Date
Alex Auvolat ee57dd922b Bump version to 0.9.1 2024-01-16 16:28:17 +01:00
Alex 9cfeea389a Merge pull request 'CLI help, comments & messages: make clear that full-length node ID = public key' (#688) from rename-public-key into main
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/688
2024-01-16 13:33:43 +00:00
Alex Auvolat 82a29bf6e5 help, comments: make clear that full-length node ID = public key
Generally, avoid using the "public key" terminology
2024-01-16 14:04:11 +01:00
Alex 707d85f602 Merge pull request 'sync garage v0.9 with garage v0.8' (#657) from sync-08-09 into main
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/657
2024-01-16 11:33:27 +00:00
Alex Auvolat 4c5be79b80 Merge tag 'v0.8.5' into sync-08-09
Garage v0.8.5

This minor release includes the following improvements and fixes:

New features:

- Configuration: make LMDB's `map_size` configurable and make `block_size` and `sled_cache_capacity` expressable as strings (such as `10M`) (#628, #630)

- Add support for binding to Unix sockets for the S3, K2V, Admin and Web API servers (#640)

- Move the `convert_db` command into the main Garage binary (#645)

- Add support for specifying RPC secret and admin tokens as environment variables (#643)

- Add `allow_world_readable_secrets` option to config file (#663, #685)

Bug fixes:

- Use `statvfs` instead of mount list to determine free space in metadata/data directories (#611, #631)

- Add missing casts to fix 32-bit build (#632)

- Fix error when none of the HTTP servers (S3/K2V/Admin/Web) is started and fix shutdown hang (#613, #633)

- Add missing CORS headers to PostObject response (#609, #656)

- Monitoring: finer histogram boundaries in Prometheus exported metrics (#531, #686)

Other:

- Documentation improvements (#641)
2024-01-16 12:12:27 +01:00
Alex 083e982f5f Merge pull request 'Garage v0.8.5' (#687) from rel-0.8.5 into main-0.8.x
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/687
2024-01-16 10:30:54 +00:00
Alex Auvolat 50643e61bf Bump version to 0.8.5 2024-01-16 10:47:33 +01:00
Alex a6421ee5a5 Merge pull request 'monitoring: finer histogram boundaries in prometheus metrics (fix #531)' (#686) from fix-531 into main-0.8.x
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/686
2024-01-15 16:44:58 +00:00
Alex 993ce74976 Merge pull request '0.8.x: config: refactor secret sourcing' (#685) from secret-sourcing into main-0.8.x
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/685
2024-01-15 16:41:50 +00:00
Alex Auvolat f512609123 monitoring: finer histogram boundaries in prometheus metrics (fix #531) 2024-01-15 17:33:35 +01:00
Alex Auvolat 97bae7213a config: additional tests for secret sourcing 2024-01-15 17:30:30 +01:00
Alex Auvolat 7228695ee2 config: refactor secret sourcing 2024-01-15 17:18:46 +01:00
Alex ee7fe27d3d Merge pull request 'Add allow_world_readable_secrets option to config file' (#663) from PicNoir/garage:nin/world-readable-conf-file into main-0.8.x
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/663
2024-01-15 15:20:16 +00:00
Alex d91a1de731 Merge pull request 'fix typo in peertube doc' (#617) from Lapineige/garage:main into main
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/617
2024-01-11 11:19:42 +00:00
Félix Baylac Jacqué f83fa02193 Add allow_world_readable_secrets option to config file
Sometimes, the secret files permissions checks gets in the way. It's
by no mean complete, it doesn't take the Posix ACLs into account among
other things. Correctly checking the ACLs would be too involving (see
https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/658#issuecomment-7102)
and would likely still fail in some weird chmod settings.

We're adding a new configuration file key allowing the user to disable
this permission check altogether.

The (already existing) env variable counterpart always take precedence
to this config file option. That's useful in cases where the
configuration file is static and cannot be easily altered.

Fixes https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/658

Co-authored-by: Florian Klink <flokli@flokli.de>
2023-10-26 18:25:13 +02:00
Alex 4b3dee2ca3 Merge pull request 's3 api: add missing CORS headers to PostObject responses (fix #609)' (#656) from fix-cors-post-object into main-0.8.x
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/656
2023-10-26 09:17:14 +00:00
Alex Auvolat ac04934dae s3 api: add missing CORS headers to PostObject responses (fix #609) 2023-10-20 10:37:48 +02:00
Alex 0215b11402 Merge pull request 'Add support for specifying rpc_secret_file, metrics_token_file and admin_token_file using environment variables' (#643) from networkException/garage:token-file-env into main-0.8.x
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/643
2023-10-19 09:33:12 +00:00
networkException 8599051c49 garage: support specifying token / secret as environment variables
this patch adds support for specifying the `rpc_secret_file`,
`metrics_token_file` and `admin_token_file` as environment variables.
2023-10-19 03:39:02 +02:00
networkException 4a19ee94bb garage: fix admin-token description 2023-10-19 03:31:50 +02:00
networkException c99cb58d71 util: move reading secret file into seperate helper
this patch moves the logic to read a secret file (and check for correct
permissions) from `secret_from_file` into a new `read_secret_file`
helper.
2023-10-19 03:29:48 +02:00
networkException 5feb6a1f64 docs: add documentation for specifying token / secret file as environment variables 2023-10-19 03:28:44 +02:00
Lapineige 47e7f9e122 another typo 2023-08-19 20:29:24 +00:00
Lapineige 5ffcdb4634 fix typo 2023-08-19 15:17:51 +00:00
26 changed files with 481 additions and 290 deletions
Generated
+10 -9
View File
@@ -1198,7 +1198,7 @@ dependencies = [
[[package]]
name = "garage"
version = "0.9.0"
version = "0.9.1"
dependencies = [
"assert-json-diff",
"async-trait",
@@ -1227,6 +1227,7 @@ dependencies = [
"hyper",
"k2v-client",
"kuska-sodiumoxide",
"mktemp",
"netapp",
"opentelemetry",
"opentelemetry-otlp",
@@ -1249,7 +1250,7 @@ dependencies = [
[[package]]
name = "garage_api"
version = "0.9.0"
version = "0.9.1"
dependencies = [
"async-trait",
"base64 0.21.3",
@@ -1295,7 +1296,7 @@ dependencies = [
[[package]]
name = "garage_block"
version = "0.9.0"
version = "0.9.1"
dependencies = [
"arc-swap",
"async-compression",
@@ -1321,7 +1322,7 @@ dependencies = [
[[package]]
name = "garage_db"
version = "0.9.0"
version = "0.9.1"
dependencies = [
"clap 4.4.0",
"err-derive",
@@ -1336,7 +1337,7 @@ dependencies = [
[[package]]
name = "garage_model"
version = "0.9.0"
version = "0.9.1"
dependencies = [
"arc-swap",
"async-trait",
@@ -1364,7 +1365,7 @@ dependencies = [
[[package]]
name = "garage_rpc"
version = "0.9.0"
version = "0.9.1"
dependencies = [
"arc-swap",
"async-trait",
@@ -1399,7 +1400,7 @@ dependencies = [
[[package]]
name = "garage_table"
version = "0.9.0"
version = "0.9.1"
dependencies = [
"arc-swap",
"async-trait",
@@ -1421,7 +1422,7 @@ dependencies = [
[[package]]
name = "garage_util"
version = "0.9.0"
version = "0.9.1"
dependencies = [
"arc-swap",
"async-trait",
@@ -1455,7 +1456,7 @@ dependencies = [
[[package]]
name = "garage_web"
version = "0.9.0"
version = "0.9.1"
dependencies = [
"err-derive",
"futures",
+61 -60
View File
@@ -33,7 +33,7 @@ args@{
ignoreLockHash,
}:
let
nixifiedLockHash = "1a87886681a3ef0b83c95addc26674a538b8a93d35bc80db8998e1fcd0821f6c";
nixifiedLockHash = "f8da48e74313b18ee372a07d6a7d63a27c7effde24ed8599dfb91ea311df10f8";
workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc;
currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock);
lockHashIgnored = if ignoreLockHash
@@ -57,16 +57,16 @@ in
{
cargo2nixVersion = "0.11.0";
workspace = {
garage_db = rustPackages.unknown.garage_db."0.9.0";
garage_util = rustPackages.unknown.garage_util."0.9.0";
garage_rpc = rustPackages.unknown.garage_rpc."0.9.0";
garage_db = rustPackages.unknown.garage_db."0.9.1";
garage_util = rustPackages.unknown.garage_util."0.9.1";
garage_rpc = rustPackages.unknown.garage_rpc."0.9.1";
format_table = rustPackages.unknown.format_table."0.1.1";
garage_table = rustPackages.unknown.garage_table."0.9.0";
garage_block = rustPackages.unknown.garage_block."0.9.0";
garage_model = rustPackages.unknown.garage_model."0.9.0";
garage_api = rustPackages.unknown.garage_api."0.9.0";
garage_web = rustPackages.unknown.garage_web."0.9.0";
garage = rustPackages.unknown.garage."0.9.0";
garage_table = rustPackages.unknown.garage_table."0.9.1";
garage_block = rustPackages.unknown.garage_block."0.9.1";
garage_model = rustPackages.unknown.garage_model."0.9.1";
garage_api = rustPackages.unknown.garage_api."0.9.1";
garage_web = rustPackages.unknown.garage_web."0.9.1";
garage = rustPackages.unknown.garage."0.9.1";
k2v-client = rustPackages.unknown.k2v-client."0.0.4";
};
"registry+https://github.com/rust-lang/crates.io-index".addr2line."0.21.0" = overridableMkRustCrate (profileName: rec {
@@ -1705,9 +1705,9 @@ in
};
});
"unknown".garage."0.9.0" = overridableMkRustCrate (profileName: rec {
"unknown".garage."0.9.1" = overridableMkRustCrate (profileName: rec {
name = "garage";
version = "0.9.0";
version = "0.9.1";
registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/garage");
features = builtins.concatLists [
@@ -1734,14 +1734,14 @@ in
format_table = (rustPackages."unknown".format_table."0.1.1" { inherit profileName; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out;
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out;
garage_api = (rustPackages."unknown".garage_api."0.9.0" { inherit profileName; }).out;
garage_block = (rustPackages."unknown".garage_block."0.9.0" { inherit profileName; }).out;
garage_db = (rustPackages."unknown".garage_db."0.9.0" { inherit profileName; }).out;
garage_model = (rustPackages."unknown".garage_model."0.9.0" { inherit profileName; }).out;
garage_rpc = (rustPackages."unknown".garage_rpc."0.9.0" { inherit profileName; }).out;
garage_table = (rustPackages."unknown".garage_table."0.9.0" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.9.0" { inherit profileName; }).out;
garage_web = (rustPackages."unknown".garage_web."0.9.0" { inherit profileName; }).out;
garage_api = (rustPackages."unknown".garage_api."0.9.1" { inherit profileName; }).out;
garage_block = (rustPackages."unknown".garage_block."0.9.1" { inherit profileName; }).out;
garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out;
garage_model = (rustPackages."unknown".garage_model."0.9.1" { inherit profileName; }).out;
garage_rpc = (rustPackages."unknown".garage_rpc."0.9.1" { inherit profileName; }).out;
garage_table = (rustPackages."unknown".garage_table."0.9.1" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out;
garage_web = (rustPackages."unknown".garage_web."0.9.1" { inherit profileName; }).out;
git_version = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
sodiumoxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }).out;
@@ -1771,15 +1771,16 @@ in
http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out;
hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.27" { inherit profileName; }).out;
k2v_client = (rustPackages."unknown".k2v-client."0.0.4" { inherit profileName; }).out;
mktemp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".mktemp."0.5.0" { inherit profileName; }).out;
serde_json = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.105" { inherit profileName; }).out;
sha2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.10.7" { inherit profileName; }).out;
static_init = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".static_init."1.0.3" { inherit profileName; }).out;
};
});
"unknown".garage_api."0.9.0" = overridableMkRustCrate (profileName: rec {
"unknown".garage_api."0.9.1" = overridableMkRustCrate (profileName: rec {
name = "garage_api";
version = "0.9.0";
version = "0.9.1";
registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/api");
features = builtins.concatLists [
@@ -1798,11 +1799,11 @@ in
form_urlencoded = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.2.0" { inherit profileName; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out;
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out;
garage_block = (rustPackages."unknown".garage_block."0.9.0" { inherit profileName; }).out;
garage_model = (rustPackages."unknown".garage_model."0.9.0" { inherit profileName; }).out;
garage_rpc = (rustPackages."unknown".garage_rpc."0.9.0" { inherit profileName; }).out;
garage_table = (rustPackages."unknown".garage_table."0.9.0" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.9.0" { inherit profileName; }).out;
garage_block = (rustPackages."unknown".garage_block."0.9.1" { inherit profileName; }).out;
garage_model = (rustPackages."unknown".garage_model."0.9.1" { inherit profileName; }).out;
garage_rpc = (rustPackages."unknown".garage_rpc."0.9.1" { inherit profileName; }).out;
garage_table = (rustPackages."unknown".garage_table."0.9.1" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
hmac = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.12.1" { inherit profileName; }).out;
http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out;
@@ -1832,9 +1833,9 @@ in
};
});
"unknown".garage_block."0.9.0" = overridableMkRustCrate (profileName: rec {
"unknown".garage_block."0.9.1" = overridableMkRustCrate (profileName: rec {
name = "garage_block";
version = "0.9.0";
version = "0.9.1";
registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/block");
features = builtins.concatLists [
@@ -1848,10 +1849,10 @@ in
bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.3.0" { inherit profileName; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out;
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out;
garage_db = (rustPackages."unknown".garage_db."0.9.0" { inherit profileName; }).out;
garage_rpc = (rustPackages."unknown".garage_rpc."0.9.0" { inherit profileName; }).out;
garage_table = (rustPackages."unknown".garage_table."0.9.0" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.9.0" { inherit profileName; }).out;
garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out;
garage_rpc = (rustPackages."unknown".garage_rpc."0.9.1" { inherit profileName; }).out;
garage_table = (rustPackages."unknown".garage_table."0.9.1" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
@@ -1864,9 +1865,9 @@ in
};
});
"unknown".garage_db."0.9.0" = overridableMkRustCrate (profileName: rec {
"unknown".garage_db."0.9.1" = overridableMkRustCrate (profileName: rec {
name = "garage_db";
version = "0.9.0";
version = "0.9.1";
registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/db");
features = builtins.concatLists [
@@ -1896,9 +1897,9 @@ in
};
});
"unknown".garage_model."0.9.0" = overridableMkRustCrate (profileName: rec {
"unknown".garage_model."0.9.1" = overridableMkRustCrate (profileName: rec {
name = "garage_model";
version = "0.9.0";
version = "0.9.1";
registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/model");
features = builtins.concatLists [
@@ -1917,11 +1918,11 @@ in
err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out;
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out;
garage_block = (rustPackages."unknown".garage_block."0.9.0" { inherit profileName; }).out;
garage_db = (rustPackages."unknown".garage_db."0.9.0" { inherit profileName; }).out;
garage_rpc = (rustPackages."unknown".garage_rpc."0.9.0" { inherit profileName; }).out;
garage_table = (rustPackages."unknown".garage_table."0.9.0" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.9.0" { inherit profileName; }).out;
garage_block = (rustPackages."unknown".garage_block."0.9.1" { inherit profileName; }).out;
garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out;
garage_rpc = (rustPackages."unknown".garage_rpc."0.9.1" { inherit profileName; }).out;
garage_table = (rustPackages."unknown".garage_table."0.9.1" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.10.0" { inherit profileName; }).out;
opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
@@ -1934,9 +1935,9 @@ in
};
});
"unknown".garage_rpc."0.9.0" = overridableMkRustCrate (profileName: rec {
"unknown".garage_rpc."0.9.1" = overridableMkRustCrate (profileName: rec {
name = "garage_rpc";
version = "0.9.0";
version = "0.9.1";
registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/rpc");
features = builtins.concatLists [
@@ -1958,8 +1959,8 @@ in
format_table = (rustPackages."unknown".format_table."0.1.1" { inherit profileName; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out;
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out;
garage_db = (rustPackages."unknown".garage_db."0.9.0" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.9.0" { inherit profileName; }).out;
garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out;
gethostname = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.4.3" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
itertools = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".itertools."0.10.5" { inherit profileName; }).out;
@@ -1982,9 +1983,9 @@ in
};
});
"unknown".garage_table."0.9.0" = overridableMkRustCrate (profileName: rec {
"unknown".garage_table."0.9.1" = overridableMkRustCrate (profileName: rec {
name = "garage_table";
version = "0.9.0";
version = "0.9.1";
registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/table");
dependencies = {
@@ -1993,9 +1994,9 @@ in
bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out;
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out;
garage_db = (rustPackages."unknown".garage_db."0.9.0" { inherit profileName; }).out;
garage_rpc = (rustPackages."unknown".garage_rpc."0.9.0" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.9.0" { inherit profileName; }).out;
garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out;
garage_rpc = (rustPackages."unknown".garage_rpc."0.9.1" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out;
opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
@@ -2007,9 +2008,9 @@ in
};
});
"unknown".garage_util."0.9.0" = overridableMkRustCrate (profileName: rec {
"unknown".garage_util."0.9.1" = overridableMkRustCrate (profileName: rec {
name = "garage_util";
version = "0.9.0";
version = "0.9.1";
registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/util");
features = builtins.concatLists [
@@ -2025,7 +2026,7 @@ in
digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.7" { inherit profileName; }).out;
err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out;
garage_db = (rustPackages."unknown".garage_db."0.9.0" { inherit profileName; }).out;
garage_db = (rustPackages."unknown".garage_db."0.9.1" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out;
http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out;
@@ -2051,18 +2052,18 @@ in
};
});
"unknown".garage_web."0.9.0" = overridableMkRustCrate (profileName: rec {
"unknown".garage_web."0.9.1" = overridableMkRustCrate (profileName: rec {
name = "garage_web";
version = "0.9.0";
version = "0.9.1";
registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/web");
dependencies = {
err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out;
garage_api = (rustPackages."unknown".garage_api."0.9.0" { inherit profileName; }).out;
garage_model = (rustPackages."unknown".garage_model."0.9.0" { inherit profileName; }).out;
garage_table = (rustPackages."unknown".garage_table."0.9.0" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.9.0" { inherit profileName; }).out;
garage_api = (rustPackages."unknown".garage_api."0.9.1" { inherit profileName; }).out;
garage_model = (rustPackages."unknown".garage_model."0.9.1" { inherit profileName; }).out;
garage_table = (rustPackages."unknown".garage_table."0.9.1" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.9.1" { inherit profileName; }).out;
http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out;
hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.27" { inherit profileName; }).out;
hyperlocal = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyperlocal."0.8.0" { inherit profileName; }).out;
+8 -8
View File
@@ -18,14 +18,14 @@ default-members = ["src/garage"]
[workspace.dependencies]
format_table = { version = "0.1.1", path = "src/format-table" }
garage_api = { version = "0.9.0", path = "src/api" }
garage_block = { version = "0.9.0", path = "src/block" }
garage_db = { version = "0.9.0", path = "src/db", default-features = false }
garage_model = { version = "0.9.0", path = "src/model", default-features = false }
garage_rpc = { version = "0.9.0", path = "src/rpc" }
garage_table = { version = "0.9.0", path = "src/table" }
garage_util = { version = "0.9.0", path = "src/util" }
garage_web = { version = "0.9.0", path = "src/web" }
garage_api = { version = "0.9.1", path = "src/api" }
garage_block = { version = "0.9.1", path = "src/block" }
garage_db = { version = "0.9.1", path = "src/db", default-features = false }
garage_model = { version = "0.9.1", path = "src/model", default-features = false }
garage_rpc = { version = "0.9.1", path = "src/rpc" }
garage_table = { version = "0.9.1", path = "src/table" }
garage_util = { version = "0.9.1", path = "src/util" }
garage_web = { version = "0.9.1", path = "src/web" }
k2v-client = { version = "0.0.4", path = "src/k2v-client" }
[profile.dev]
+2 -2
View File
@@ -146,7 +146,7 @@ Keep the Key ID and the Secret key in a pad, they will be needed later.
We need two buckets, one for normal videos (named peertube-video) and one for webtorrent videos (named peertube-playlist).
```bash
garage bucket create peertube-video
garage bucket create peertube-videos
garage bucket create peertube-playlist
```
@@ -216,7 +216,7 @@ object_storage:
# Same settings but for webtorrent videos
videos:
bucket_name: 'peertube-video'
bucket_name: 'peertube-videos'
prefix: ''
# You must fill this field to make Peertube use our reverse proxy/website logic
base_url: 'http://peertube-videos.web.garage.localhost'
+5 -5
View File
@@ -85,14 +85,14 @@ to store 2 TB of data in total.
## Get a Docker image
Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
We encourage you to use a fixed tag (eg. `v0.9.0`) and not the `latest` tag.
For this example, we will use the latest published version at the time of the writing which is `v0.9.0` but it's up to you
We encourage you to use a fixed tag (eg. `v0.9.1`) and not the `latest` tag.
For this example, we will use the latest published version at the time of the writing which is `v0.9.1` but it's up to you
to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
For example:
```
sudo docker pull dxflrs/garage:v0.9.0
sudo docker pull dxflrs/garage:v0.9.1
```
## Deploying and configuring Garage
@@ -157,7 +157,7 @@ docker run \
-v /etc/garage.toml:/etc/garage.toml \
-v /var/lib/garage/meta:/var/lib/garage/meta \
-v /var/lib/garage/data:/var/lib/garage/data \
dxflrs/garage:v0.9.0
dxflrs/garage:v0.9.1
```
With this command line, Garage should be started automatically at each boot.
@@ -171,7 +171,7 @@ If you want to use `docker-compose`, you may use the following `docker-compose.y
version: "3"
services:
garage:
image: dxflrs/garage:v0.9.0
image: dxflrs/garage:v0.9.1
network_mode: "host"
restart: unless-stopped
volumes:
+19 -3
View File
@@ -394,7 +394,7 @@ Compression is done synchronously, setting a value too high will add latency to
This value can be different between nodes, compression is done by the node which receive the
API call.
#### `rpc_secret`, `rpc_secret_file` or `GARAGE_RPC_SECRET` (env) {#rpc_secret}
#### `rpc_secret`, `rpc_secret_file` or `GARAGE_RPC_SECRET`, `GARAGE_RPC_SECRET_FILE` (env) {#rpc_secret}
Garage uses a secret key, called an RPC secret, that is shared between all
nodes of the cluster in order to identify these nodes and allow them to
@@ -406,6 +406,9 @@ Since Garage `v0.8.2`, the RPC secret can also be stored in a file whose path is
given in the configuration variable `rpc_secret_file`, or specified as an
environment variable `GARAGE_RPC_SECRET`.
Since Garage `v0.8.5` and `v0.9.1`, you can also specify the path of a file
storing the secret as the `GARAGE_RPC_SECRET_FILE` environment variable.
#### `rpc_bind_addr` {#rpc_bind_addr}
The address and port on which to bind for inter-cluster communcations
@@ -438,6 +441,17 @@ be obtained by running `garage node id` and then included directly in the
key will be returned by `garage node id` and you will have to add the IP
yourself.
### `allow_world_readable_secrets`
Garage checks the permissions of your secret files to make sure they're not
world-readable. In some cases, the check might fail and consider your files as
world-readable even if they're not, for instance when using Posix ACLs.
Setting `allow_world_readable_secrets` to `true` bypass this
permission verification.
Alternatively, you can set the `GARAGE_ALLOW_WORLD_READABLE_SECRETS`
environment variable to `true` to bypass the permissions check.
### The `[consul_discovery]` section
@@ -583,7 +597,7 @@ See [administration API reference](@/documentation/reference-manual/admin-api.md
Alternatively, since `v0.8.5`, a path can be used to create a unix socket. Note that for security reasons,
the socket will have 0220 mode. Make sure to set user and group permissions accordingly.
#### `metrics_token`, `metrics_token_file` or `GARAGE_METRICS_TOKEN` (env) {#admin_metrics_token}
#### `metrics_token`, `metrics_token_file` or `GARAGE_METRICS_TOKEN`, `GARAGE_METRICS_TOKEN_FILE` (env) {#admin_metrics_token}
The token for accessing the Metrics endpoint. If this token is not set, the
Metrics endpoint can be accessed without access control.
@@ -593,8 +607,9 @@ You can use any random string for this value. We recommend generating a random t
`metrics_token` was introduced in Garage `v0.7.2`.
`metrics_token_file` and the `GARAGE_METRICS_TOKEN` environment variable are supported since Garage `v0.8.2`.
`GARAGE_METRICS_TOKEN_FILE` is supported since `v0.8.5` / `v0.9.1`.
#### `admin_token`, `admin_token_file` or `GARAGE_ADMIN_TOKEN` (env) {#admin_token}
#### `admin_token`, `admin_token_file` or `GARAGE_ADMIN_TOKEN`, `GARAGE_ADMIN_TOKEN_FILE` (env) {#admin_token}
The token for accessing all of the other administration endpoints. If this
token is not set, access to these endpoints is disabled entirely.
@@ -604,6 +619,7 @@ You can use any random string for this value. We recommend generating a random t
`admin_token` was introduced in Garage `v0.7.2`.
`admin_token_file` and the `GARAGE_ADMIN_TOKEN` environment variable are supported since Garage `v0.8.2`.
`GARAGE_ADMIN_TOKEN_FILE` is supported since `v0.8.5` / `v0.9.1`.
#### `trace_sink` {#admin_trace_sink}
+1 -1
View File
@@ -21,4 +21,4 @@ version: 0.4.1
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "v0.9.0"
appVersion: "v0.9.1"
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_api"
version = "0.9.0"
version = "0.9.1"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+10 -3
View File
@@ -15,6 +15,7 @@ use serde::Deserialize;
use garage_model::garage::Garage;
use crate::s3::cors::*;
use crate::s3::error::*;
use crate::s3::put::{get_headers, save_stream};
use crate::s3::xml as s3_xml;
@@ -242,7 +243,7 @@ pub async fn handle_post_object(
let etag = format!("\"{}\"", md5);
let resp = if let Some(mut target) = params
let mut resp = if let Some(mut target) = params
.get("success_action_redirect")
.and_then(|h| h.to_str().ok())
.and_then(|u| url::Url::parse(u).ok())
@@ -262,8 +263,7 @@ pub async fn handle_post_object(
} else {
let path = head
.uri
.into_parts()
.path_and_query
.path_and_query()
.map(|paq| paq.path().to_string())
.unwrap_or_else(|| "/".to_string());
let authority = head
@@ -308,6 +308,13 @@ pub async fn handle_post_object(
}
};
let matching_cors_rule =
find_matching_cors_rule(&bucket, &Request::from_parts(head, Body::empty()))?;
if let Some(rule) = matching_cors_rule {
add_cors_headers(&mut resp, rule)
.ok_or_internal_error("Invalid bucket CORS configuration")?;
}
Ok(resp)
}
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_block"
version = "0.9.0"
version = "0.9.1"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_db"
version = "0.9.0"
version = "0.9.1"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+2 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage"
version = "0.9.0"
version = "0.9.1"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
@@ -67,6 +67,7 @@ chrono = "0.4"
http = "0.2"
hmac = "0.12"
hyper = { version = "0.14", features = ["client", "http1", "runtime"] }
mktemp = "0.5"
sha2 = "0.10"
static_init = "1.0"
+1 -1
View File
@@ -43,7 +43,7 @@ pub fn node_id_command(config_file: PathBuf, quiet: bool) -> Result<(), Error> {
idstr
);
eprintln!(
"where <remote_node> is their own node identifier in the format: <pubkey>@<ip>:<port>"
"where <remote_node> is their own node identifier in the format: <full-node-id>@<ip>:<port>"
);
eprintln!();
eprintln!("This node identifier can also be added as a bootstrap node in other node's garage.toml files:");
+5 -3
View File
@@ -64,7 +64,8 @@ pub enum Command {
#[derive(StructOpt, Debug)]
pub enum NodeOperation {
/// Print identifier (public key) of this Garage node
/// Print the full node ID (public key) of this Garage node, and its publicly reachable IP
/// address and port if they are specified in config file under `rpc_public_addr`
#[structopt(name = "id", version = garage_version())]
NodeId(NodeIdOpt),
@@ -82,8 +83,9 @@ pub struct NodeIdOpt {
#[derive(StructOpt, Debug)]
pub struct ConnectNodeOpt {
/// Node public key and address, in the format:
/// `<public key hexadecimal>@<ip or hostname>:<port>`
/// Full node ID (public key) and IP address and port, in the format:
/// `<full node ID>@<ip or hostname>:<port>`.
/// You can retrieve this information on the target node using `garage node id`.
pub(crate) node: String,
}
+5 -36
View File
@@ -7,6 +7,7 @@ extern crate tracing;
mod admin;
mod cli;
mod repair;
mod secrets;
mod server;
#[cfg(feature = "telemetry-otlp")]
mod tracing_setup;
@@ -28,7 +29,6 @@ use structopt::StructOpt;
use netapp::util::parse_and_resolve_peer_addr;
use netapp::NetworkKey;
use garage_util::config::Config;
use garage_util::error::*;
use garage_rpc::system::*;
@@ -38,6 +38,7 @@ use garage_model::helper::error::Error as HelperError;
use admin::*;
use cli::*;
use secrets::Secrets;
#[derive(StructOpt, Debug)]
#[structopt(
@@ -45,8 +46,7 @@ use cli::*;
about = "S3-compatible object store for self-hosted geo-distributed deployments"
)]
struct Opt {
/// Host to connect to for admin operations, in the format:
/// <public-key>@<ip>:<port>
/// Host to connect to for admin operations, in the format: <full-node-id>@<ip>:<port>
#[structopt(short = "h", long = "rpc-host", env = "GARAGE_RPC_HOST")]
pub rpc_host: Option<String>,
@@ -66,24 +66,6 @@ struct Opt {
cmd: Command,
}
#[derive(StructOpt, Debug)]
pub struct Secrets {
/// RPC secret network key, used to replace rpc_secret in config.toml when running the
/// daemon or doing admin operations
#[structopt(short = "s", long = "rpc-secret", env = "GARAGE_RPC_SECRET")]
pub rpc_secret: Option<String>,
/// Metrics API authentication token, replaces admin.metrics_token in config.toml when
/// running the Garage daemon
#[structopt(long = "admin-token", env = "GARAGE_ADMIN_TOKEN")]
pub admin_token: Option<String>,
/// Metrics API authentication token, replaces admin.metrics_token in config.toml when
/// running the Garage daemon
#[structopt(long = "metrics-token", env = "GARAGE_METRICS_TOKEN")]
pub metrics_token: Option<String>,
}
#[tokio::main]
async fn main() {
// Initialize version and features info
@@ -218,7 +200,7 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
// Find and parse the address of the target host
let (id, addr, is_default_addr) = if let Some(h) = opt.rpc_host {
let (id, addrs) = parse_and_resolve_peer_addr(&h).ok_or_else(|| format!("Invalid RPC remote node identifier: {}. Expected format is <pubkey>@<IP or hostname>:<port>.", h))?;
let (id, addrs) = parse_and_resolve_peer_addr(&h).ok_or_else(|| format!("Invalid RPC remote node identifier: {}. Expected format is <full node id>@<IP or hostname>:<port>.", h))?;
(id, addrs[0], false)
} else {
let node_id = garage_rpc::system::read_node_id(&config.as_ref().unwrap().metadata_dir)
@@ -248,7 +230,7 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
addr
);
}
Err(e).err_context("Unable to connect to destination RPC host. Check that you are using the same value of rpc_secret as them, and that you have their correct public key.")?;
Err(e).err_context("Unable to connect to destination RPC host. Check that you are using the same value of rpc_secret as them, and that you have their correct full-length node ID (public key).")?;
}
let system_rpc_endpoint = netapp.endpoint::<SystemRpc, ()>(SYSTEM_RPC_PATH.into());
@@ -261,16 +243,3 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
Ok(x) => Ok(x),
}
}
fn fill_secrets(mut config: Config, secrets: Secrets) -> Config {
if secrets.rpc_secret.is_some() {
config.rpc_secret = secrets.rpc_secret;
}
if secrets.admin_token.is_some() {
config.admin.admin_token = secrets.admin_token;
}
if secrets.metrics_token.is_some() {
config.admin.metrics_token = secrets.metrics_token;
}
config
}
+2 -2
View File
@@ -6,7 +6,7 @@ use garage_util::error::*;
use garage_model::garage::Garage;
use crate::cli::structs::*;
use crate::{fill_secrets, Secrets};
use crate::secrets::{fill_secrets, Secrets};
pub async fn offline_repair(
config_file: PathBuf,
@@ -20,7 +20,7 @@ pub async fn offline_repair(
}
info!("Loading configuration...");
let config = fill_secrets(read_config(config_file)?, secrets);
let config = fill_secrets(read_config(config_file)?, secrets)?;
info!("Initializing Garage main data store...");
let garage = Garage::new(config)?;
+318
View File
@@ -0,0 +1,318 @@
use structopt::StructOpt;
use garage_util::config::Config;
use garage_util::error::Error;
/// Structure for secret values or paths that are passed as CLI arguments or environment
/// variables, instead of in the config file.
#[derive(StructOpt, Debug, Default, Clone)]
pub struct Secrets {
/// Skip permission check on files containing secrets
#[cfg(unix)]
#[structopt(
long = "allow-world-readable-secrets",
env = "GARAGE_ALLOW_WORLD_READABLE_SECRETS"
)]
pub allow_world_readable_secrets: Option<bool>,
/// RPC secret network key, used to replace rpc_secret in config.toml when running the
/// daemon or doing admin operations
#[structopt(short = "s", long = "rpc-secret", env = "GARAGE_RPC_SECRET")]
pub rpc_secret: Option<String>,
/// RPC secret network key, used to replace rpc_secret in config.toml and rpc-secret
/// when running the daemon or doing admin operations
#[structopt(long = "rpc-secret-file", env = "GARAGE_RPC_SECRET_FILE")]
pub rpc_secret_file: Option<String>,
/// Admin API authentication token, replaces admin.admin_token in config.toml when
/// running the Garage daemon
#[structopt(long = "admin-token", env = "GARAGE_ADMIN_TOKEN")]
pub admin_token: Option<String>,
/// Admin API authentication token file path, replaces admin.admin_token in config.toml
/// and admin-token when running the Garage daemon
#[structopt(long = "admin-token-file", env = "GARAGE_ADMIN_TOKEN_FILE")]
pub admin_token_file: Option<String>,
/// Metrics API authentication token, replaces admin.metrics_token in config.toml when
/// running the Garage daemon
#[structopt(long = "metrics-token", env = "GARAGE_METRICS_TOKEN")]
pub metrics_token: Option<String>,
/// Metrics API authentication token file path, replaces admin.metrics_token in config.toml
/// and metrics-token when running the Garage daemon
#[structopt(long = "metrics-token-file", env = "GARAGE_METRICS_TOKEN_FILE")]
pub metrics_token_file: Option<String>,
}
/// Single function to fill all secrets in the Config struct from their correct source (value
/// from config or CLI param or env variable or read from a file specified in config or CLI
/// param or env variable)
pub fn fill_secrets(mut config: Config, secrets: Secrets) -> Result<Config, Error> {
let allow_world_readable = secrets
.allow_world_readable_secrets
.unwrap_or(config.allow_world_readable_secrets);
fill_secret(
&mut config.rpc_secret,
&config.rpc_secret_file,
&secrets.rpc_secret,
&secrets.rpc_secret_file,
"rpc_secret",
allow_world_readable,
)?;
fill_secret(
&mut config.admin.admin_token,
&config.admin.admin_token_file,
&secrets.admin_token,
&secrets.admin_token_file,
"admin.admin_token",
allow_world_readable,
)?;
fill_secret(
&mut config.admin.metrics_token,
&config.admin.metrics_token_file,
&secrets.metrics_token,
&secrets.metrics_token_file,
"admin.metrics_token",
allow_world_readable,
)?;
Ok(config)
}
fn fill_secret(
config_secret: &mut Option<String>,
config_secret_file: &Option<String>,
cli_secret: &Option<String>,
cli_secret_file: &Option<String>,
name: &'static str,
allow_world_readable: bool,
) -> Result<(), Error> {
let cli_value = match (&cli_secret, &cli_secret_file) {
(Some(_), Some(_)) => {
return Err(format!("only one of `{}` and `{}_file` can be set", name, name).into());
}
(Some(secret), None) => Some(secret.to_string()),
(None, Some(file)) => Some(read_secret_file(file, allow_world_readable)?),
(None, None) => None,
};
if let Some(val) = cli_value {
if config_secret.is_some() || config_secret_file.is_some() {
debug!("Overriding secret `{}` using value specified using CLI argument or environnement variable.", name);
}
*config_secret = Some(val);
} else if let Some(file_path) = &config_secret_file {
if config_secret.is_some() {
return Err(format!("only one of `{}` and `{}_file` can be set", name, name).into());
}
*config_secret = Some(read_secret_file(file_path, allow_world_readable)?);
}
Ok(())
}
fn read_secret_file(file_path: &String, allow_world_readable: bool) -> Result<String, Error> {
if !allow_world_readable {
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
let metadata = std::fs::metadata(file_path)?;
if metadata.mode() & 0o077 != 0 {
return Err(format!("File {} is world-readable! (mode: 0{:o}, expected 0600)\nRefusing to start until this is fixed, or environment variable GARAGE_ALLOW_WORLD_READABLE_SECRETS is set to true.", file_path, metadata.mode()).into());
}
}
}
let secret_buf = std::fs::read_to_string(file_path)?;
// trim_end: allows for use case such as `echo "$(openssl rand -hex 32)" > somefile`.
// also editors sometimes add a trailing newline
Ok(String::from(secret_buf.trim_end()))
}
#[cfg(test)]
mod tests {
use std::fs::File;
use std::io::Write;
use garage_util::config::read_config;
use garage_util::error::Error;
use super::*;
#[test]
fn test_rpc_secret_file_works() -> Result<(), Error> {
let path_secret = mktemp::Temp::new_file()?;
let mut file_secret = File::create(path_secret.as_path())?;
writeln!(file_secret, "foo")?;
drop(file_secret);
let path_config = mktemp::Temp::new_file()?;
let mut file_config = File::create(path_config.as_path())?;
let path_secret_path = path_secret.as_path();
writeln!(
file_config,
r#"
metadata_dir = "/tmp/garage/meta"
data_dir = "/tmp/garage/data"
replication_mode = "3"
rpc_bind_addr = "[::]:3901"
rpc_secret_file = "{}"
[s3_api]
s3_region = "garage"
api_bind_addr = "[::]:3900"
"#,
path_secret_path.display()
)?;
drop(file_config);
// Second configuration file, same as previous one
// except it allows world-readable secrets.
let path_config_allow_world_readable = mktemp::Temp::new_file()?;
let mut file_config_allow_world_readable =
File::create(path_config_allow_world_readable.as_path())?;
writeln!(
file_config_allow_world_readable,
r#"
metadata_dir = "/tmp/garage/meta"
data_dir = "/tmp/garage/data"
replication_mode = "3"
rpc_bind_addr = "[::]:3901"
rpc_secret_file = "{}"
allow_world_readable_secrets = true
[s3_api]
s3_region = "garage"
api_bind_addr = "[::]:3900"
"#,
path_secret_path.display()
)?;
drop(file_config_allow_world_readable);
let config = read_config(path_config.to_path_buf())?;
let config = fill_secrets(config, Secrets::default())?;
assert_eq!("foo", config.rpc_secret.unwrap());
// ---- Check non world-readable secrets config ----
#[cfg(unix)]
{
let secrets_allow_world_readable = Secrets {
allow_world_readable_secrets: Some(true),
..Default::default()
};
let secrets_no_allow_world_readable = Secrets {
allow_world_readable_secrets: Some(false),
..Default::default()
};
use std::os::unix::fs::PermissionsExt;
let metadata = std::fs::metadata(&path_secret_path)?;
let mut perm = metadata.permissions();
perm.set_mode(0o660);
std::fs::set_permissions(&path_secret_path, perm)?;
// Config file that just specifies the path
let config = read_config(path_config.to_path_buf())?;
assert!(fill_secrets(config, Secrets::default()).is_err());
let config = read_config(path_config.to_path_buf())?;
assert!(fill_secrets(config, secrets_allow_world_readable.clone()).is_ok());
let config = read_config(path_config.to_path_buf())?;
assert!(fill_secrets(config, secrets_no_allow_world_readable.clone()).is_err());
// Config file that also specifies to allow world_readable_secrets
let config = read_config(path_config_allow_world_readable.to_path_buf())?;
assert!(fill_secrets(config, Secrets::default()).is_ok());
let config = read_config(path_config_allow_world_readable.to_path_buf())?;
assert!(fill_secrets(config, secrets_allow_world_readable).is_ok());
let config = read_config(path_config_allow_world_readable.to_path_buf())?;
assert!(fill_secrets(config, secrets_no_allow_world_readable).is_err());
}
// ---- Check alternative secrets specified on CLI ----
let path_secret2 = mktemp::Temp::new_file()?;
let mut file_secret2 = File::create(path_secret2.as_path())?;
writeln!(file_secret2, "bar")?;
drop(file_secret2);
let config = read_config(path_config.to_path_buf())?;
let config = fill_secrets(
config,
Secrets {
rpc_secret: Some("baz".into()),
..Default::default()
},
)?;
assert_eq!(config.rpc_secret.as_deref(), Some("baz"));
let config = read_config(path_config.to_path_buf())?;
let config = fill_secrets(
config,
Secrets {
rpc_secret_file: Some(path_secret2.display().to_string()),
..Default::default()
},
)?;
assert_eq!(config.rpc_secret.as_deref(), Some("bar"));
let config = read_config(path_config.to_path_buf())?;
assert!(fill_secrets(
config,
Secrets {
rpc_secret: Some("baz".into()),
rpc_secret_file: Some(path_secret2.display().to_string()),
..Default::default()
}
)
.is_err());
drop(path_secret);
drop(path_secret2);
drop(path_config);
drop(path_config_allow_world_readable);
Ok(())
}
#[test]
fn test_rcp_secret_and_rpc_secret_file_cannot_be_set_both() -> Result<(), Error> {
let path_config = mktemp::Temp::new_file()?;
let mut file_config = File::create(path_config.as_path())?;
writeln!(
file_config,
r#"
metadata_dir = "/tmp/garage/meta"
data_dir = "/tmp/garage/data"
replication_mode = "3"
rpc_bind_addr = "[::]:3901"
rpc_secret= "dummy"
rpc_secret_file = "dummy"
[s3_api]
s3_region = "garage"
api_bind_addr = "[::]:3900"
"#
)?;
let config = read_config(path_config.to_path_buf())?;
assert_eq!(
"only one of `rpc_secret` and `rpc_secret_file` can be set",
fill_secrets(config, Secrets::default())
.unwrap_err()
.to_string()
);
drop(path_config);
drop(file_config);
Ok(())
}
}
+10 -3
View File
@@ -15,9 +15,9 @@ use garage_web::WebServer;
use garage_api::k2v::api_server::K2VApiServer;
use crate::admin::*;
use crate::secrets::{fill_secrets, Secrets};
#[cfg(feature = "telemetry-otlp")]
use crate::tracing_setup::*;
use crate::{fill_secrets, Secrets};
async fn wait_from(mut chan: watch::Receiver<bool>) {
while !*chan.borrow() {
@@ -29,12 +29,19 @@ async fn wait_from(mut chan: watch::Receiver<bool>) {
pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Error> {
info!("Loading configuration...");
let config = fill_secrets(read_config(config_file)?, secrets);
let config = fill_secrets(read_config(config_file)?, secrets)?;
// ---- Initialize Garage internals ----
#[cfg(feature = "metrics")]
let metrics_exporter = opentelemetry_prometheus::exporter().init();
let metrics_exporter = opentelemetry_prometheus::exporter()
.with_default_summary_quantiles(vec![0.25, 0.5, 0.75, 0.9, 0.95, 0.99])
.with_default_histogram_boundaries(vec![
0.001, 0.0015, 0.002, 0.003, 0.005, 0.007, 0.01, 0.015, 0.02, 0.03, 0.05, 0.07, 0.1,
0.15, 0.2, 0.3, 0.5, 0.7, 1., 1.5, 2., 3., 5., 7., 10., 15., 20., 30., 40., 50., 60.,
70., 100.,
])
.init();
info!("Initializing Garage main data store...");
let garage = Garage::new(config.clone())?;
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_model"
version = "0.9.0"
version = "0.9.1"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_rpc"
version = "0.9.0"
version = "0.9.1"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+1 -1
View File
@@ -148,7 +148,7 @@ impl ConsulDiscovery {
ret.push((pubkey, SocketAddr::new(ip, ent.service_port)));
} else {
warn!(
"Could not process node spec from Consul: {:?} (invalid IP or public key)",
"Could not process node spec from Consul: {:?} (invalid IP address or node ID/pubkey)",
ent
);
}
+1 -1
View File
@@ -57,7 +57,7 @@ pub const SYSTEM_RPC_PATH: &str = "garage_rpc/membership.rs/SystemRpc";
pub enum SystemRpc {
/// Response to successfull advertisements
Ok,
/// Request to connect to a specific node (in <pubkey>@<host>:<port> format)
/// Request to connect to a specific node (in <pubkey>@<host>:<port> format, pubkey = full-length node ID)
Connect(String),
/// Ask other node its cluster layout. Answered with AdvertiseClusterLayout
PullClusterLayout,
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_table"
version = "0.9.0"
version = "0.9.1"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_util"
version = "0.9.0"
version = "0.9.1"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
+12 -143
View File
@@ -1,6 +1,5 @@
//! Contains type and functions related to Garage configuration file
use std::convert::TryFrom;
use std::io::Read;
use std::net::SocketAddr;
use std::path::PathBuf;
@@ -45,11 +44,15 @@ pub struct Config {
)]
pub compression_level: Option<i32>,
/// Skip the permission check of secret files. Useful when
/// POSIX ACLs (or more complex chmods) are used.
#[serde(default)]
pub allow_world_readable_secrets: bool,
/// RPC secret key: 32 bytes hex encoded
pub rpc_secret: Option<String>,
/// Optional file where RPC secret key is read from
pub rpc_secret_file: Option<String>,
/// Address to bind for RPC
pub rpc_bind_addr: SocketAddr,
/// Public IP address of this node
@@ -221,6 +224,13 @@ pub struct KubernetesDiscoveryConfig {
pub skip_crd: bool,
}
/// Read and parse configuration
pub fn read_config(config_file: PathBuf) -> Result<Config, Error> {
let config = std::fs::read_to_string(config_file)?;
Ok(toml::from_str(&config)?)
}
fn default_db_engine() -> String {
"lmdb".into()
}
@@ -235,68 +245,6 @@ fn default_block_size() -> usize {
1048576
}
/// Read and parse configuration
pub fn read_config(config_file: PathBuf) -> Result<Config, Error> {
let mut file = std::fs::OpenOptions::new()
.read(true)
.open(config_file.as_path())?;
let mut config = String::new();
file.read_to_string(&mut config)?;
let mut parsed_config: Config = toml::from_str(&config)?;
secret_from_file(
&mut parsed_config.rpc_secret,
&parsed_config.rpc_secret_file,
"rpc_secret",
)?;
secret_from_file(
&mut parsed_config.admin.metrics_token,
&parsed_config.admin.metrics_token_file,
"admin.metrics_token",
)?;
secret_from_file(
&mut parsed_config.admin.admin_token,
&parsed_config.admin.admin_token_file,
"admin.admin_token",
)?;
Ok(parsed_config)
}
fn secret_from_file(
secret: &mut Option<String>,
secret_file: &Option<String>,
name: &'static str,
) -> Result<(), Error> {
match (&secret, &secret_file) {
(_, None) => {
// no-op
}
(Some(_), Some(_)) => {
return Err(format!("only one of `{}` and `{}_file` can be set", name, name).into());
}
(None, Some(file_path)) => {
#[cfg(unix)]
if std::env::var("GARAGE_ALLOW_WORLD_READABLE_SECRETS").as_deref() != Ok("true") {
use std::os::unix::fs::MetadataExt;
let metadata = std::fs::metadata(file_path)?;
if metadata.mode() & 0o077 != 0 {
return Err(format!("File {} is world-readable! (mode: 0{:o}, expected 0600)\nRefusing to start until this is fixed, or environment variable GARAGE_ALLOW_WORLD_READABLE_SECRETS is set to true.", file_path, metadata.mode()).into());
}
}
let mut file = std::fs::OpenOptions::new().read(true).open(file_path)?;
let mut secret_buf = String::new();
file.read_to_string(&mut secret_buf)?;
// trim_end: allows for use case such as `echo "$(openssl rand -hex 32)" > somefile`.
// also editors sometimes add a trailing newline
*secret = Some(String::from(secret_buf.trim_end()));
}
}
Ok(())
}
fn default_compression() -> Option<i32> {
Some(1)
}
@@ -425,83 +373,4 @@ mod tests {
Ok(())
}
#[test]
fn test_rpc_secret_file_works() -> Result<(), Error> {
let path_secret = mktemp::Temp::new_file()?;
let mut file_secret = File::create(path_secret.as_path())?;
writeln!(file_secret, "foo")?;
drop(file_secret);
let path_config = mktemp::Temp::new_file()?;
let mut file_config = File::create(path_config.as_path())?;
let path_secret_path = path_secret.as_path();
writeln!(
file_config,
r#"
metadata_dir = "/tmp/garage/meta"
data_dir = "/tmp/garage/data"
replication_mode = "3"
rpc_bind_addr = "[::]:3901"
rpc_secret_file = "{}"
[s3_api]
s3_region = "garage"
api_bind_addr = "[::]:3900"
"#,
path_secret_path.display()
)?;
let config = super::read_config(path_config.to_path_buf())?;
assert_eq!("foo", config.rpc_secret.unwrap());
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let metadata = std::fs::metadata(&path_secret_path)?;
let mut perm = metadata.permissions();
perm.set_mode(0o660);
std::fs::set_permissions(&path_secret_path, perm)?;
std::env::set_var("GARAGE_ALLOW_WORLD_READABLE_SECRETS", "false");
assert!(super::read_config(path_config.to_path_buf()).is_err());
std::env::set_var("GARAGE_ALLOW_WORLD_READABLE_SECRETS", "true");
assert!(super::read_config(path_config.to_path_buf()).is_ok());
}
drop(path_config);
drop(path_secret);
drop(file_config);
Ok(())
}
#[test]
fn test_rcp_secret_and_rpc_secret_file_cannot_be_set_both() -> Result<(), Error> {
let path_config = mktemp::Temp::new_file()?;
let mut file_config = File::create(path_config.as_path())?;
writeln!(
file_config,
r#"
metadata_dir = "/tmp/garage/meta"
data_dir = "/tmp/garage/data"
replication_mode = "3"
rpc_bind_addr = "[::]:3901"
rpc_secret= "dummy"
rpc_secret_file = "dummy"
[s3_api]
s3_region = "garage"
api_bind_addr = "[::]:3900"
"#
)?;
assert_eq!(
"only one of `rpc_secret` and `rpc_secret_file` can be set",
super::read_config(path_config.to_path_buf())
.unwrap_err()
.to_string()
);
drop(path_config);
drop(file_config);
Ok(())
}
}
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "garage_web"
version = "0.9.0"
version = "0.9.1"
authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
edition = "2018"
license = "AGPL-3.0"