mirror of
https://github.com/deuxfleurs-org/garage.git
synced 2026-08-14 16:13:11 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Quentin Dufour
|
1be1715522 |
+7
-7
@@ -14,7 +14,7 @@ steps:
|
||||
- name: build
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
commands:
|
||||
- nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-build --no-build-output --attr clippy.amd64 --argstr git_version $DRONE_COMMIT
|
||||
|
||||
- name: unit + func tests
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
@@ -34,7 +34,7 @@ steps:
|
||||
- name: integration tests
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
commands:
|
||||
- nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-build --no-build-output --attr clippy.amd64 --argstr git_version $DRONE_COMMIT
|
||||
- nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
|
||||
|
||||
trigger:
|
||||
@@ -57,7 +57,7 @@ steps:
|
||||
- name: build
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
commands:
|
||||
- nix-build --no-build-output --attr pkgs.amd64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-build --no-build-output --attr pkgs.amd64.release --argstr git_version $DRONE_COMMIT
|
||||
- nix-shell --attr rust --run "./script/not-dynamic.sh result/bin/garage"
|
||||
|
||||
- name: integration
|
||||
@@ -108,7 +108,7 @@ steps:
|
||||
- name: build
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
commands:
|
||||
- nix-build --no-build-output --attr pkgs.i386.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-build --no-build-output --attr pkgs.i386.release --argstr git_version $DRONE_COMMIT
|
||||
- nix-shell --attr rust --run "./script/not-dynamic.sh result/bin/garage"
|
||||
|
||||
- name: integration
|
||||
@@ -158,7 +158,7 @@ steps:
|
||||
- name: build
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
commands:
|
||||
- nix-build --no-build-output --attr pkgs.arm64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-build --no-build-output --attr pkgs.arm64.release --argstr git_version $DRONE_COMMIT
|
||||
- nix-shell --attr rust --run "./script/not-dynamic.sh result/bin/garage"
|
||||
|
||||
- name: push static binary
|
||||
@@ -203,7 +203,7 @@ steps:
|
||||
- name: build
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
commands:
|
||||
- nix-build --no-build-output --attr pkgs.arm.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-build --no-build-output --attr pkgs.arm.release --argstr git_version $DRONE_COMMIT
|
||||
- nix-shell --attr rust --run "./script/not-dynamic.sh result/bin/garage"
|
||||
|
||||
- name: push static binary
|
||||
@@ -269,6 +269,6 @@ trigger:
|
||||
|
||||
---
|
||||
kind: signature
|
||||
hmac: 362639b4c9541ad9bd06ff7f72b5235b2b0216bcb16eececd25285b6fe94ba6f
|
||||
hmac: ccec0e06bf6676a705d6a4e63dc322691148bc72e72a4aaa54be6c5fd22652a2
|
||||
|
||||
...
|
||||
|
||||
Generated
+1
-1
@@ -1176,6 +1176,7 @@ dependencies = [
|
||||
"futures-util",
|
||||
"garage_util 0.7.0",
|
||||
"gethostname",
|
||||
"git-version",
|
||||
"hex",
|
||||
"hyper",
|
||||
"k8s-openapi",
|
||||
@@ -1275,7 +1276,6 @@ dependencies = [
|
||||
"err-derive 0.3.1",
|
||||
"futures",
|
||||
"garage_db",
|
||||
"git-version",
|
||||
"hex",
|
||||
"http",
|
||||
"hyper",
|
||||
|
||||
@@ -55,7 +55,7 @@ in
|
||||
garage_api = rustPackages.unknown.garage_api."0.7.0";
|
||||
garage_web = rustPackages.unknown.garage_web."0.7.0";
|
||||
garage = rustPackages.unknown.garage."0.7.0";
|
||||
k2v-client = rustPackages.unknown.k2v-client."0.0.1";
|
||||
k2v-client = rustPackages.unknown.k2v-client."0.1.0";
|
||||
};
|
||||
"registry+https://github.com/rust-lang/crates.io-index".ahash."0.7.6" = overridableMkRustCrate (profileName: rec {
|
||||
name = "ahash";
|
||||
@@ -749,7 +749,7 @@ in
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b"; };
|
||||
dependencies = {
|
||||
${ if hostPlatform.config == "aarch64-linux-android" || hostPlatform.config == "aarch64-apple-darwin" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; };
|
||||
${ if hostPlatform.config == "aarch64-linux-android" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" || hostPlatform.config == "aarch64-apple-darwin" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; };
|
||||
};
|
||||
});
|
||||
|
||||
@@ -1625,6 +1625,7 @@ in
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "futures_util" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "garage_util" else null } = rustPackages."unknown".garage_util."0.7.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "gethostname" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.2.3" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "git_version" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "hex" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "hyper" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.18" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" then "k8s_openapi" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.13.1" { inherit profileName; };
|
||||
@@ -1735,7 +1736,6 @@ in
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "err_derive" else null } = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "futures" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "garage_db" else null } = rustPackages."unknown".garage_db."0.8.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "git_version" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "hex" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "http" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "hyper" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.18" { inherit profileName; };
|
||||
@@ -2327,9 +2327,9 @@ in
|
||||
};
|
||||
});
|
||||
|
||||
"unknown".k2v-client."0.0.1" = overridableMkRustCrate (profileName: rec {
|
||||
"unknown".k2v-client."0.1.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "k2v-client";
|
||||
version = "0.0.1";
|
||||
version = "0.1.0";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/k2v-client");
|
||||
features = builtins.concatLists [
|
||||
@@ -2756,7 +2756,7 @@ in
|
||||
[ "os-poll" ]
|
||||
];
|
||||
dependencies = {
|
||||
${ if hostPlatform.parsed.kernel.name == "wasi" || hostPlatform.isUnix then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; };
|
||||
${ if hostPlatform.isUnix || hostPlatform.parsed.kernel.name == "wasi" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; };
|
||||
log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; };
|
||||
${ if hostPlatform.isWindows then "miow" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".miow."0.3.7" { inherit profileName; };
|
||||
${ if hostPlatform.isWindows then "ntapi" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".ntapi."0.3.7" { inherit profileName; };
|
||||
@@ -4534,7 +4534,7 @@ in
|
||||
];
|
||||
dependencies = {
|
||||
bitflags = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; };
|
||||
${ if hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; };
|
||||
${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; };
|
||||
${ if !(hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android") then "parking_lot" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.11.2" { inherit profileName; };
|
||||
${ if !(hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android") then "parking_lot_core" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot_core."0.8.5" { inherit profileName; };
|
||||
static_init_macro = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".static_init_macro."1.0.2" { profileName = "__noProfile"; };
|
||||
@@ -5554,11 +5554,11 @@ in
|
||||
[ "default" ]
|
||||
];
|
||||
dependencies = {
|
||||
${ if hostPlatform.config == "aarch64-uwp-windows-msvc" || hostPlatform.config == "aarch64-pc-windows-msvc" then "windows_aarch64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.32.0" { inherit profileName; };
|
||||
${ if hostPlatform.config == "aarch64-pc-windows-msvc" || hostPlatform.config == "aarch64-uwp-windows-msvc" then "windows_aarch64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.32.0" { inherit profileName; };
|
||||
${ if hostPlatform.config == "i686-uwp-windows-gnu" || hostPlatform.config == "i686-pc-windows-gnu" then "windows_i686_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.32.0" { inherit profileName; };
|
||||
${ if hostPlatform.config == "i686-pc-windows-msvc" || hostPlatform.config == "i686-uwp-windows-msvc" then "windows_i686_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.32.0" { inherit profileName; };
|
||||
${ if hostPlatform.config == "i686-uwp-windows-msvc" || hostPlatform.config == "i686-pc-windows-msvc" then "windows_i686_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.32.0" { inherit profileName; };
|
||||
${ if hostPlatform.config == "x86_64-pc-windows-gnu" || hostPlatform.config == "x86_64-uwp-windows-gnu" then "windows_x86_64_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.32.0" { inherit profileName; };
|
||||
${ if hostPlatform.config == "x86_64-pc-windows-msvc" || hostPlatform.config == "x86_64-uwp-windows-msvc" then "windows_x86_64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.32.0" { inherit profileName; };
|
||||
${ if hostPlatform.config == "x86_64-uwp-windows-msvc" || hostPlatform.config == "x86_64-pc-windows-msvc" then "windows_x86_64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.32.0" { inherit profileName; };
|
||||
};
|
||||
});
|
||||
|
||||
|
||||
+4
-4
@@ -9,8 +9,8 @@ let
|
||||
pkgs = import pkgsSrc { };
|
||||
compile = import ./nix/compile.nix;
|
||||
build_debug_and_release = (target: {
|
||||
debug = (compile { inherit target git_version; release = false; }).workspace.garage { compileMode = "build"; };
|
||||
release = (compile { inherit target git_version; release = true; }).workspace.garage { compileMode = "build"; };
|
||||
debug = (compile { inherit target; release = false; }).workspace.garage { compileMode = "build"; };
|
||||
release = (compile { inherit target; release = true; }).workspace.garage { compileMode = "build"; };
|
||||
});
|
||||
test = (rustPkgs: pkgs.symlinkJoin {
|
||||
name ="garage-tests";
|
||||
@@ -25,9 +25,9 @@ in {
|
||||
arm = build_debug_and_release "armv6l-unknown-linux-musleabihf";
|
||||
};
|
||||
test = {
|
||||
amd64 = test (compile { inherit git_version; target = "x86_64-unknown-linux-musl"; });
|
||||
amd64 = test (compile { target = "x86_64-unknown-linux-musl"; });
|
||||
};
|
||||
clippy = {
|
||||
amd64 = (compile { inherit git_version; compiler = "clippy"; }).workspace.garage { compileMode = "build"; } ;
|
||||
amd64 = (compile { compiler = "clippy"; }).workspace.garage { compileMode = "build"; } ;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
+++
|
||||
title = "Benchmarks"
|
||||
weight = 60
|
||||
sort_by = "weight"
|
||||
template = "documentation.html"
|
||||
+++
|
||||
|
||||
Hello
|
||||
@@ -0,0 +1,17 @@
|
||||
+++
|
||||
title = "Abstraction cost"
|
||||
weight = 30
|
||||
+++
|
||||
|
||||
We take as our baseline the raw disk sequential write performance.
|
||||
We then compare Garage's performances to it, the difference represents what we call our "abstraction cost".
|
||||
|
||||
fsync, chunking, compression, pipelining, synchronization
|
||||
|
||||
# raw perf VS garage
|
||||
|
||||
# garage tmpfs VS garage std
|
||||
|
||||
# garage various multipart sizes
|
||||
|
||||
# garage 0.7.2.1 VS upstream
|
||||
@@ -0,0 +1,18 @@
|
||||
+++
|
||||
title = "Failure & recovery"
|
||||
weight = 50
|
||||
+++
|
||||
|
||||
# Failure impact
|
||||
|
||||
Failures will lead to timeouts, which in turn could
|
||||
lead to failed requests (this is a bug if failure enters in Garage tolerance)
|
||||
and to increased latency as some retries might be performed.
|
||||
|
||||
How we proceed: we pause (`kill -STOP xxx`) one Garage process.
|
||||
The idea is we don't want to close the TCP connection that would signal too easily
|
||||
that a crash occured. Instead, we want to simulate a network error
|
||||
or an overloaded process, ie. a 'non-collaborating' crash.
|
||||
|
||||
|
||||
# Recovery impact
|
||||
@@ -0,0 +1,14 @@
|
||||
+++
|
||||
title = "Industry tools"
|
||||
weight = 60
|
||||
+++
|
||||
|
||||
|
||||
# minio warp
|
||||
|
||||
# intel-cloud cosbench
|
||||
|
||||
# (non retenu)
|
||||
|
||||
- wasabi s3-benchmark
|
||||
- https://github.com/dvassallo/s3-benchmark
|
||||
@@ -0,0 +1,10 @@
|
||||
+++
|
||||
title = "Liveness"
|
||||
weight = 40
|
||||
+++
|
||||
|
||||
freedom from starvation, backpressure, etc.
|
||||
|
||||
# Responsiveness under read/write load
|
||||
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
+++
|
||||
title = "Network sensitiveness"
|
||||
weight = 10
|
||||
+++
|
||||
|
||||
impact of node count and their latency
|
||||
|
||||
# Latency amplification
|
||||
|
||||
# Cluster size impact
|
||||
|
||||
# Time-To-First-Byte (TTFB)
|
||||
|
||||
with various object size
|
||||
@@ -0,0 +1,16 @@
|
||||
+++
|
||||
title = "Pushing limits"
|
||||
weight = 60
|
||||
+++
|
||||
|
||||
# Many objects
|
||||
|
||||
# Huge objects
|
||||
|
||||
# Many nodes (horizontal scalability)
|
||||
|
||||
|
||||
|
||||
# Large nodes (vertical scalability)
|
||||
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
+++
|
||||
title = "Real world"
|
||||
weight = 80
|
||||
+++
|
||||
|
||||
# Nextcloud
|
||||
|
||||
# Peertube
|
||||
@@ -280,25 +280,3 @@ Traefik's caching middleware is only available on [entreprise version](https://d
|
||||
[http.middlewares]
|
||||
[http.middlewares.gzip_compress.compress]
|
||||
```
|
||||
|
||||
## Caddy
|
||||
|
||||
Your Caddy configuration can be as simple as:
|
||||
|
||||
```caddy
|
||||
s3.garage.tld, *.s3.garage.tld {
|
||||
reverse_proxy localhost:3900 192.168.1.2:3900 example.tld:3900
|
||||
}
|
||||
|
||||
*.web.garage.tld {
|
||||
reverse_proxy localhost:3902 192.168.1.2:3900 example.tld:3900
|
||||
}
|
||||
|
||||
admin.garage.tld {
|
||||
reverse_proxy localhost:3903
|
||||
}
|
||||
```
|
||||
|
||||
But at the same time, the `reverse_proxy` is very flexible.
|
||||
For a production deployment, you should [read its documentation](https://caddyserver.com/docs/caddyfile/directives/reverse_proxy) as it supports features like DNS discovery of upstreams, load balancing with checks, streaming parameters, etc.
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
+++
|
||||
title = "Development"
|
||||
weight = 6
|
||||
weight = 70
|
||||
sort_by = "weight"
|
||||
template = "documentation.html"
|
||||
+++
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
+++
|
||||
title = "Working Documents"
|
||||
weight = 7
|
||||
weight = 80
|
||||
sort_by = "weight"
|
||||
template = "documentation.html"
|
||||
+++
|
||||
|
||||
+12
-12
@@ -118,7 +118,6 @@ let
|
||||
But we want to ship these additional features when we release Garage.
|
||||
In the end, we chose to exclude all features from debug builds while putting (all of) them in the release builds.
|
||||
Currently, the only feature of Garage is kubernetes-discovery from the garage_rpc crate.
|
||||
The experimental feature k2v is also enabled here in all builds.
|
||||
*/
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
name = "garage";
|
||||
@@ -130,9 +129,18 @@ let
|
||||
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
name = "garage_rpc";
|
||||
overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
|
||||
overrideAttrs = drv:
|
||||
(if git_version != null then {
|
||||
/* [3] */ preConfigure = ''
|
||||
${drv.preConfigure or ""}
|
||||
export GIT_VERSION="${git_version}"
|
||||
'';
|
||||
} else {})
|
||||
// {
|
||||
/* [1] */ setBuildEnv = (buildEnv drv);
|
||||
};
|
||||
overrideArgs = old: {
|
||||
/* [4] */ features = if release then [ "kubernetes-discovery" "k2v" ] else [ "k2v" ];
|
||||
/* [4] */ features = if release then [ "kubernetes-discovery" ] else [];
|
||||
};
|
||||
})
|
||||
|
||||
@@ -143,15 +151,7 @@ let
|
||||
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
name = "garage_util";
|
||||
overrideAttrs = drv:
|
||||
(if git_version != null then {
|
||||
/* [3] */ preConfigure = ''
|
||||
${drv.preConfigure or ""}
|
||||
export GIT_VERSION="${git_version}"
|
||||
'';
|
||||
} else {})
|
||||
//
|
||||
{ /* [1] */ setBuildEnv = (buildEnv drv); };
|
||||
overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
|
||||
})
|
||||
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
|
||||
@@ -3,7 +3,6 @@ extern crate tracing;
|
||||
|
||||
pub mod manager;
|
||||
pub mod repair;
|
||||
pub mod resync;
|
||||
|
||||
mod block;
|
||||
mod metrics;
|
||||
|
||||
+511
-56
@@ -1,19 +1,32 @@
|
||||
use std::convert::TryInto;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use arc_swap::ArcSwapOption;
|
||||
use async_trait::async_trait;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use futures::future::*;
|
||||
use tokio::fs;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::sync::{mpsc, Mutex};
|
||||
use tokio::select;
|
||||
use tokio::sync::{mpsc, watch, Mutex, Notify};
|
||||
|
||||
use opentelemetry::{
|
||||
trace::{FutureExt as OtelFutureExt, TraceContextExt, Tracer},
|
||||
Context, KeyValue,
|
||||
};
|
||||
|
||||
use garage_db as db;
|
||||
use garage_db::counted_tree_hack::CountedTree;
|
||||
|
||||
use garage_util::background::*;
|
||||
use garage_util::data::*;
|
||||
use garage_util::error::*;
|
||||
use garage_util::metrics::RecordDuration;
|
||||
use garage_util::time::*;
|
||||
use garage_util::tranquilizer::Tranquilizer;
|
||||
|
||||
use garage_rpc::system::System;
|
||||
use garage_rpc::*;
|
||||
@@ -24,13 +37,23 @@ use crate::block::*;
|
||||
use crate::metrics::*;
|
||||
use crate::rc::*;
|
||||
use crate::repair::*;
|
||||
use crate::resync::*;
|
||||
|
||||
/// Size under which data will be stored inlined in database instead of as files
|
||||
pub const INLINE_THRESHOLD: usize = 3072;
|
||||
|
||||
// Timeout for RPCs that read and write blocks to remote nodes
|
||||
pub(crate) const BLOCK_RW_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
const BLOCK_RW_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
// Timeout for RPCs that ask other nodes whether they need a copy
|
||||
// of a given block before we delete it locally
|
||||
const NEED_BLOCK_QUERY_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
// The delay between the time where a resync operation fails
|
||||
// and the time when it is retried, with exponential backoff
|
||||
// (multiplied by 2, 4, 8, 16, etc. for every consecutive failure).
|
||||
const RESYNC_RETRY_DELAY: Duration = Duration::from_secs(60);
|
||||
// The minimum retry delay is 60 seconds = 1 minute
|
||||
// The maximum retry delay is 60 seconds * 2^6 = 60 seconds << 6 = 64 minutes (~1 hour)
|
||||
const RESYNC_RETRY_DELAY_MAX_BACKOFF_POWER: u64 = 6;
|
||||
|
||||
// The delay between the moment when the reference counter
|
||||
// drops to zero, and the moment where we allow ourselves
|
||||
@@ -67,18 +90,22 @@ pub struct BlockManager {
|
||||
pub data_dir: PathBuf,
|
||||
|
||||
compression_level: Option<i32>,
|
||||
background_tranquility: u32,
|
||||
|
||||
mutation_lock: Mutex<BlockManagerLocked>,
|
||||
|
||||
pub(crate) rc: BlockRc,
|
||||
pub resync: BlockResyncManager,
|
||||
|
||||
resync_queue: CountedTree,
|
||||
resync_notify: Notify,
|
||||
resync_errors: CountedTree,
|
||||
|
||||
pub(crate) system: Arc<System>,
|
||||
pub(crate) endpoint: Arc<Endpoint<BlockRpc, Self>>,
|
||||
endpoint: Arc<Endpoint<BlockRpc, Self>>,
|
||||
|
||||
pub(crate) metrics: BlockManagerMetrics,
|
||||
metrics: BlockManagerMetrics,
|
||||
|
||||
tx_scrub_command: mpsc::Sender<ScrubWorkerCommand>,
|
||||
tx_scrub_command: ArcSwapOption<mpsc::Sender<ScrubWorkerCommand>>,
|
||||
}
|
||||
|
||||
// This custom struct contains functions that must only be ran
|
||||
@@ -86,11 +113,18 @@ pub struct BlockManager {
|
||||
// it INSIDE a Mutex.
|
||||
struct BlockManagerLocked();
|
||||
|
||||
enum ResyncIterResult {
|
||||
BusyDidSomething,
|
||||
BusyDidNothing,
|
||||
IdleFor(Duration),
|
||||
}
|
||||
|
||||
impl BlockManager {
|
||||
pub fn new(
|
||||
db: &db::Db,
|
||||
data_dir: PathBuf,
|
||||
compression_level: Option<i32>,
|
||||
background_tranquility: u32,
|
||||
replication: TableShardedReplication,
|
||||
system: Arc<System>,
|
||||
) -> Arc<Self> {
|
||||
@@ -99,7 +133,17 @@ impl BlockManager {
|
||||
.expect("Unable to open block_local_rc tree");
|
||||
let rc = BlockRc::new(rc);
|
||||
|
||||
let resync = BlockResyncManager::new(db, &system);
|
||||
let resync_queue = db
|
||||
.open_tree("block_local_resync_queue")
|
||||
.expect("Unable to open block_local_resync_queue tree");
|
||||
let resync_queue =
|
||||
CountedTree::new(resync_queue).expect("Could not count block_local_resync_queue");
|
||||
|
||||
let resync_errors = db
|
||||
.open_tree("block_local_resync_errors")
|
||||
.expect("Unable to open block_local_resync_errors tree");
|
||||
let resync_errors =
|
||||
CountedTree::new(resync_errors).expect("Could not count block_local_resync_errors");
|
||||
|
||||
let endpoint = system
|
||||
.netapp
|
||||
@@ -107,39 +151,32 @@ impl BlockManager {
|
||||
|
||||
let manager_locked = BlockManagerLocked();
|
||||
|
||||
let metrics = BlockManagerMetrics::new(resync.queue.clone(), resync.errors.clone());
|
||||
|
||||
let (scrub_tx, scrub_rx) = mpsc::channel(1);
|
||||
let metrics = BlockManagerMetrics::new(resync_queue.clone(), resync_errors.clone());
|
||||
|
||||
let block_manager = Arc::new(Self {
|
||||
replication,
|
||||
data_dir,
|
||||
compression_level,
|
||||
background_tranquility,
|
||||
mutation_lock: Mutex::new(manager_locked),
|
||||
rc,
|
||||
resync,
|
||||
resync_queue,
|
||||
resync_notify: Notify::new(),
|
||||
resync_errors,
|
||||
system,
|
||||
endpoint,
|
||||
metrics,
|
||||
tx_scrub_command: scrub_tx,
|
||||
tx_scrub_command: ArcSwapOption::new(None),
|
||||
});
|
||||
block_manager.endpoint.set_handler(block_manager.clone());
|
||||
|
||||
// Spawn a bunch of resync workers
|
||||
for index in 0..MAX_RESYNC_WORKERS {
|
||||
let worker = ResyncWorker::new(index, block_manager.clone());
|
||||
block_manager.system.background.spawn_worker(worker);
|
||||
}
|
||||
|
||||
// Spawn scrub worker
|
||||
let scrub_worker = ScrubWorker::new(block_manager.clone(), scrub_rx);
|
||||
block_manager.system.background.spawn_worker(scrub_worker);
|
||||
block_manager.clone().spawn_background_workers();
|
||||
|
||||
block_manager
|
||||
}
|
||||
|
||||
/// Ask nodes that might have a (possibly compressed) block for it
|
||||
pub(crate) async fn rpc_get_raw_block(&self, hash: &Hash) -> Result<DataBlock, Error> {
|
||||
async fn rpc_get_raw_block(&self, hash: &Hash) -> Result<DataBlock, Error> {
|
||||
let who = self.replication.read_nodes(hash);
|
||||
let resps = self
|
||||
.system
|
||||
@@ -191,6 +228,20 @@ impl BlockManager {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get lenght of resync queue
|
||||
pub fn resync_queue_len(&self) -> Result<usize, Error> {
|
||||
// This currently can't return an error because the CountedTree hack
|
||||
// doesn't error on .len(), but this will change when we remove the hack
|
||||
// (hopefully someday!)
|
||||
Ok(self.resync_queue.len())
|
||||
}
|
||||
|
||||
/// Get number of blocks that have an error
|
||||
pub fn resync_errors_len(&self) -> Result<usize, Error> {
|
||||
// (see resync_queue_len comment)
|
||||
Ok(self.resync_errors.len())
|
||||
}
|
||||
|
||||
/// Get number of items in the refcount table
|
||||
pub fn rc_len(&self) -> Result<usize, Error> {
|
||||
Ok(self.rc.rc.len()?)
|
||||
@@ -198,7 +249,13 @@ impl BlockManager {
|
||||
|
||||
/// Send command to start/stop/manager scrub worker
|
||||
pub async fn send_scrub_command(&self, cmd: ScrubWorkerCommand) {
|
||||
let _ = self.tx_scrub_command.send(cmd).await;
|
||||
let _ = self
|
||||
.tx_scrub_command
|
||||
.load()
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.send(cmd)
|
||||
.await;
|
||||
}
|
||||
|
||||
//// ----- Managing the reference counter ----
|
||||
@@ -219,7 +276,7 @@ impl BlockManager {
|
||||
// we will fecth it from someone.
|
||||
let this = self.clone();
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = this.resync.put_to_resync(&hash, 2 * BLOCK_RW_TIMEOUT) {
|
||||
if let Err(e) = this.put_to_resync(&hash, 2 * BLOCK_RW_TIMEOUT) {
|
||||
error!("Block {:?} could not be put in resync queue: {}.", hash, e);
|
||||
}
|
||||
});
|
||||
@@ -241,9 +298,7 @@ impl BlockManager {
|
||||
// after that delay has passed.
|
||||
let this = self.clone();
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = this
|
||||
.resync
|
||||
.put_to_resync(&hash, BLOCK_GC_DELAY + Duration::from_secs(10))
|
||||
if let Err(e) = this.put_to_resync(&hash, BLOCK_GC_DELAY + Duration::from_secs(10))
|
||||
{
|
||||
error!("Block {:?} could not be put in resync queue: {}.", hash, e);
|
||||
}
|
||||
@@ -255,11 +310,7 @@ impl BlockManager {
|
||||
// ---- Reading and writing blocks locally ----
|
||||
|
||||
/// Write a block to disk
|
||||
pub(crate) async fn write_block(
|
||||
&self,
|
||||
hash: &Hash,
|
||||
data: &DataBlock,
|
||||
) -> Result<BlockRpc, Error> {
|
||||
async fn write_block(&self, hash: &Hash, data: &DataBlock) -> Result<BlockRpc, Error> {
|
||||
let write_size = data.inner_buffer().len() as u64;
|
||||
|
||||
let res = self
|
||||
@@ -295,7 +346,7 @@ impl BlockManager {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
// Not found but maybe we should have had it ??
|
||||
self.resync.put_to_resync(hash, 2 * BLOCK_RW_TIMEOUT)?;
|
||||
self.put_to_resync(hash, 2 * BLOCK_RW_TIMEOUT)?;
|
||||
return Err(Into::into(e));
|
||||
}
|
||||
};
|
||||
@@ -322,37 +373,24 @@ impl BlockManager {
|
||||
.await
|
||||
.move_block_to_corrupted(hash, self)
|
||||
.await?;
|
||||
self.resync.put_to_resync(hash, Duration::from_millis(0))?;
|
||||
self.put_to_resync(hash, Duration::from_millis(0))?;
|
||||
return Err(Error::CorruptData(*hash));
|
||||
}
|
||||
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
/// Check if this node has a block and whether it needs it
|
||||
pub(crate) async fn check_block_status(&self, hash: &Hash) -> Result<BlockStatus, Error> {
|
||||
self.mutation_lock
|
||||
/// Check if this node should have a block, but don't actually have it
|
||||
async fn need_block(&self, hash: &Hash) -> Result<bool, Error> {
|
||||
let BlockStatus { exists, needed } = self
|
||||
.mutation_lock
|
||||
.lock()
|
||||
.await
|
||||
.check_block_status(hash, self)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Check if this node should have a block, but don't actually have it
|
||||
async fn need_block(&self, hash: &Hash) -> Result<bool, Error> {
|
||||
let BlockStatus { exists, needed } = self.check_block_status(hash).await?;
|
||||
.await?;
|
||||
Ok(needed.is_nonzero() && !exists)
|
||||
}
|
||||
|
||||
/// Delete block if it is not needed anymore
|
||||
pub(crate) async fn delete_if_unneeded(&self, hash: &Hash) -> Result<(), Error> {
|
||||
self.mutation_lock
|
||||
.lock()
|
||||
.await
|
||||
.delete_if_unneeded(hash, self)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Utility: gives the path of the directory in which a block should be found
|
||||
fn block_dir(&self, hash: &Hash) -> PathBuf {
|
||||
let mut path = self.data_dir.clone();
|
||||
@@ -379,6 +417,305 @@ impl BlockManager {
|
||||
path.set_extension("");
|
||||
fs::metadata(&path).await.map(|_| false).map_err(Into::into)
|
||||
}
|
||||
|
||||
// ---- Resync loop ----
|
||||
|
||||
// This part manages a queue of blocks that need to be
|
||||
// "resynchronized", i.e. that need to have a check that
|
||||
// they are at present if we need them, or that they are
|
||||
// deleted once the garbage collection delay has passed.
|
||||
//
|
||||
// Here are some explanations on how the resync queue works.
|
||||
// There are two Sled trees that are used to have information
|
||||
// about the status of blocks that need to be resynchronized:
|
||||
//
|
||||
// - resync_queue: a tree that is ordered first by a timestamp
|
||||
// (in milliseconds since Unix epoch) that is the time at which
|
||||
// the resync must be done, and second by block hash.
|
||||
// The key in this tree is just:
|
||||
// concat(timestamp (8 bytes), hash (32 bytes))
|
||||
// The value is the same 32-byte hash.
|
||||
//
|
||||
// - resync_errors: a tree that indicates for each block
|
||||
// if the last resync resulted in an error, and if so,
|
||||
// the following two informations (see the ErrorCounter struct):
|
||||
// - how many consecutive resync errors for this block?
|
||||
// - when was the last try?
|
||||
// These two informations are used to implement an
|
||||
// exponential backoff retry strategy.
|
||||
// The key in this tree is the 32-byte hash of the block,
|
||||
// and the value is the encoded ErrorCounter value.
|
||||
//
|
||||
// We need to have these two trees, because the resync queue
|
||||
// is not just a queue of items to process, but a set of items
|
||||
// that are waiting a specific delay until we can process them
|
||||
// (the delay being necessary both internally for the exponential
|
||||
// backoff strategy, and exposed as a parameter when adding items
|
||||
// to the queue, e.g. to wait until the GC delay has passed).
|
||||
// This is why we need one tree ordered by time, and one
|
||||
// ordered by identifier of item to be processed (block hash).
|
||||
//
|
||||
// When the worker wants to process an item it takes from
|
||||
// resync_queue, it checks in resync_errors that if there is an
|
||||
// exponential back-off delay to await, it has passed before we
|
||||
// process the item. If not, the item in the queue is skipped
|
||||
// (but added back for later processing after the time of the
|
||||
// delay).
|
||||
//
|
||||
// An alternative that would have seemed natural is to
|
||||
// only add items to resync_queue with a processing time that is
|
||||
// after the delay, but there are several issues with this:
|
||||
// - This requires to synchronize updates to resync_queue and
|
||||
// resync_errors (with the current model, there is only one thread,
|
||||
// the worker thread, that accesses resync_errors,
|
||||
// so no need to synchronize) by putting them both in a lock.
|
||||
// This would mean that block_incref might need to take a lock
|
||||
// before doing its thing, meaning it has much more chances of
|
||||
// not completing successfully if something bad happens to Garage.
|
||||
// Currently Garage is not able to recover from block_incref that
|
||||
// doesn't complete successfully, because it is necessary to ensure
|
||||
// the consistency between the state of the block manager and
|
||||
// information in the BlockRef table.
|
||||
// - If a resync fails, we put that block in the resync_errors table,
|
||||
// and also add it back to resync_queue to be processed after
|
||||
// the exponential back-off delay,
|
||||
// but maybe the block is already scheduled to be resynced again
|
||||
// at another time that is before the exponential back-off delay,
|
||||
// and we have no way to check that easily. This means that
|
||||
// in all cases, we need to check the resync_errors table
|
||||
// in the resync loop at the time when a block is popped from
|
||||
// the resync_queue.
|
||||
// Overall, the current design is therefore simpler and more robust
|
||||
// because it tolerates inconsistencies between the resync_queue
|
||||
// and resync_errors table (items being scheduled in resync_queue
|
||||
// for times that are earlier than the exponential back-off delay
|
||||
// is a natural condition that is handled properly).
|
||||
|
||||
fn spawn_background_workers(self: Arc<Self>) {
|
||||
// Launch a background workers for background resync loop processing
|
||||
let background = self.system.background.clone();
|
||||
let worker = ResyncWorker::new(self.clone());
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(Duration::from_secs(10)).await;
|
||||
background.spawn_worker(worker);
|
||||
});
|
||||
|
||||
// Launch a background worker for data store scrubs
|
||||
let (scrub_tx, scrub_rx) = mpsc::channel(1);
|
||||
self.tx_scrub_command.store(Some(Arc::new(scrub_tx)));
|
||||
let scrub_worker = ScrubWorker::new(self.clone(), scrub_rx);
|
||||
self.system.background.spawn_worker(scrub_worker);
|
||||
}
|
||||
|
||||
pub(crate) fn put_to_resync(&self, hash: &Hash, delay: Duration) -> db::Result<()> {
|
||||
let when = now_msec() + delay.as_millis() as u64;
|
||||
self.put_to_resync_at(hash, when)
|
||||
}
|
||||
|
||||
fn put_to_resync_at(&self, hash: &Hash, when: u64) -> db::Result<()> {
|
||||
trace!("Put resync_queue: {} {:?}", when, hash);
|
||||
let mut key = u64::to_be_bytes(when).to_vec();
|
||||
key.extend(hash.as_ref());
|
||||
self.resync_queue.insert(key, hash.as_ref())?;
|
||||
self.resync_notify.notify_waiters();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn resync_iter(&self) -> Result<ResyncIterResult, db::Error> {
|
||||
if let Some((time_bytes, hash_bytes)) = self.resync_queue.first()? {
|
||||
let time_msec = u64::from_be_bytes(time_bytes[0..8].try_into().unwrap());
|
||||
let now = now_msec();
|
||||
|
||||
if now >= time_msec {
|
||||
let hash = Hash::try_from(&hash_bytes[..]).unwrap();
|
||||
|
||||
if let Some(ec) = self.resync_errors.get(hash.as_slice())? {
|
||||
let ec = ErrorCounter::decode(&ec);
|
||||
if now < ec.next_try() {
|
||||
// if next retry after an error is not yet,
|
||||
// don't do resync and return early, but still
|
||||
// make sure the item is still in queue at expected time
|
||||
self.put_to_resync_at(&hash, ec.next_try())?;
|
||||
// ec.next_try() > now >= time_msec, so this remove
|
||||
// is not removing the one we added just above
|
||||
// (we want to do the remove after the insert to ensure
|
||||
// that the item is not lost if we crash in-between)
|
||||
self.resync_queue.remove(time_bytes)?;
|
||||
return Ok(ResyncIterResult::BusyDidNothing);
|
||||
}
|
||||
}
|
||||
|
||||
let tracer = opentelemetry::global::tracer("garage");
|
||||
let trace_id = gen_uuid();
|
||||
let span = tracer
|
||||
.span_builder("Resync block")
|
||||
.with_trace_id(
|
||||
opentelemetry::trace::TraceId::from_hex(&hex::encode(
|
||||
&trace_id.as_slice()[..16],
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.with_attributes(vec![KeyValue::new("block", format!("{:?}", hash))])
|
||||
.start(&tracer);
|
||||
|
||||
let res = self
|
||||
.resync_block(&hash)
|
||||
.with_context(Context::current_with_span(span))
|
||||
.bound_record_duration(&self.metrics.resync_duration)
|
||||
.await;
|
||||
|
||||
self.metrics.resync_counter.add(1);
|
||||
|
||||
if let Err(e) = &res {
|
||||
self.metrics.resync_error_counter.add(1);
|
||||
warn!("Error when resyncing {:?}: {}", hash, e);
|
||||
|
||||
let err_counter = match self.resync_errors.get(hash.as_slice())? {
|
||||
Some(ec) => ErrorCounter::decode(&ec).add1(now + 1),
|
||||
None => ErrorCounter::new(now + 1),
|
||||
};
|
||||
|
||||
self.resync_errors
|
||||
.insert(hash.as_slice(), err_counter.encode())?;
|
||||
|
||||
self.put_to_resync_at(&hash, err_counter.next_try())?;
|
||||
// err_counter.next_try() >= now + 1 > now,
|
||||
// the entry we remove from the queue is not
|
||||
// the entry we inserted with put_to_resync_at
|
||||
self.resync_queue.remove(time_bytes)?;
|
||||
} else {
|
||||
self.resync_errors.remove(hash.as_slice())?;
|
||||
self.resync_queue.remove(time_bytes)?;
|
||||
}
|
||||
|
||||
Ok(ResyncIterResult::BusyDidSomething)
|
||||
} else {
|
||||
Ok(ResyncIterResult::IdleFor(Duration::from_millis(
|
||||
time_msec - now,
|
||||
)))
|
||||
}
|
||||
} else {
|
||||
// Here we wait either for a notification that an item has been
|
||||
// added to the queue, or for a constant delay of 10 secs to expire.
|
||||
// The delay avoids a race condition where the notification happens
|
||||
// between the time we checked the queue and the first poll
|
||||
// to resync_notify.notified(): if that happens, we'll just loop
|
||||
// back 10 seconds later, which is fine.
|
||||
Ok(ResyncIterResult::IdleFor(Duration::from_secs(10)))
|
||||
}
|
||||
}
|
||||
|
||||
async fn resync_block(&self, hash: &Hash) -> Result<(), Error> {
|
||||
let BlockStatus { exists, needed } = self
|
||||
.mutation_lock
|
||||
.lock()
|
||||
.await
|
||||
.check_block_status(hash, self)
|
||||
.await?;
|
||||
|
||||
if exists != needed.is_needed() || exists != needed.is_nonzero() {
|
||||
debug!(
|
||||
"Resync block {:?}: exists {}, nonzero rc {}, deletable {}",
|
||||
hash,
|
||||
exists,
|
||||
needed.is_nonzero(),
|
||||
needed.is_deletable(),
|
||||
);
|
||||
}
|
||||
|
||||
if exists && needed.is_deletable() {
|
||||
info!("Resync block {:?}: offloading and deleting", hash);
|
||||
|
||||
let mut who = self.replication.write_nodes(hash);
|
||||
if who.len() < self.replication.write_quorum() {
|
||||
return Err(Error::Message("Not trying to offload block because we don't have a quorum of nodes to write to".to_string()));
|
||||
}
|
||||
who.retain(|id| *id != self.system.id);
|
||||
|
||||
let msg = Arc::new(BlockRpc::NeedBlockQuery(*hash));
|
||||
let who_needs_fut = who.iter().map(|to| {
|
||||
self.system.rpc.call_arc(
|
||||
&self.endpoint,
|
||||
*to,
|
||||
msg.clone(),
|
||||
RequestStrategy::with_priority(PRIO_BACKGROUND)
|
||||
.with_timeout(NEED_BLOCK_QUERY_TIMEOUT),
|
||||
)
|
||||
});
|
||||
let who_needs_resps = join_all(who_needs_fut).await;
|
||||
|
||||
let mut need_nodes = vec![];
|
||||
for (node, needed) in who.iter().zip(who_needs_resps.into_iter()) {
|
||||
match needed.err_context("NeedBlockQuery RPC")? {
|
||||
BlockRpc::NeedBlockReply(needed) => {
|
||||
if needed {
|
||||
need_nodes.push(*node);
|
||||
}
|
||||
}
|
||||
m => {
|
||||
return Err(Error::unexpected_rpc_message(m));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !need_nodes.is_empty() {
|
||||
trace!(
|
||||
"Block {:?} needed by {} nodes, sending",
|
||||
hash,
|
||||
need_nodes.len()
|
||||
);
|
||||
|
||||
for node in need_nodes.iter() {
|
||||
self.metrics
|
||||
.resync_send_counter
|
||||
.add(1, &[KeyValue::new("to", format!("{:?}", node))]);
|
||||
}
|
||||
|
||||
let put_block_message = self.read_block(hash).await?;
|
||||
self.system
|
||||
.rpc
|
||||
.try_call_many(
|
||||
&self.endpoint,
|
||||
&need_nodes[..],
|
||||
put_block_message,
|
||||
RequestStrategy::with_priority(PRIO_BACKGROUND)
|
||||
.with_quorum(need_nodes.len())
|
||||
.with_timeout(BLOCK_RW_TIMEOUT),
|
||||
)
|
||||
.await
|
||||
.err_context("PutBlock RPC")?;
|
||||
}
|
||||
info!(
|
||||
"Deleting unneeded block {:?}, offload finished ({} / {})",
|
||||
hash,
|
||||
need_nodes.len(),
|
||||
who.len()
|
||||
);
|
||||
|
||||
self.mutation_lock
|
||||
.lock()
|
||||
.await
|
||||
.delete_if_unneeded(hash, self)
|
||||
.await?;
|
||||
|
||||
self.rc.clear_deleted_block_rc(hash)?;
|
||||
}
|
||||
|
||||
if needed.is_nonzero() && !exists {
|
||||
info!(
|
||||
"Resync block {:?}: fetching absent but needed block (refcount > 0)",
|
||||
hash
|
||||
);
|
||||
|
||||
let block_data = self.rpc_get_raw_block(hash).await?;
|
||||
|
||||
self.metrics.resync_recv_counter.add(1);
|
||||
|
||||
self.write_block(hash, &block_data).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -397,9 +734,80 @@ impl EndpointHandler<BlockRpc> for BlockManager {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct BlockStatus {
|
||||
pub(crate) exists: bool,
|
||||
pub(crate) needed: RcEntry,
|
||||
struct ResyncWorker {
|
||||
manager: Arc<BlockManager>,
|
||||
tranquilizer: Tranquilizer,
|
||||
next_delay: Duration,
|
||||
}
|
||||
|
||||
impl ResyncWorker {
|
||||
fn new(manager: Arc<BlockManager>) -> Self {
|
||||
Self {
|
||||
manager,
|
||||
tranquilizer: Tranquilizer::new(30),
|
||||
next_delay: Duration::from_secs(10),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Worker for ResyncWorker {
|
||||
fn name(&self) -> String {
|
||||
"Block resync worker".into()
|
||||
}
|
||||
|
||||
fn info(&self) -> Option<String> {
|
||||
let mut ret = vec![];
|
||||
let qlen = self.manager.resync_queue_len().unwrap_or(0);
|
||||
let elen = self.manager.resync_errors_len().unwrap_or(0);
|
||||
if qlen > 0 {
|
||||
ret.push(format!("{} blocks in queue", qlen));
|
||||
}
|
||||
if elen > 0 {
|
||||
ret.push(format!("{} blocks in error state", elen));
|
||||
}
|
||||
if !ret.is_empty() {
|
||||
Some(ret.join(", "))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
|
||||
self.tranquilizer.reset();
|
||||
match self.manager.resync_iter().await {
|
||||
Ok(ResyncIterResult::BusyDidSomething) => Ok(self
|
||||
.tranquilizer
|
||||
.tranquilize_worker(self.manager.background_tranquility)),
|
||||
Ok(ResyncIterResult::BusyDidNothing) => Ok(WorkerState::Busy),
|
||||
Ok(ResyncIterResult::IdleFor(delay)) => {
|
||||
self.next_delay = delay;
|
||||
Ok(WorkerState::Idle)
|
||||
}
|
||||
Err(e) => {
|
||||
// The errors that we have here are only Sled errors
|
||||
// We don't really know how to handle them so just ¯\_(ツ)_/¯
|
||||
// (there is kind of an assumption that Sled won't error on us,
|
||||
// if it does there is not much we can do -- TODO should we just panic?)
|
||||
// Here we just give the error to the worker manager,
|
||||
// it will print it to the logs and increment a counter
|
||||
Err(e.into())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
|
||||
select! {
|
||||
_ = tokio::time::sleep(self.next_delay) => (),
|
||||
_ = self.manager.resync_notify.notified() => (),
|
||||
};
|
||||
WorkerState::Busy
|
||||
}
|
||||
}
|
||||
|
||||
struct BlockStatus {
|
||||
exists: bool,
|
||||
needed: RcEntry,
|
||||
}
|
||||
|
||||
impl BlockManagerLocked {
|
||||
@@ -504,3 +912,50 @@ impl BlockManagerLocked {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Counts the number of errors when resyncing a block,
|
||||
/// and the time of the last try.
|
||||
/// Used to implement exponential backoff.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct ErrorCounter {
|
||||
errors: u64,
|
||||
last_try: u64,
|
||||
}
|
||||
|
||||
impl ErrorCounter {
|
||||
fn new(now: u64) -> Self {
|
||||
Self {
|
||||
errors: 1,
|
||||
last_try: now,
|
||||
}
|
||||
}
|
||||
|
||||
fn decode(data: &[u8]) -> Self {
|
||||
Self {
|
||||
errors: u64::from_be_bytes(data[0..8].try_into().unwrap()),
|
||||
last_try: u64::from_be_bytes(data[8..16].try_into().unwrap()),
|
||||
}
|
||||
}
|
||||
fn encode(&self) -> Vec<u8> {
|
||||
[
|
||||
u64::to_be_bytes(self.errors),
|
||||
u64::to_be_bytes(self.last_try),
|
||||
]
|
||||
.concat()
|
||||
}
|
||||
|
||||
fn add1(self, now: u64) -> Self {
|
||||
Self {
|
||||
errors: self.errors + 1,
|
||||
last_try: now,
|
||||
}
|
||||
}
|
||||
|
||||
fn delay_msec(&self) -> u64 {
|
||||
(RESYNC_RETRY_DELAY.as_millis() as u64)
|
||||
<< std::cmp::min(self.errors - 1, RESYNC_RETRY_DELAY_MAX_BACKOFF_POWER)
|
||||
}
|
||||
fn next_try(&self) -> u64 {
|
||||
self.last_try + self.delay_msec()
|
||||
}
|
||||
}
|
||||
|
||||
+6
-28
@@ -19,17 +19,7 @@ use garage_util::tranquilizer::Tranquilizer;
|
||||
|
||||
use crate::manager::*;
|
||||
|
||||
// Full scrub every 30 days
|
||||
const SCRUB_INTERVAL: Duration = Duration::from_secs(3600 * 24 * 30);
|
||||
// Scrub tranquility is initially set to 4, but can be changed in the CLI
|
||||
// and the updated version is persisted over Garage restarts
|
||||
const INITIAL_SCRUB_TRANQUILITY: u32 = 4;
|
||||
|
||||
// ---- ---- ----
|
||||
// FIRST KIND OF REPAIR: FINDING MISSING BLOCKS/USELESS BLOCKS
|
||||
// This is a one-shot repair operation that can be launched,
|
||||
// checks everything, and then exits.
|
||||
// ---- ---- ----
|
||||
const SCRUB_INTERVAL: Duration = Duration::from_secs(3600 * 24 * 30); // full scrub every 30 days
|
||||
|
||||
pub struct RepairWorker {
|
||||
manager: Arc<BlockManager>,
|
||||
@@ -112,9 +102,7 @@ impl Worker for RepairWorker {
|
||||
}
|
||||
|
||||
for hash in batch_of_hashes.into_iter() {
|
||||
self.manager
|
||||
.resync
|
||||
.put_to_resync(&hash, Duration::from_secs(0))?;
|
||||
self.manager.put_to_resync(&hash, Duration::from_secs(0))?;
|
||||
self.next_start = Some(hash)
|
||||
}
|
||||
|
||||
@@ -126,9 +114,7 @@ impl Worker for RepairWorker {
|
||||
// This allows us to find blocks we are storing but don't actually need,
|
||||
// so that we can offload them if necessary and then delete them locally.
|
||||
if let Some(hash) = bi.next().await? {
|
||||
self.manager
|
||||
.resync
|
||||
.put_to_resync(&hash, Duration::from_secs(0))?;
|
||||
self.manager.put_to_resync(&hash, Duration::from_secs(0))?;
|
||||
Ok(WorkerState::Busy)
|
||||
} else {
|
||||
Ok(WorkerState::Done)
|
||||
@@ -142,13 +128,7 @@ impl Worker for RepairWorker {
|
||||
}
|
||||
}
|
||||
|
||||
// ---- ---- ----
|
||||
// SECOND KIND OF REPAIR: SCRUBBING THE DATASTORE
|
||||
// This is significantly more complex than the process above,
|
||||
// as it is a continuously-running task that triggers automatically
|
||||
// every SCRUB_INTERVAL, but can also be triggered manually
|
||||
// and whose parameter (esp. speed) can be controlled at runtime.
|
||||
// ---- ---- ----
|
||||
// ----
|
||||
|
||||
pub struct ScrubWorker {
|
||||
manager: Arc<BlockManager>,
|
||||
@@ -196,7 +176,7 @@ impl ScrubWorker {
|
||||
Ok(v) => v,
|
||||
Err(_) => ScrubWorkerPersisted {
|
||||
time_last_complete_scrub: 0,
|
||||
tranquility: INITIAL_SCRUB_TRANQUILITY,
|
||||
tranquility: 4,
|
||||
corruptions_detected: 0,
|
||||
},
|
||||
};
|
||||
@@ -363,9 +343,7 @@ impl Worker for ScrubWorker {
|
||||
}
|
||||
}
|
||||
|
||||
// ---- ---- ----
|
||||
// UTILITY FOR ENUMERATING THE BLOCK STORE
|
||||
// ---- ---- ----
|
||||
// ----
|
||||
|
||||
struct BlockStoreIterator {
|
||||
path: Vec<ReadingDir>,
|
||||
|
||||
@@ -1,590 +0,0 @@
|
||||
use std::collections::HashSet;
|
||||
use std::convert::TryInto;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use async_trait::async_trait;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use futures::future::*;
|
||||
use tokio::select;
|
||||
use tokio::sync::{watch, Notify};
|
||||
|
||||
use opentelemetry::{
|
||||
trace::{FutureExt as OtelFutureExt, TraceContextExt, Tracer},
|
||||
Context, KeyValue,
|
||||
};
|
||||
|
||||
use garage_db as db;
|
||||
use garage_db::counted_tree_hack::CountedTree;
|
||||
|
||||
use garage_util::background::*;
|
||||
use garage_util::data::*;
|
||||
use garage_util::error::*;
|
||||
use garage_util::metrics::RecordDuration;
|
||||
use garage_util::persister::Persister;
|
||||
use garage_util::time::*;
|
||||
use garage_util::tranquilizer::Tranquilizer;
|
||||
|
||||
use garage_rpc::system::System;
|
||||
use garage_rpc::*;
|
||||
|
||||
use garage_table::replication::TableReplication;
|
||||
|
||||
use crate::manager::*;
|
||||
|
||||
// Timeout for RPCs that ask other nodes whether they need a copy
|
||||
// of a given block before we delete it locally
|
||||
pub(crate) const NEED_BLOCK_QUERY_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
// The delay between the time where a resync operation fails
|
||||
// and the time when it is retried, with exponential backoff
|
||||
// (multiplied by 2, 4, 8, 16, etc. for every consecutive failure).
|
||||
pub(crate) const RESYNC_RETRY_DELAY: Duration = Duration::from_secs(60);
|
||||
// The minimum retry delay is 60 seconds = 1 minute
|
||||
// The maximum retry delay is 60 seconds * 2^6 = 60 seconds << 6 = 64 minutes (~1 hour)
|
||||
pub(crate) const RESYNC_RETRY_DELAY_MAX_BACKOFF_POWER: u64 = 6;
|
||||
|
||||
// No more than 4 resync workers can be running in the system
|
||||
pub(crate) const MAX_RESYNC_WORKERS: usize = 4;
|
||||
// Resync tranquility is initially set to 2, but can be changed in the CLI
|
||||
// and the updated version is persisted over Garage restarts
|
||||
const INITIAL_RESYNC_TRANQUILITY: u32 = 2;
|
||||
|
||||
pub struct BlockResyncManager {
|
||||
pub(crate) queue: CountedTree,
|
||||
pub(crate) notify: Notify,
|
||||
pub(crate) errors: CountedTree,
|
||||
|
||||
busy_set: BusySet,
|
||||
|
||||
persister: Persister<ResyncPersistedConfig>,
|
||||
persisted: ArcSwap<ResyncPersistedConfig>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone, Copy)]
|
||||
struct ResyncPersistedConfig {
|
||||
n_workers: usize,
|
||||
tranquility: u32,
|
||||
}
|
||||
|
||||
enum ResyncIterResult {
|
||||
BusyDidSomething,
|
||||
BusyDidNothing,
|
||||
IdleFor(Duration),
|
||||
}
|
||||
|
||||
type BusySet = Arc<Mutex<HashSet<Vec<u8>>>>;
|
||||
|
||||
struct BusyBlock {
|
||||
time_bytes: Vec<u8>,
|
||||
hash_bytes: Vec<u8>,
|
||||
busy_set: BusySet,
|
||||
}
|
||||
|
||||
impl BlockResyncManager {
|
||||
pub(crate) fn new(db: &db::Db, system: &System) -> Self {
|
||||
let queue = db
|
||||
.open_tree("block_local_resync_queue")
|
||||
.expect("Unable to open block_local_resync_queue tree");
|
||||
let queue = CountedTree::new(queue).expect("Could not count block_local_resync_queue");
|
||||
|
||||
let errors = db
|
||||
.open_tree("block_local_resync_errors")
|
||||
.expect("Unable to open block_local_resync_errors tree");
|
||||
let errors = CountedTree::new(errors).expect("Could not count block_local_resync_errors");
|
||||
|
||||
let persister = Persister::new(&system.metadata_dir, "resync_cfg");
|
||||
let persisted = match persister.load() {
|
||||
Ok(v) => v,
|
||||
Err(_) => ResyncPersistedConfig {
|
||||
n_workers: 1,
|
||||
tranquility: INITIAL_RESYNC_TRANQUILITY,
|
||||
},
|
||||
};
|
||||
|
||||
Self {
|
||||
queue,
|
||||
notify: Notify::new(),
|
||||
errors,
|
||||
busy_set: Arc::new(Mutex::new(HashSet::new())),
|
||||
persister,
|
||||
persisted: ArcSwap::new(Arc::new(persisted)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get lenght of resync queue
|
||||
pub fn queue_len(&self) -> Result<usize, Error> {
|
||||
// This currently can't return an error because the CountedTree hack
|
||||
// doesn't error on .len(), but this will change when we remove the hack
|
||||
// (hopefully someday!)
|
||||
Ok(self.queue.len())
|
||||
}
|
||||
|
||||
/// Get number of blocks that have an error
|
||||
pub fn errors_len(&self) -> Result<usize, Error> {
|
||||
// (see queue_len comment)
|
||||
Ok(self.errors.len())
|
||||
}
|
||||
|
||||
// ---- Resync loop ----
|
||||
|
||||
// This part manages a queue of blocks that need to be
|
||||
// "resynchronized", i.e. that need to have a check that
|
||||
// they are at present if we need them, or that they are
|
||||
// deleted once the garbage collection delay has passed.
|
||||
//
|
||||
// Here are some explanations on how the resync queue works.
|
||||
// There are two Sled trees that are used to have information
|
||||
// about the status of blocks that need to be resynchronized:
|
||||
//
|
||||
// - resync.queue: a tree that is ordered first by a timestamp
|
||||
// (in milliseconds since Unix epoch) that is the time at which
|
||||
// the resync must be done, and second by block hash.
|
||||
// The key in this tree is just:
|
||||
// concat(timestamp (8 bytes), hash (32 bytes))
|
||||
// The value is the same 32-byte hash.
|
||||
//
|
||||
// - resync.errors: a tree that indicates for each block
|
||||
// if the last resync resulted in an error, and if so,
|
||||
// the following two informations (see the ErrorCounter struct):
|
||||
// - how many consecutive resync errors for this block?
|
||||
// - when was the last try?
|
||||
// These two informations are used to implement an
|
||||
// exponential backoff retry strategy.
|
||||
// The key in this tree is the 32-byte hash of the block,
|
||||
// and the value is the encoded ErrorCounter value.
|
||||
//
|
||||
// We need to have these two trees, because the resync queue
|
||||
// is not just a queue of items to process, but a set of items
|
||||
// that are waiting a specific delay until we can process them
|
||||
// (the delay being necessary both internally for the exponential
|
||||
// backoff strategy, and exposed as a parameter when adding items
|
||||
// to the queue, e.g. to wait until the GC delay has passed).
|
||||
// This is why we need one tree ordered by time, and one
|
||||
// ordered by identifier of item to be processed (block hash).
|
||||
//
|
||||
// When the worker wants to process an item it takes from
|
||||
// resync.queue, it checks in resync.errors that if there is an
|
||||
// exponential back-off delay to await, it has passed before we
|
||||
// process the item. If not, the item in the queue is skipped
|
||||
// (but added back for later processing after the time of the
|
||||
// delay).
|
||||
//
|
||||
// An alternative that would have seemed natural is to
|
||||
// only add items to resync.queue with a processing time that is
|
||||
// after the delay, but there are several issues with this:
|
||||
// - This requires to synchronize updates to resync.queue and
|
||||
// resync.errors (with the current model, there is only one thread,
|
||||
// the worker thread, that accesses resync.errors,
|
||||
// so no need to synchronize) by putting them both in a lock.
|
||||
// This would mean that block_incref might need to take a lock
|
||||
// before doing its thing, meaning it has much more chances of
|
||||
// not completing successfully if something bad happens to Garage.
|
||||
// Currently Garage is not able to recover from block_incref that
|
||||
// doesn't complete successfully, because it is necessary to ensure
|
||||
// the consistency between the state of the block manager and
|
||||
// information in the BlockRef table.
|
||||
// - If a resync fails, we put that block in the resync.errors table,
|
||||
// and also add it back to resync.queue to be processed after
|
||||
// the exponential back-off delay,
|
||||
// but maybe the block is already scheduled to be resynced again
|
||||
// at another time that is before the exponential back-off delay,
|
||||
// and we have no way to check that easily. This means that
|
||||
// in all cases, we need to check the resync.errors table
|
||||
// in the resync loop at the time when a block is popped from
|
||||
// the resync.queue.
|
||||
// Overall, the current design is therefore simpler and more robust
|
||||
// because it tolerates inconsistencies between the resync.queue
|
||||
// and resync.errors table (items being scheduled in resync.queue
|
||||
// for times that are earlier than the exponential back-off delay
|
||||
// is a natural condition that is handled properly).
|
||||
|
||||
pub(crate) fn put_to_resync(&self, hash: &Hash, delay: Duration) -> db::Result<()> {
|
||||
let when = now_msec() + delay.as_millis() as u64;
|
||||
self.put_to_resync_at(hash, when)
|
||||
}
|
||||
|
||||
pub(crate) fn put_to_resync_at(&self, hash: &Hash, when: u64) -> db::Result<()> {
|
||||
trace!("Put resync_queue: {} {:?}", when, hash);
|
||||
let mut key = u64::to_be_bytes(when).to_vec();
|
||||
key.extend(hash.as_ref());
|
||||
self.queue.insert(key, hash.as_ref())?;
|
||||
self.notify.notify_waiters();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn resync_iter(&self, manager: &BlockManager) -> Result<ResyncIterResult, db::Error> {
|
||||
if let Some(block) = self.get_block_to_resync()? {
|
||||
let time_msec = u64::from_be_bytes(block.time_bytes[0..8].try_into().unwrap());
|
||||
let now = now_msec();
|
||||
|
||||
if now >= time_msec {
|
||||
let hash = Hash::try_from(&block.hash_bytes[..]).unwrap();
|
||||
|
||||
if let Some(ec) = self.errors.get(hash.as_slice())? {
|
||||
let ec = ErrorCounter::decode(&ec);
|
||||
if now < ec.next_try() {
|
||||
// if next retry after an error is not yet,
|
||||
// don't do resync and return early, but still
|
||||
// make sure the item is still in queue at expected time
|
||||
self.put_to_resync_at(&hash, ec.next_try())?;
|
||||
// ec.next_try() > now >= time_msec, so this remove
|
||||
// is not removing the one we added just above
|
||||
// (we want to do the remove after the insert to ensure
|
||||
// that the item is not lost if we crash in-between)
|
||||
self.queue.remove(&block.time_bytes)?;
|
||||
return Ok(ResyncIterResult::BusyDidNothing);
|
||||
}
|
||||
}
|
||||
|
||||
let tracer = opentelemetry::global::tracer("garage");
|
||||
let trace_id = gen_uuid();
|
||||
let span = tracer
|
||||
.span_builder("Resync block")
|
||||
.with_trace_id(
|
||||
opentelemetry::trace::TraceId::from_hex(&hex::encode(
|
||||
&trace_id.as_slice()[..16],
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.with_attributes(vec![KeyValue::new("block", format!("{:?}", hash))])
|
||||
.start(&tracer);
|
||||
|
||||
let res = self
|
||||
.resync_block(manager, &hash)
|
||||
.with_context(Context::current_with_span(span))
|
||||
.bound_record_duration(&manager.metrics.resync_duration)
|
||||
.await;
|
||||
|
||||
manager.metrics.resync_counter.add(1);
|
||||
|
||||
if let Err(e) = &res {
|
||||
manager.metrics.resync_error_counter.add(1);
|
||||
warn!("Error when resyncing {:?}: {}", hash, e);
|
||||
|
||||
let err_counter = match self.errors.get(hash.as_slice())? {
|
||||
Some(ec) => ErrorCounter::decode(&ec).add1(now + 1),
|
||||
None => ErrorCounter::new(now + 1),
|
||||
};
|
||||
|
||||
self.errors.insert(hash.as_slice(), err_counter.encode())?;
|
||||
|
||||
self.put_to_resync_at(&hash, err_counter.next_try())?;
|
||||
// err_counter.next_try() >= now + 1 > now,
|
||||
// the entry we remove from the queue is not
|
||||
// the entry we inserted with put_to_resync_at
|
||||
self.queue.remove(&block.time_bytes)?;
|
||||
} else {
|
||||
self.errors.remove(hash.as_slice())?;
|
||||
self.queue.remove(&block.time_bytes)?;
|
||||
}
|
||||
|
||||
Ok(ResyncIterResult::BusyDidSomething)
|
||||
} else {
|
||||
Ok(ResyncIterResult::IdleFor(Duration::from_millis(
|
||||
time_msec - now,
|
||||
)))
|
||||
}
|
||||
} else {
|
||||
// Here we wait either for a notification that an item has been
|
||||
// added to the queue, or for a constant delay of 10 secs to expire.
|
||||
// The delay avoids a race condition where the notification happens
|
||||
// between the time we checked the queue and the first poll
|
||||
// to resync_notify.notified(): if that happens, we'll just loop
|
||||
// back 10 seconds later, which is fine.
|
||||
Ok(ResyncIterResult::IdleFor(Duration::from_secs(10)))
|
||||
}
|
||||
}
|
||||
|
||||
fn get_block_to_resync(&self) -> Result<Option<BusyBlock>, db::Error> {
|
||||
let mut busy = self.busy_set.lock().unwrap();
|
||||
for it in self.queue.iter()? {
|
||||
let (time_bytes, hash_bytes) = it?;
|
||||
if !busy.contains(&time_bytes) {
|
||||
busy.insert(time_bytes.clone());
|
||||
return Ok(Some(BusyBlock {
|
||||
time_bytes,
|
||||
hash_bytes,
|
||||
busy_set: self.busy_set.clone(),
|
||||
}));
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn resync_block(&self, manager: &BlockManager, hash: &Hash) -> Result<(), Error> {
|
||||
let BlockStatus { exists, needed } = manager.check_block_status(hash).await?;
|
||||
|
||||
if exists != needed.is_needed() || exists != needed.is_nonzero() {
|
||||
debug!(
|
||||
"Resync block {:?}: exists {}, nonzero rc {}, deletable {}",
|
||||
hash,
|
||||
exists,
|
||||
needed.is_nonzero(),
|
||||
needed.is_deletable(),
|
||||
);
|
||||
}
|
||||
|
||||
if exists && needed.is_deletable() {
|
||||
info!("Resync block {:?}: offloading and deleting", hash);
|
||||
|
||||
let mut who = manager.replication.write_nodes(hash);
|
||||
if who.len() < manager.replication.write_quorum() {
|
||||
return Err(Error::Message("Not trying to offload block because we don't have a quorum of nodes to write to".to_string()));
|
||||
}
|
||||
who.retain(|id| *id != manager.system.id);
|
||||
|
||||
let msg = Arc::new(BlockRpc::NeedBlockQuery(*hash));
|
||||
let who_needs_fut = who.iter().map(|to| {
|
||||
manager.system.rpc.call_arc(
|
||||
&manager.endpoint,
|
||||
*to,
|
||||
msg.clone(),
|
||||
RequestStrategy::with_priority(PRIO_BACKGROUND)
|
||||
.with_timeout(NEED_BLOCK_QUERY_TIMEOUT),
|
||||
)
|
||||
});
|
||||
let who_needs_resps = join_all(who_needs_fut).await;
|
||||
|
||||
let mut need_nodes = vec![];
|
||||
for (node, needed) in who.iter().zip(who_needs_resps.into_iter()) {
|
||||
match needed.err_context("NeedBlockQuery RPC")? {
|
||||
BlockRpc::NeedBlockReply(needed) => {
|
||||
if needed {
|
||||
need_nodes.push(*node);
|
||||
}
|
||||
}
|
||||
m => {
|
||||
return Err(Error::unexpected_rpc_message(m));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !need_nodes.is_empty() {
|
||||
trace!(
|
||||
"Block {:?} needed by {} nodes, sending",
|
||||
hash,
|
||||
need_nodes.len()
|
||||
);
|
||||
|
||||
for node in need_nodes.iter() {
|
||||
manager
|
||||
.metrics
|
||||
.resync_send_counter
|
||||
.add(1, &[KeyValue::new("to", format!("{:?}", node))]);
|
||||
}
|
||||
|
||||
let put_block_message = manager.read_block(hash).await?;
|
||||
manager
|
||||
.system
|
||||
.rpc
|
||||
.try_call_many(
|
||||
&manager.endpoint,
|
||||
&need_nodes[..],
|
||||
put_block_message,
|
||||
RequestStrategy::with_priority(PRIO_BACKGROUND)
|
||||
.with_quorum(need_nodes.len())
|
||||
.with_timeout(BLOCK_RW_TIMEOUT),
|
||||
)
|
||||
.await
|
||||
.err_context("PutBlock RPC")?;
|
||||
}
|
||||
info!(
|
||||
"Deleting unneeded block {:?}, offload finished ({} / {})",
|
||||
hash,
|
||||
need_nodes.len(),
|
||||
who.len()
|
||||
);
|
||||
|
||||
manager.delete_if_unneeded(hash).await?;
|
||||
|
||||
manager.rc.clear_deleted_block_rc(hash)?;
|
||||
}
|
||||
|
||||
if needed.is_nonzero() && !exists {
|
||||
info!(
|
||||
"Resync block {:?}: fetching absent but needed block (refcount > 0)",
|
||||
hash
|
||||
);
|
||||
|
||||
let block_data = manager.rpc_get_raw_block(hash).await?;
|
||||
|
||||
manager.metrics.resync_recv_counter.add(1);
|
||||
|
||||
manager.write_block(hash, &block_data).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn update_persisted(
|
||||
&self,
|
||||
update: impl Fn(&mut ResyncPersistedConfig),
|
||||
) -> Result<(), Error> {
|
||||
let mut cfg: ResyncPersistedConfig = *self.persisted.load().as_ref();
|
||||
update(&mut cfg);
|
||||
self.persister.save_async(&cfg).await?;
|
||||
self.persisted.store(Arc::new(cfg));
|
||||
self.notify.notify_waiters();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn set_n_workers(&self, n_workers: usize) -> Result<(), Error> {
|
||||
if !(1..=MAX_RESYNC_WORKERS).contains(&n_workers) {
|
||||
return Err(Error::Message(format!(
|
||||
"Invalid number of resync workers, must be between 1 and {}",
|
||||
MAX_RESYNC_WORKERS
|
||||
)));
|
||||
}
|
||||
self.update_persisted(|cfg| cfg.n_workers = n_workers).await
|
||||
}
|
||||
|
||||
pub async fn set_tranquility(&self, tranquility: u32) -> Result<(), Error> {
|
||||
self.update_persisted(|cfg| cfg.tranquility = tranquility)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for BusyBlock {
|
||||
fn drop(&mut self) {
|
||||
let mut busy = self.busy_set.lock().unwrap();
|
||||
busy.remove(&self.time_bytes);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct ResyncWorker {
|
||||
index: usize,
|
||||
manager: Arc<BlockManager>,
|
||||
tranquilizer: Tranquilizer,
|
||||
next_delay: Duration,
|
||||
}
|
||||
|
||||
impl ResyncWorker {
|
||||
pub(crate) fn new(index: usize, manager: Arc<BlockManager>) -> Self {
|
||||
Self {
|
||||
index,
|
||||
manager,
|
||||
tranquilizer: Tranquilizer::new(30),
|
||||
next_delay: Duration::from_secs(10),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Worker for ResyncWorker {
|
||||
fn name(&self) -> String {
|
||||
format!("Block resync worker #{}", self.index + 1)
|
||||
}
|
||||
|
||||
fn info(&self) -> Option<String> {
|
||||
let persisted = self.manager.resync.persisted.load();
|
||||
|
||||
if self.index >= persisted.n_workers {
|
||||
return Some("(unused)".into());
|
||||
}
|
||||
|
||||
let mut ret = vec![];
|
||||
ret.push(format!("tranquility = {}", persisted.tranquility));
|
||||
|
||||
let qlen = self.manager.resync.queue_len().unwrap_or(0);
|
||||
if qlen > 0 {
|
||||
ret.push(format!("{} blocks in queue", qlen));
|
||||
}
|
||||
|
||||
let elen = self.manager.resync.errors_len().unwrap_or(0);
|
||||
if elen > 0 {
|
||||
ret.push(format!("{} blocks in error state", elen));
|
||||
}
|
||||
|
||||
Some(ret.join(", "))
|
||||
}
|
||||
|
||||
async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
|
||||
if self.index >= self.manager.resync.persisted.load().n_workers {
|
||||
return Ok(WorkerState::Idle);
|
||||
}
|
||||
|
||||
self.tranquilizer.reset();
|
||||
match self.manager.resync.resync_iter(&self.manager).await {
|
||||
Ok(ResyncIterResult::BusyDidSomething) => Ok(self
|
||||
.tranquilizer
|
||||
.tranquilize_worker(self.manager.resync.persisted.load().tranquility)),
|
||||
Ok(ResyncIterResult::BusyDidNothing) => Ok(WorkerState::Busy),
|
||||
Ok(ResyncIterResult::IdleFor(delay)) => {
|
||||
self.next_delay = delay;
|
||||
Ok(WorkerState::Idle)
|
||||
}
|
||||
Err(e) => {
|
||||
// The errors that we have here are only Sled errors
|
||||
// We don't really know how to handle them so just ¯\_(ツ)_/¯
|
||||
// (there is kind of an assumption that Sled won't error on us,
|
||||
// if it does there is not much we can do -- TODO should we just panic?)
|
||||
// Here we just give the error to the worker manager,
|
||||
// it will print it to the logs and increment a counter
|
||||
Err(e.into())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
|
||||
while self.index >= self.manager.resync.persisted.load().n_workers {
|
||||
self.manager.resync.notify.notified().await
|
||||
}
|
||||
|
||||
select! {
|
||||
_ = tokio::time::sleep(self.next_delay) => (),
|
||||
_ = self.manager.resync.notify.notified() => (),
|
||||
};
|
||||
|
||||
WorkerState::Busy
|
||||
}
|
||||
}
|
||||
|
||||
/// Counts the number of errors when resyncing a block,
|
||||
/// and the time of the last try.
|
||||
/// Used to implement exponential backoff.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct ErrorCounter {
|
||||
errors: u64,
|
||||
last_try: u64,
|
||||
}
|
||||
|
||||
impl ErrorCounter {
|
||||
fn new(now: u64) -> Self {
|
||||
Self {
|
||||
errors: 1,
|
||||
last_try: now,
|
||||
}
|
||||
}
|
||||
|
||||
fn decode(data: &[u8]) -> Self {
|
||||
Self {
|
||||
errors: u64::from_be_bytes(data[0..8].try_into().unwrap()),
|
||||
last_try: u64::from_be_bytes(data[8..16].try_into().unwrap()),
|
||||
}
|
||||
}
|
||||
fn encode(&self) -> Vec<u8> {
|
||||
[
|
||||
u64::to_be_bytes(self.errors),
|
||||
u64::to_be_bytes(self.last_try),
|
||||
]
|
||||
.concat()
|
||||
}
|
||||
|
||||
fn add1(self, now: u64) -> Self {
|
||||
Self {
|
||||
errors: self.errors + 1,
|
||||
last_try: now,
|
||||
}
|
||||
}
|
||||
|
||||
fn delay_msec(&self) -> u64 {
|
||||
(RESYNC_RETRY_DELAY.as_millis() as u64)
|
||||
<< std::cmp::min(self.errors - 1, RESYNC_RETRY_DELAY_MAX_BACKOFF_POWER)
|
||||
}
|
||||
fn next_try(&self) -> u64 {
|
||||
self.last_try + self.delay_msec()
|
||||
}
|
||||
}
|
||||
+2
-30
@@ -15,8 +15,6 @@ use garage_table::*;
|
||||
|
||||
use garage_rpc::*;
|
||||
|
||||
use garage_block::repair::ScrubWorkerCommand;
|
||||
|
||||
use garage_model::bucket_alias_table::*;
|
||||
use garage_model::bucket_table::*;
|
||||
use garage_model::garage::Garage;
|
||||
@@ -781,13 +779,13 @@ impl AdminRpcHandler {
|
||||
writeln!(
|
||||
&mut ret,
|
||||
" resync queue length: {}",
|
||||
self.garage.block_manager.resync.queue_len()?
|
||||
self.garage.block_manager.resync_queue_len()?
|
||||
)
|
||||
.unwrap();
|
||||
writeln!(
|
||||
&mut ret,
|
||||
" blocks with resync errors: {}",
|
||||
self.garage.block_manager.resync.errors_len()?
|
||||
self.garage.block_manager.resync_errors_len()?
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
@@ -838,32 +836,6 @@ impl AdminRpcHandler {
|
||||
let workers = self.garage.background.get_worker_info();
|
||||
Ok(AdminRpc::WorkerList(workers, opt))
|
||||
}
|
||||
WorkerCmd::Set { opt } => match opt {
|
||||
WorkerSetCmd::ScrubTranquility { tranquility } => {
|
||||
let scrub_command = ScrubWorkerCommand::SetTranquility(tranquility);
|
||||
self.garage
|
||||
.block_manager
|
||||
.send_scrub_command(scrub_command)
|
||||
.await;
|
||||
Ok(AdminRpc::Ok("Scrub tranquility updated".into()))
|
||||
}
|
||||
WorkerSetCmd::ResyncNWorkers { n_workers } => {
|
||||
self.garage
|
||||
.block_manager
|
||||
.resync
|
||||
.set_n_workers(n_workers)
|
||||
.await?;
|
||||
Ok(AdminRpc::Ok("Number of resync workers updated".into()))
|
||||
}
|
||||
WorkerSetCmd::ResyncTranquility { tranquility } => {
|
||||
self.garage
|
||||
.block_manager
|
||||
.resync
|
||||
.set_tranquility(tranquility)
|
||||
.await?;
|
||||
Ok(AdminRpc::Ok("Resync tranquility updated".into()))
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+50
-70
@@ -1,65 +1,64 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use garage_util::version;
|
||||
use structopt::StructOpt;
|
||||
|
||||
#[derive(StructOpt, Debug)]
|
||||
pub enum Command {
|
||||
/// Run Garage server
|
||||
#[structopt(name = "server", version = version::garage())]
|
||||
#[structopt(name = "server")]
|
||||
Server,
|
||||
|
||||
/// Get network status
|
||||
#[structopt(name = "status", version = version::garage())]
|
||||
#[structopt(name = "status")]
|
||||
Status,
|
||||
|
||||
/// Operations on individual Garage nodes
|
||||
#[structopt(name = "node", version = version::garage())]
|
||||
#[structopt(name = "node")]
|
||||
Node(NodeOperation),
|
||||
|
||||
/// Operations on the assignation of node roles in the cluster layout
|
||||
#[structopt(name = "layout", version = version::garage())]
|
||||
#[structopt(name = "layout")]
|
||||
Layout(LayoutOperation),
|
||||
|
||||
/// Operations on buckets
|
||||
#[structopt(name = "bucket", version = version::garage())]
|
||||
#[structopt(name = "bucket")]
|
||||
Bucket(BucketOperation),
|
||||
|
||||
/// Operations on S3 access keys
|
||||
#[structopt(name = "key", version = version::garage())]
|
||||
#[structopt(name = "key")]
|
||||
Key(KeyOperation),
|
||||
|
||||
/// Run migrations from previous Garage version
|
||||
/// (DO NOT USE WITHOUT READING FULL DOCUMENTATION)
|
||||
#[structopt(name = "migrate", version = version::garage())]
|
||||
#[structopt(name = "migrate")]
|
||||
Migrate(MigrateOpt),
|
||||
|
||||
/// Start repair of node data on remote node
|
||||
#[structopt(name = "repair", version = version::garage())]
|
||||
#[structopt(name = "repair")]
|
||||
Repair(RepairOpt),
|
||||
|
||||
/// Offline reparation of node data (these repairs must be run offline
|
||||
/// directly on the server node)
|
||||
#[structopt(name = "offline-repair", version = version::garage())]
|
||||
#[structopt(name = "offline-repair")]
|
||||
OfflineRepair(OfflineRepairOpt),
|
||||
|
||||
/// Gather node statistics
|
||||
#[structopt(name = "stats", version = version::garage())]
|
||||
#[structopt(name = "stats")]
|
||||
Stats(StatsOpt),
|
||||
|
||||
/// Manage background workers
|
||||
#[structopt(name = "worker", version = version::garage())]
|
||||
#[structopt(name = "worker")]
|
||||
Worker(WorkerOpt),
|
||||
}
|
||||
|
||||
#[derive(StructOpt, Debug)]
|
||||
pub enum NodeOperation {
|
||||
/// Print identifier (public key) of this Garage node
|
||||
#[structopt(name = "id", version = version::garage())]
|
||||
#[structopt(name = "id")]
|
||||
NodeId(NodeIdOpt),
|
||||
|
||||
/// Connect to Garage node that is currently isolated from the system
|
||||
#[structopt(name = "connect", version = version::garage())]
|
||||
#[structopt(name = "connect")]
|
||||
Connect(ConnectNodeOpt),
|
||||
}
|
||||
|
||||
@@ -80,23 +79,23 @@ pub struct ConnectNodeOpt {
|
||||
#[derive(StructOpt, Debug)]
|
||||
pub enum LayoutOperation {
|
||||
/// Assign role to Garage node
|
||||
#[structopt(name = "assign", version = version::garage())]
|
||||
#[structopt(name = "assign")]
|
||||
Assign(AssignRoleOpt),
|
||||
|
||||
/// Remove role from Garage cluster node
|
||||
#[structopt(name = "remove", version = version::garage())]
|
||||
#[structopt(name = "remove")]
|
||||
Remove(RemoveRoleOpt),
|
||||
|
||||
/// Show roles currently assigned to nodes and changes staged for commit
|
||||
#[structopt(name = "show", version = version::garage())]
|
||||
#[structopt(name = "show")]
|
||||
Show,
|
||||
|
||||
/// Apply staged changes to cluster layout
|
||||
#[structopt(name = "apply", version = version::garage())]
|
||||
#[structopt(name = "apply")]
|
||||
Apply(ApplyLayoutOpt),
|
||||
|
||||
/// Revert staged changes to cluster layout
|
||||
#[structopt(name = "revert", version = version::garage())]
|
||||
#[structopt(name = "revert")]
|
||||
Revert(RevertLayoutOpt),
|
||||
}
|
||||
|
||||
@@ -151,43 +150,43 @@ pub struct RevertLayoutOpt {
|
||||
#[derive(Serialize, Deserialize, StructOpt, Debug)]
|
||||
pub enum BucketOperation {
|
||||
/// List buckets
|
||||
#[structopt(name = "list", version = version::garage())]
|
||||
#[structopt(name = "list")]
|
||||
List,
|
||||
|
||||
/// Get bucket info
|
||||
#[structopt(name = "info", version = version::garage())]
|
||||
#[structopt(name = "info")]
|
||||
Info(BucketOpt),
|
||||
|
||||
/// Create bucket
|
||||
#[structopt(name = "create", version = version::garage())]
|
||||
#[structopt(name = "create")]
|
||||
Create(BucketOpt),
|
||||
|
||||
/// Delete bucket
|
||||
#[structopt(name = "delete", version = version::garage())]
|
||||
#[structopt(name = "delete")]
|
||||
Delete(DeleteBucketOpt),
|
||||
|
||||
/// Alias bucket under new name
|
||||
#[structopt(name = "alias", version = version::garage())]
|
||||
#[structopt(name = "alias")]
|
||||
Alias(AliasBucketOpt),
|
||||
|
||||
/// Remove bucket alias
|
||||
#[structopt(name = "unalias", version = version::garage())]
|
||||
#[structopt(name = "unalias")]
|
||||
Unalias(UnaliasBucketOpt),
|
||||
|
||||
/// Allow key to read or write to bucket
|
||||
#[structopt(name = "allow", version = version::garage())]
|
||||
#[structopt(name = "allow")]
|
||||
Allow(PermBucketOpt),
|
||||
|
||||
/// Deny key from reading or writing to bucket
|
||||
#[structopt(name = "deny", version = version::garage())]
|
||||
#[structopt(name = "deny")]
|
||||
Deny(PermBucketOpt),
|
||||
|
||||
/// Expose as website or not
|
||||
#[structopt(name = "website", version = version::garage())]
|
||||
#[structopt(name = "website")]
|
||||
Website(WebsiteOpt),
|
||||
|
||||
/// Set the quotas for this bucket
|
||||
#[structopt(name = "set-quotas", version = version::garage())]
|
||||
#[structopt(name = "set-quotas")]
|
||||
SetQuotas(SetQuotasOpt),
|
||||
}
|
||||
|
||||
@@ -293,35 +292,35 @@ pub struct SetQuotasOpt {
|
||||
#[derive(Serialize, Deserialize, StructOpt, Debug)]
|
||||
pub enum KeyOperation {
|
||||
/// List keys
|
||||
#[structopt(name = "list", version = version::garage())]
|
||||
#[structopt(name = "list")]
|
||||
List,
|
||||
|
||||
/// Get key info
|
||||
#[structopt(name = "info", version = version::garage())]
|
||||
#[structopt(name = "info")]
|
||||
Info(KeyOpt),
|
||||
|
||||
/// Create new key
|
||||
#[structopt(name = "new", version = version::garage())]
|
||||
#[structopt(name = "new")]
|
||||
New(KeyNewOpt),
|
||||
|
||||
/// Rename key
|
||||
#[structopt(name = "rename", version = version::garage())]
|
||||
#[structopt(name = "rename")]
|
||||
Rename(KeyRenameOpt),
|
||||
|
||||
/// Delete key
|
||||
#[structopt(name = "delete", version = version::garage())]
|
||||
#[structopt(name = "delete")]
|
||||
Delete(KeyDeleteOpt),
|
||||
|
||||
/// Set permission flags for key
|
||||
#[structopt(name = "allow", version = version::garage())]
|
||||
#[structopt(name = "allow")]
|
||||
Allow(KeyPermOpt),
|
||||
|
||||
/// Unset permission flags for key
|
||||
#[structopt(name = "deny", version = version::garage())]
|
||||
#[structopt(name = "deny")]
|
||||
Deny(KeyPermOpt),
|
||||
|
||||
/// Import key
|
||||
#[structopt(name = "import", version = version::garage())]
|
||||
#[structopt(name = "import")]
|
||||
Import(KeyImportOpt),
|
||||
}
|
||||
|
||||
@@ -393,7 +392,7 @@ pub struct MigrateOpt {
|
||||
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
|
||||
pub enum MigrateWhat {
|
||||
/// Migrate buckets and permissions from v0.5.0
|
||||
#[structopt(name = "buckets050", version = version::garage())]
|
||||
#[structopt(name = "buckets050")]
|
||||
Buckets050,
|
||||
}
|
||||
|
||||
@@ -414,19 +413,19 @@ pub struct RepairOpt {
|
||||
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
|
||||
pub enum RepairWhat {
|
||||
/// Only do a full sync of metadata tables
|
||||
#[structopt(name = "tables", version = version::garage())]
|
||||
#[structopt(name = "tables")]
|
||||
Tables,
|
||||
/// Only repair (resync/rebalance) the set of stored blocks
|
||||
#[structopt(name = "blocks", version = version::garage())]
|
||||
#[structopt(name = "blocks")]
|
||||
Blocks,
|
||||
/// Only redo the propagation of object deletions to the version table (slow)
|
||||
#[structopt(name = "versions", version = version::garage())]
|
||||
#[structopt(name = "versions")]
|
||||
Versions,
|
||||
/// Only redo the propagation of version deletions to the block ref table (extremely slow)
|
||||
#[structopt(name = "block_refs", version = version::garage())]
|
||||
#[structopt(name = "block_refs")]
|
||||
BlockRefs,
|
||||
/// Verify integrity of all blocks on disc (extremely slow, i/o intensive)
|
||||
#[structopt(name = "scrub", version = version::garage())]
|
||||
#[structopt(name = "scrub")]
|
||||
Scrub {
|
||||
#[structopt(subcommand)]
|
||||
cmd: ScrubCmd,
|
||||
@@ -436,19 +435,19 @@ pub enum RepairWhat {
|
||||
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
|
||||
pub enum ScrubCmd {
|
||||
/// Start scrub
|
||||
#[structopt(name = "start", version = version::garage())]
|
||||
#[structopt(name = "start")]
|
||||
Start,
|
||||
/// Pause scrub (it will resume automatically after 24 hours)
|
||||
#[structopt(name = "pause", version = version::garage())]
|
||||
#[structopt(name = "pause")]
|
||||
Pause,
|
||||
/// Resume paused scrub
|
||||
#[structopt(name = "resume", version = version::garage())]
|
||||
#[structopt(name = "resume")]
|
||||
Resume,
|
||||
/// Cancel scrub in progress
|
||||
#[structopt(name = "cancel", version = version::garage())]
|
||||
#[structopt(name = "cancel")]
|
||||
Cancel,
|
||||
/// Set tranquility level for in-progress and future scrubs
|
||||
#[structopt(name = "set-tranquility", version = version::garage())]
|
||||
#[structopt(name = "set-tranquility")]
|
||||
SetTranquility {
|
||||
#[structopt()]
|
||||
tranquility: u32,
|
||||
@@ -469,10 +468,10 @@ pub struct OfflineRepairOpt {
|
||||
pub enum OfflineRepairWhat {
|
||||
/// Repair K2V item counters
|
||||
#[cfg(feature = "k2v")]
|
||||
#[structopt(name = "k2v_item_counters", version = version::garage())]
|
||||
#[structopt(name = "k2v_item_counters")]
|
||||
K2VItemCounters,
|
||||
/// Repair object counters
|
||||
#[structopt(name = "object_counters", version = version::garage())]
|
||||
#[structopt(name = "object_counters")]
|
||||
ObjectCounters,
|
||||
}
|
||||
|
||||
@@ -496,17 +495,11 @@ pub struct WorkerOpt {
|
||||
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
|
||||
pub enum WorkerCmd {
|
||||
/// List all workers on Garage node
|
||||
#[structopt(name = "list", version = version::garage())]
|
||||
#[structopt(name = "list")]
|
||||
List {
|
||||
#[structopt(flatten)]
|
||||
opt: WorkerListOpt,
|
||||
},
|
||||
/// Set worker parameter
|
||||
#[structopt(name = "set", version = version::garage())]
|
||||
Set {
|
||||
#[structopt(subcommand)]
|
||||
opt: WorkerSetCmd,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone, Copy)]
|
||||
@@ -518,16 +511,3 @@ pub struct WorkerListOpt {
|
||||
#[structopt(short = "e", long = "errors")]
|
||||
pub errors: bool,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
|
||||
pub enum WorkerSetCmd {
|
||||
/// Set tranquility of scrub operations
|
||||
#[structopt(name = "scrub-tranquility", version = version::garage())]
|
||||
ScrubTranquility { tranquility: u32 },
|
||||
/// Set number of concurrent block resync workers
|
||||
#[structopt(name = "resync-n-workers", version = version::garage())]
|
||||
ResyncNWorkers { n_workers: usize },
|
||||
/// Set tranquility of block resync operations
|
||||
#[structopt(name = "resync-tranquility", version = version::garage())]
|
||||
ResyncTranquility { tranquility: u32 },
|
||||
}
|
||||
|
||||
+1
-2
@@ -22,7 +22,6 @@ use garage_util::error::*;
|
||||
|
||||
use garage_rpc::system::*;
|
||||
use garage_rpc::*;
|
||||
use garage_util::version;
|
||||
|
||||
use garage_model::helper::error::Error as HelperError;
|
||||
|
||||
@@ -30,7 +29,7 @@ use admin::*;
|
||||
use cli::*;
|
||||
|
||||
#[derive(StructOpt, Debug)]
|
||||
#[structopt(name = "garage", version = version::garage(), about = "S3-compatible object store for self-hosted geo-distributed deployments")]
|
||||
#[structopt(name = "garage")]
|
||||
struct Opt {
|
||||
/// Host to connect to for admin operations, in the format:
|
||||
/// <public-key>@<ip>:<port>
|
||||
|
||||
@@ -159,6 +159,7 @@ impl Garage {
|
||||
&db,
|
||||
config.data_dir.clone(),
|
||||
config.compression_level,
|
||||
config.block_manager_background_tranquility,
|
||||
data_rep_param,
|
||||
system.clone(),
|
||||
);
|
||||
|
||||
@@ -19,6 +19,7 @@ garage_util = { version = "0.7.0", path = "../util" }
|
||||
arc-swap = "1.0"
|
||||
bytes = "1.0"
|
||||
gethostname = "0.2"
|
||||
git-version = "0.3.4"
|
||||
hex = "0.4"
|
||||
tracing = "0.1.30"
|
||||
rand = "0.8"
|
||||
|
||||
+5
-2
@@ -27,7 +27,6 @@ use garage_util::data::*;
|
||||
use garage_util::error::*;
|
||||
use garage_util::persister::Persister;
|
||||
use garage_util::time::*;
|
||||
use garage_util::version;
|
||||
|
||||
use crate::consul::*;
|
||||
#[cfg(feature = "kubernetes-discovery")]
|
||||
@@ -321,7 +320,11 @@ impl System {
|
||||
// also available through RPC) ----
|
||||
|
||||
pub fn garage_version(&self) -> &'static str {
|
||||
version::garage()
|
||||
option_env!("GIT_VERSION").unwrap_or(git_version::git_version!(
|
||||
prefix = "git:",
|
||||
cargo_prefix = "cargo:",
|
||||
fallback = "unknown"
|
||||
))
|
||||
}
|
||||
|
||||
pub fn get_known_nodes(&self) -> Vec<KnownNodeInfo> {
|
||||
|
||||
@@ -24,7 +24,6 @@ hex = "0.4"
|
||||
tracing = "0.1.30"
|
||||
rand = "0.8"
|
||||
sha2 = "0.9"
|
||||
git-version = "0.3.4"
|
||||
|
||||
chrono = "0.4"
|
||||
rmp-serde = "0.15"
|
||||
@@ -44,6 +43,5 @@ hyper = "0.14"
|
||||
|
||||
opentelemetry = { version = "0.17", features = [ "rt-tokio", "metrics", "trace" ] }
|
||||
|
||||
|
||||
[features]
|
||||
k2v = []
|
||||
|
||||
@@ -23,6 +23,10 @@ pub struct Config {
|
||||
#[serde(default = "default_block_size")]
|
||||
pub block_size: usize,
|
||||
|
||||
/// Size of data blocks to save to disk
|
||||
#[serde(default = "default_block_manager_background_tranquility")]
|
||||
pub block_manager_background_tranquility: u32,
|
||||
|
||||
/// Replication mode. Supported values:
|
||||
/// - none, 1 -> no replication
|
||||
/// - 2 -> 2-way replication
|
||||
@@ -143,6 +147,9 @@ fn default_sled_flush_every_ms() -> u64 {
|
||||
fn default_block_size() -> usize {
|
||||
1048576
|
||||
}
|
||||
fn default_block_manager_background_tranquility() -> u32 {
|
||||
2
|
||||
}
|
||||
|
||||
/// Read and parse configuration
|
||||
pub fn read_config(config_file: PathBuf) -> Result<Config, Error> {
|
||||
|
||||
@@ -14,4 +14,3 @@ pub mod persister;
|
||||
pub mod time;
|
||||
pub mod token_bucket;
|
||||
pub mod tranquilizer;
|
||||
pub mod version;
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
pub fn garage() -> &'static str {
|
||||
option_env!("GIT_VERSION").unwrap_or(git_version::git_version!(
|
||||
prefix = "git:",
|
||||
cargo_prefix = "cargo:",
|
||||
fallback = "unknown"
|
||||
))
|
||||
}
|
||||
Reference in New Issue
Block a user