bump version to v1.3.0

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1164

.woodpecker/debug.yaml

@@ -1,3 +1,6 @@
+labels:
+  nix: "enabled"
+
 when:
   event:
   - push

.woodpecker/publish.yaml

@@ -1,3 +1,6 @@
+labels:
+  nix: "enabled"
+
 when:
   event:
   - deployment

.woodpecker/release.yaml

@@ -1,3 +1,6 @@
+labels:
+  nix: "enabled"
+
 when:
   event:
   - deployment

Cargo.lock

@@ -1280,7 +1280,7 @@ dependencies = [
 
 [[package]]
 name = "garage"
-version = "1.2.0"
+version = "1.3.0"
 dependencies = [
  "assert-json-diff",
  "async-trait",
@@ -1321,7 +1321,6 @@ dependencies = [
  "opentelemetry-prometheus",
  "parse_duration",
  "serde",
- "serde_bytes",
  "serde_json",
  "sha1",
  "sha2",
@@ -1337,7 +1336,7 @@ dependencies = [
 
 [[package]]
 name = "garage_api_admin"
-version = "1.2.0"
+version = "1.3.0"
 dependencies = [
  "argon2",
  "async-trait",
@@ -1363,7 +1362,7 @@ dependencies = [
 
 [[package]]
 name = "garage_api_common"
-version = "1.2.0"
+version = "1.3.0"
 dependencies = [
  "base64 0.21.7",
  "bytes",
@@ -1397,7 +1396,7 @@ dependencies = [
 
 [[package]]
 name = "garage_api_k2v"
-version = "1.2.0"
+version = "1.3.0"
 dependencies = [
  "base64 0.21.7",
  "err-derive",
@@ -1420,7 +1419,7 @@ dependencies = [
 
 [[package]]
 name = "garage_api_s3"
-version = "1.2.0"
+version = "1.3.0"
 dependencies = [
  "aes-gcm",
  "async-compression",
@@ -1465,7 +1464,7 @@ dependencies = [
 
 [[package]]
 name = "garage_block"
-version = "1.2.0"
+version = "1.3.0"
 dependencies = [
  "arc-swap",
  "async-compression",
@@ -1490,7 +1489,7 @@ dependencies = [
 
 [[package]]
 name = "garage_db"
-version = "1.2.0"
+version = "1.3.0"
 dependencies = [
  "err-derive",
  "fjall",
@@ -1505,7 +1504,7 @@ dependencies = [
 
 [[package]]
 name = "garage_model"
-version = "1.2.0"
+version = "1.3.0"
 dependencies = [
  "async-trait",
  "base64 0.21.7",
@@ -1532,7 +1531,7 @@ dependencies = [
 
 [[package]]
 name = "garage_net"
-version = "1.2.0"
+version = "1.3.0"
 dependencies = [
  "arc-swap",
  "bytes",
@@ -1557,7 +1556,7 @@ dependencies = [
 
 [[package]]
 name = "garage_rpc"
-version = "1.2.0"
+version = "1.3.0"
 dependencies = [
  "arc-swap",
  "async-trait",
@@ -1589,7 +1588,7 @@ dependencies = [
 
 [[package]]
 name = "garage_table"
-version = "1.2.0"
+version = "1.3.0"
 dependencies = [
  "arc-swap",
  "async-trait",
@@ -1610,7 +1609,7 @@ dependencies = [
 
 [[package]]
 name = "garage_util"
-version = "1.2.0"
+version = "1.3.0"
 dependencies = [
  "arc-swap",
  "async-trait",
@@ -1642,7 +1641,7 @@ dependencies = [
 
 [[package]]
 name = "garage_web"
-version = "1.2.0"
+version = "1.3.0"
 dependencies = [
  "err-derive",
  "garage_api_common",
@@ -1811,11 +1810,11 @@ dependencies = [
 
 [[package]]
 name = "hashlink"
-version = "0.9.1"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af"
+checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1"
 dependencies = [
- "hashbrown 0.14.5",
+ "hashbrown 0.15.2",
 ]
 
 [[package]]
@@ -2620,9 +2619,9 @@ dependencies = [
 
 [[package]]
 name = "libsqlite3-sys"
-version = "0.28.0"
+version = "0.35.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f"
+checksum = "133c182a6a2c87864fe97778797e46c7e999672690dc9fa3ee8e241aa4a9c13f"
 dependencies = [
  "cc",
  "pkg-config",
@@ -3479,9 +3478,9 @@ dependencies = [
 
 [[package]]
 name = "r2d2_sqlite"
-version = "0.24.0"
+version = "0.31.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a982edf65c129796dba72f8775b292ef482b40d035e827a9825b3bc07ccc5f2"
+checksum = "63417e83dc891797eea3ad379f52a5986da4bca0d6ef28baf4d14034dd111b0c"
 dependencies = [
  "r2d2",
  "rusqlite",
@@ -3670,9 +3669,9 @@ checksum = "3cd14fd5e3b777a7422cca79358c57a8f6e3a703d9ac187448d0daf220c2407f"
 
 [[package]]
 name = "rusqlite"
-version = "0.31.0"
+version = "0.37.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae"
+checksum = "165ca6e57b20e1351573e3729b958bc62f0e48025386970b6e4d29e7a7e71f3f"
 dependencies = [
  "bitflags 2.9.0",
  "fallible-iterator",

Cargo.toml

@@ -24,18 +24,18 @@ default-members = ["src/garage"]
 
 # Internal Garage crates
 format_table = { version = "0.1.1", path = "src/format-table" }
-garage_api_common = { version = "1.2.0", path = "src/api/common" }
-garage_api_admin = { version = "1.2.0", path = "src/api/admin" }
-garage_api_s3 = { version = "1.2.0", path = "src/api/s3" }
-garage_api_k2v = { version = "1.2.0", path = "src/api/k2v" }
-garage_block = { version = "1.2.0", path = "src/block" }
-garage_db = { version = "1.2.0", path = "src/db", default-features = false }
-garage_model = { version = "1.2.0", path = "src/model", default-features = false }
-garage_net = { version = "1.2.0", path = "src/net" }
-garage_rpc = { version = "1.2.0", path = "src/rpc" }
-garage_table = { version = "1.2.0", path = "src/table" }
-garage_util = { version = "1.2.0", path = "src/util" }
-garage_web = { version = "1.2.0", path = "src/web" }
+garage_api_common = { version = "1.3.0", path = "src/api/common" }
+garage_api_admin = { version = "1.3.0", path = "src/api/admin" }
+garage_api_s3 = { version = "1.3.0", path = "src/api/s3" }
+garage_api_k2v = { version = "1.3.0", path = "src/api/k2v" }
+garage_block = { version = "1.3.0", path = "src/block" }
+garage_db = { version = "1.3.0", path = "src/db", default-features = false }
+garage_model = { version = "1.3.0", path = "src/model", default-features = false }
+garage_net = { version = "1.3.0", path = "src/net" }
+garage_rpc = { version = "1.3.0", path = "src/rpc" }
+garage_table = { version = "1.3.0", path = "src/table" }
+garage_util = { version = "1.3.0", path = "src/util" }
+garage_web = { version = "1.3.0", path = "src/web" }
 k2v-client = { version = "0.0.4", path = "src/k2v-client" }
 
 # External crates from crates.io
@@ -88,9 +88,9 @@ tracing-journald = "0.3.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 
 heed = { version = "0.11", default-features = false, features = ["lmdb"] }
-rusqlite = "0.31.0"
+rusqlite = "0.37"
 r2d2 = "0.8"
-r2d2_sqlite = "0.24"
+r2d2_sqlite = "0.31"
 fjall = "2.4"
 
 async-compression = { version = "0.4", features = ["tokio", "zstd"] }

doc/book/connect/backup.md

@@ -161,3 +161,49 @@ kopia repository validate-provider
 
 You can then run all the standard kopia commands: `kopia snapshot create`, `kopia mount`...
 Everything should work out-of-the-box.
+
+## Plakar
+
+Create your key and bucket on Garage server:
+
+```bash
+garage key create my-plakar-key
+garage bucket create plakar-backups
+garage bucket allow plakar-backups --read --write --key my-plakar-key
+…
+```
+
+On Plakar server, add your Garage as a storage location:
+```bash
+plakar store add garageS3 s3://my-garage.tld/plakar-backups \
+region=garage # Or as you've specified in garage.toml \
+access_key=<Key ID from "garage key info my-plakar-key"> \
+secret_access_key=<Secret key from "garage key info my-plakar-key">
+```
+
+Then create the repository.
+```bash
+plakar at @garageS3 create -plaintext # Unencrypted
+# or
+plakar at @garageS3 create #encrypted
+```
+
+If you encrypt your backups (Plakar default), you will need to define a strong passphrase. Do not forget to save your password safely. It will be needed to decrypt your backups.
+
+
+After the repository has been created, check that everything works as expected (that might give an empty result as no file has been added yet, but no error message):
+```bash
+plakar at @garageS3 check
+```
+
+Now that everything is configure, you can use Garage as your backups storage. For instance sync it with a local backup storage:
+```bash
+$ plakar at ~/backups sync to @garageS3
+```
+
+Or list the S3 storage content:
+```bash
+$ plakar at @garageS3 ls
+```
+
+More information in Plakar documentation: https://www.plakar.io/docs/main/quickstart/

doc/book/cookbook/real-world.md

@@ -96,14 +96,14 @@ to store 2 TB of data in total.
 ## Get a Docker image
 
 Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
-We encourage you to use a fixed tag (eg. `v1.2.0`) and not the `latest` tag.
-For this example, we will use the latest published version at the time of the writing which is `v1.2.0` but it's up to you
+We encourage you to use a fixed tag (eg. `v1.3.0`) and not the `latest` tag.
+For this example, we will use the latest published version at the time of the writing which is `v1.3.0` but it's up to you
 to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
 
 For example:
 
 ```
-sudo docker pull dxflrs/garage:v1.2.0
+sudo docker pull dxflrs/garage:v1.3.0
 ```
 
 ## Deploying and configuring Garage
@@ -171,7 +171,7 @@ docker run \
   -v /etc/garage.toml:/etc/garage.toml \
   -v /var/lib/garage/meta:/var/lib/garage/meta \
   -v /var/lib/garage/data:/var/lib/garage/data \
-  dxflrs/garage:v1.2.0
+  dxflrs/garage:v1.3.0
 ```
 
 With this command line, Garage should be started automatically at each boot.
@@ -185,7 +185,7 @@ If you want to use `docker-compose`, you may use the following `docker-compose.y
 version: "3"
 services:
   garage:
-    image: dxflrs/garage:v1.2.0
+    image: dxflrs/garage:v1.3.0
     network_mode: "host"
     restart: unless-stopped
     volumes:

doc/book/quick-start/_index.md

@@ -132,7 +132,7 @@ docker run \
   -v /path/to/garage.toml:/etc/garage.toml \
   -v /path/to/garage/meta:/var/lib/garage/meta \
   -v /path/to/garage/data:/var/lib/garage/data \
-  dxflrs/garage:v1.2.0
+  dxflrs/garage:v1.3.0
 ```
 
 Under Linux, you can substitute `--network host` for `-p 3900:3900 -p 3901:3901 -p 3902:3902 -p 3903:3903`

doc/book/reference-manual/configuration.md

@@ -24,6 +24,7 @@ db_engine = "lmdb"
 
 block_size = "1M"
 block_ram_buffer_max = "256MiB"
+block_max_concurrent_reads = 16
 
 lmdb_map_size = "1T"
 
@@ -96,6 +97,7 @@ The following gives details about each available configuration option.
 Top-level configuration options, in alphabetical order:
 [`allow_punycode`](#allow_punycode),
 [`allow_world_readable_secrets`](#allow_world_readable_secrets),
+[`block_max_concurrent_reads`](`block_max_concurrent_reads),
 [`block_ram_buffer_max`](#block_ram_buffer_max),
 [`block_size`](#block_size),
 [`bootstrap_peers`](#bootstrap_peers),
@@ -522,6 +524,29 @@ node.
 
 The default value is 256MiB.
 
+#### `block_max_concurrent_reads` (since `v1.3.0` / `v2.1.0`) {#block_max_concurrent_reads}
+
+The maximum number of blocks (individual files in the data directory) open
+simultaneously for reading.
+
+Reducing this number does not limit the number of data blocks that can be
+transferred through the network simultaneously. This mechanism was just added
+as a backpressure mechanism for HDD read speed: it helps avoid a situation
+where too many requests are coming in and Garage is reading too many block
+files simultaneously, thus not making timely progress on any of the reads.
+
+When a request to read a data block comes in through the network, the requests
+awaits for one of the `block_max_concurrent_reads` slots to be available
+(internally implemented using a Semaphore object). Once it acquired a read
+slot, it reads the entire block file to RAM and frees the slot as soon as the
+block file is finished reading. Only after the slot is released will the
+block's data start being transferred over the network.  If the request fails to
+acquire a reading slot wihtin 15 seconds, it fails with a timeout error.
+Timeout events can be monitored through the `block_read_semaphore_timeouts`
+metric in Prometheus: a non-zero number of such events indicates an I/O
+bottleneck on HDD read speed.
+
+
 #### `lmdb_map_size` {#lmdb_map_size}
 
 This parameters can be used to set the map size used by LMDB,

doc/drafts/admin-api.md

@@ -70,7 +70,7 @@ Example response body:
 ```json
 {
   "node": "b10c110e4e854e5aa3f4637681befac755154b20059ec163254ddbfae86b09df",
-  "garageVersion": "v1.2.0",
+  "garageVersion": "v1.3.0",
   "garageFeatures": [
     "k2v",
     "lmdb",

script/helm/garage/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: garage
 description: S3-compatible object store for small self-hosted geo-distributed deployments
 type: application
-version: 0.7.1
-appVersion: "v1.2.0"
+version: 0.7.2
+appVersion: "v1.3.0"
 home: https://garagehq.deuxfleurs.fr/
 icon: https://garagehq.deuxfleurs.fr/images/garage-logo.svg
 

script/helm/garage/README.md

@@ -1,6 +1,6 @@
 # garage
 
-![Version: 0.7.1](https://img.shields.io/badge/Version-0.7.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.2.0](https://img.shields.io/badge/AppVersion-v1.2.0-informational?style=flat-square)
+![Version: 0.7.2](https://img.shields.io/badge/Version-0.7.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.3.0](https://img.shields.io/badge/AppVersion-v1.3.0-informational?style=flat-square)
 
 S3-compatible object store for small self-hosted geo-distributed deployments
 

src/api/admin/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_api_admin"
-version = "1.2.0"
+version = "1.3.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/api/common/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_api_common"
-version = "1.2.0"
+version = "1.3.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/api/common/generic_server.rs

@@ -33,7 +33,6 @@ use garage_util::metrics::{gen_trace_id, RecordDuration};
 use garage_util::socket_address::UnixOrTCPSocketAddress;
 
 use crate::helpers::{BoxBody, ErrorBody};
-use crate::signature::payload::Authorization;
 
 pub trait ApiEndpoint: Send + Sync + 'static {
 	fn name(&self) -> &'static str;
@@ -62,7 +61,7 @@ pub trait ApiHandler: Send + Sync + 'static {
 
 	/// Returns the key id used to authenticate this request. The ID returned must be safe to
 	/// log.
-	fn key_id_from_request(&self, req: &Request<IncomingBody>) -> Option<String> {
+	fn key_id_from_request(&self, _req: &Request<IncomingBody>) -> Option<String> {
 		None
 	}
 }

src/api/k2v/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_api_k2v"
-version = "1.2.0"
+version = "1.3.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/api/s3/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_api_s3"
-version = "1.2.0"
+version = "1.3.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/block/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_block"
-version = "1.2.0"
+version = "1.3.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/block/manager.rs

@@ -50,6 +50,8 @@ pub const INLINE_THRESHOLD: usize = 3072;
 // to delete the block locally.
 pub(crate) const BLOCK_GC_DELAY: Duration = Duration::from_secs(600);
 
+const BLOCK_READ_SEMAPHORE_TIMEOUT: Duration = Duration::from_secs(15);
+
 /// RPC messages used to share blocks of data between nodes
 #[derive(Debug, Serialize, Deserialize)]
 pub enum BlockRpc {
@@ -87,6 +89,7 @@ pub struct BlockManager {
 	disable_scrub: bool,
 
 	mutation_lock: Vec<Mutex<BlockManagerLocked>>,
+	read_semaphore: Semaphore,
 
 	pub rc: BlockRc,
 	pub resync: BlockResyncManager,
@@ -176,6 +179,8 @@ impl BlockManager {
 				.iter()
 				.map(|_| Mutex::new(BlockManagerLocked()))
 				.collect::<Vec<_>>(),
+
+			read_semaphore: Semaphore::new(config.block_max_concurrent_reads),
 			rc,
 			resync,
 			system,
@@ -557,9 +562,6 @@ impl BlockManager {
 			match self.find_block(hash).await {
 				Some(p) => self.read_block_from(hash, &p).await,
 				None => {
-					// Not found but maybe we should have had it ??
-					self.resync
-						.put_to_resync(hash, 2 * self.system.rpc_helper().rpc_timeout())?;
 					return Err(Error::Message(format!(
 						"block {:?} not found on node",
 						hash
@@ -581,6 +583,15 @@ impl BlockManager {
 	) -> Result<DataBlock, Error> {
 		let (header, path) = block_path.as_parts_ref();
 
+		let permit = tokio::select! {
+			sem = self.read_semaphore.acquire() => sem.ok_or_message("acquire read semaphore")?,
+			_ = tokio::time::sleep(BLOCK_READ_SEMAPHORE_TIMEOUT) => {
+				self.metrics.block_read_semaphore_timeouts.add(1);
+				debug!("read block {:?}: read_semaphore acquire timeout", hash);
+				return Err(Error::Message("read block: read_semaphore acquire timeout".into()));
+			}
+		};
+
 		let mut f = fs::File::open(&path).await?;
 		let mut data = vec![];
 		f.read_to_end(&mut data).await?;
@@ -605,6 +616,8 @@ impl BlockManager {
 			return Err(Error::CorruptData(*hash));
 		}
 
+		drop(permit);
+
 		Ok(data)
 	}
 

src/block/metrics.rs

@@ -22,6 +22,7 @@ pub struct BlockManagerMetrics {
 
 	pub(crate) bytes_read: BoundCounter<u64>,
 	pub(crate) block_read_duration: BoundValueRecorder<f64>,
+	pub(crate) block_read_semaphore_timeouts: BoundCounter<u64>,
 	pub(crate) bytes_written: BoundCounter<u64>,
 	pub(crate) block_write_duration: BoundValueRecorder<f64>,
 	pub(crate) delete_counter: BoundCounter<u64>,
@@ -119,6 +120,11 @@ impl BlockManagerMetrics {
 				.with_description("Duration of block read operations")
 				.init()
 				.bind(&[]),
+			block_read_semaphore_timeouts: meter
+				.u64_counter("block.read_semaphore_timeouts")
+				.with_description("Number of block reads that failed due to semaphore acquire timeout")
+				.init()
+				.bind(&[]),
 			bytes_written: meter
 				.u64_counter("block.bytes_written")
 				.with_description("Number of bytes written to disk")

src/block/resync.rs

@@ -133,6 +133,14 @@ impl BlockResyncManager {
 		)))
 	}
 
+	/// Clear the entire resync queue and list of errored blocks
+	/// Corresponds to `garage repair clear-resync-queue`
+	pub fn clear_resync_queue(&self) -> Result<(), Error> {
+		self.queue.clear()?;
+		self.errors.clear()?;
+		Ok(())
+	}
+
 	pub fn register_bg_vars(&self, vars: &mut vars::BgVars) {
 		let notify = self.notify.clone();
 		vars.register_rw(

src/db/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_db"
-version = "1.2.0"
+version = "1.3.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/db/lib.rs

@@ -106,32 +106,44 @@ impl Db {
 			result: Cell::new(None),
 		};
 		let tx_res = self.0.transaction(&f);
-		let ret = f
-			.result
-			.into_inner()
-			.expect("Transaction did not store result");
+		let fn_res = f.result.into_inner();
 
-		match tx_res {
-			Ok(on_commit) => match ret {
-				Ok(value) => {
-					on_commit.into_iter().for_each(|f| f());
-					Ok(value)
-				}
-				_ => unreachable!(),
-			},
-			Err(TxError::Abort(())) => match ret {
-				Err(TxError::Abort(e)) => Err(TxError::Abort(e)),
-				_ => unreachable!(),
-			},
-			Err(TxError::Db(e2)) => match ret {
-				// Ok was stored -> the error occurred when finalizing
-				// transaction
-				Ok(_) => Err(TxError::Db(e2)),
-				// An error was already stored: that's the one we want to
-				// return
-				Err(TxError::Db(e)) => Err(TxError::Db(e)),
-				_ => unreachable!(),
-			},
+		match (tx_res, fn_res) {
+			(Ok(on_commit), Some(Ok(value))) => {
+				// Transaction succeeded
+				// TxFn stored the value to return to the user in fn_res
+				// tx_res contains the on_commit list of callbacks, run them now
+				on_commit.into_iter().for_each(|f| f());
+				Ok(value)
+			}
+			(Err(TxError::Abort(())), Some(Err(TxError::Abort(e)))) => {
+				// Transaction was aborted by user code
+				// The abort error value is stored in fn_res
+				Err(TxError::Abort(e))
+			}
+			(Err(TxError::Db(_tx_e)), Some(Err(TxError::Db(fn_e)))) => {
+				// Transaction encountered a DB error in user code
+				// The error value encountered is the one in fn_res,
+				// tx_res contains only a dummy error message
+				Err(TxError::Db(fn_e))
+			}
+			(Err(TxError::Db(tx_e)), None) => {
+				// Transaction encounterred a DB error when initializing the transaction,
+				// before user code was called
+				Err(TxError::Db(tx_e))
+			}
+			(Err(TxError::Db(tx_e)), Some(Ok(_))) => {
+				// Transaction encounterred a DB error when commiting the transaction,
+				// after user code was called
+				Err(TxError::Db(tx_e))
+			}
+			(tx_res, fn_res) => {
+				panic!(
+					"unexpected error case: tx_res={:?}, fn_res={:?}",
+					tx_res.map(|_| "..."),
+					fn_res.map(|x| x.map(|_| "...").map_err(|_| "..."))
+				);
+			}
 		}
 	}
 

src/db/sqlite_adapter.rs

@@ -151,30 +151,16 @@ impl IDb for SqliteDb {
 	}
 
 	fn snapshot(&self, base_path: &PathBuf) -> Result<()> {
-		fn progress(p: rusqlite::backup::Progress) {
-			use std::sync::atomic::{AtomicU64, Ordering};
-			use std::time::{SystemTime, UNIX_EPOCH};
-
-			static LAST_LOG_TIME: AtomicU64 = AtomicU64::new(0);
-
-			let now = SystemTime::now()
-				.duration_since(UNIX_EPOCH)
-				.expect("Fix your clock :o")
-				.as_millis() as u64;
-			if now >= LAST_LOG_TIME.load(Ordering::Relaxed) + 10 * 1000 {
-				let percent = (p.pagecount - p.remaining) * 100 / p.pagecount;
-				info!("Sqlite snapshot progress: {}%", percent);
-
-				LAST_LOG_TIME.fetch_max(now, Ordering::Relaxed);
-			}
-		}
-
 		std::fs::create_dir_all(base_path)?;
-		let path = Engine::Sqlite.db_path(&base_path);
+		let path = Engine::Sqlite
+			.db_path(&base_path)
+			.into_os_string()
+			.into_string()
+			.map_err(|_| Error("invalid sqlite path string".into()))?;
 
-		self.db
-			.get()?
-			.backup(rusqlite::DatabaseName::Main, path, Some(progress))?;
+		info!("Start sqlite VACUUM INTO `{}`", path);
+		self.db.get()?.execute("VACUUM INTO ?1", params![path])?;
+		info!("Finished sqlite VACUUM INTO `{}`", path);
 
 		Ok(())
 	}

src/garage/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage"
-version = "1.2.0"
+version = "1.3.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -49,8 +49,6 @@ structopt.workspace = true
 git-version.workspace = true
 
 serde.workspace = true
-serde_json.workspace = true
-serde_bytes.workspace = true
 
 futures.workspace = true
 tokio.workspace = true

src/garage/cli/structs.rs

@@ -71,10 +71,6 @@ pub enum NodeOperation {
 	/// Connect to Garage node that is currently isolated from the system
 	#[structopt(name = "connect", version = garage_version())]
 	Connect(ConnectNodeOpt),
-
-	/// Dump the content of a metadata table as JSON lines
-	#[structopt(name = "dump", version = garage_version())]
-	Dump(DumpNodeOpt),
 }
 
 #[derive(StructOpt, Debug)]
@@ -92,12 +88,6 @@ pub struct ConnectNodeOpt {
 	pub(crate) node: String,
 }
 
-#[derive(StructOpt, Debug)]
-pub struct DumpNodeOpt {
-	/// Name of the data table to dump
-	pub(crate) what: String,
-}
-
 #[derive(StructOpt, Debug)]
 pub enum LayoutOperation {
 	/// Assign role to Garage node
@@ -476,6 +466,10 @@ pub enum RepairWhat {
 	/// Repair (resync/rebalance) the set of stored blocks in the cluster
 	#[structopt(name = "blocks", version = garage_version())]
 	Blocks,
+	/// Clear the block resync queue. The list of blocks in errored state
+	/// is cleared as well. You MUST run `garage repair blocks` after invoking this.
+	#[structopt(name = "clear-resync-queue", version = garage_version())]
+	ClearResyncQueue,
 	/// Repropagate object deletions to the version table
 	#[structopt(name = "versions", version = garage_version())]
 	Versions,

src/garage/main.rs

@@ -145,14 +145,11 @@ async fn main() {
 	let res = match opt.cmd {
 		Command::Server => server::run_server(opt.config_file, opt.secrets).await,
 		Command::OfflineRepair(repair_opt) => {
-			repair::offline::offline_repair(opt.config_file, opt.secrets, repair_opt)
+			repair::offline::offline_repair(opt.config_file, opt.secrets, repair_opt).await
 		}
 		Command::ConvertDb(conv_opt) => {
 			cli::convert_db::do_conversion(conv_opt).map_err(From::from)
 		}
-		Command::Node(NodeOperation::Dump(dump_opt)) => {
-			repair::offline::dump(opt.config_file, opt.secrets, dump_opt)
-		}
 		Command::Node(NodeOperation::NodeId(node_id_opt)) => {
 			node_id_command(opt.config_file, node_id_opt.quiet)
 		}

src/garage/repair/offline.rs

@@ -1,18 +1,14 @@
-use std::io::Write;
 use std::path::PathBuf;
 
-use serde::Serialize;
-
 use garage_util::config::*;
 use garage_util::error::*;
 
 use garage_model::garage::Garage;
-use garage_table::{replication::TableReplication, *};
 
 use crate::cli::structs::*;
 use crate::secrets::{fill_secrets, Secrets};
 
-pub fn offline_repair(
+pub async fn offline_repair(
 	config_file: PathBuf,
 	secrets: Secrets,
 	opt: OfflineRepairOpt,
@@ -49,100 +45,3 @@ pub fn offline_repair(
 
 	Ok(())
 }
-
-pub fn dump(config_file: PathBuf, secrets: Secrets, opt: DumpNodeOpt) -> Result<(), Error> {
-	let what = opt.what.as_str();
-
-	info!("Loading configuration...");
-	let config = fill_secrets(read_config(config_file)?, secrets)?;
-
-	info!("Initializing Garage main data store...");
-	let garage = Garage::new(config)?;
-
-	match what {
-		"bucket" | "buckets" => dump_table_inner(&garage.bucket_table),
-		"bucket_alias" | "bucket_aliases" => dump_table_inner(&garage.bucket_alias_table),
-		"key" | "keys" => dump_table_inner(&garage.key_table),
-		"object" | "objects" => dump_table_inner(&garage.object_table),
-		"object_counter" | "object_counters" => Err(Error::Message(
-			"object_counters cannot be JSON-serialized".into(),
-		)),
-		"mpu" => dump_table_inner(&garage.mpu_table),
-		"mpu_counter" | "mpu_counters" => Err(Error::Message(
-			"mpu_counters cannot be JSON-serialized".into(),
-		)),
-		"version" | "versions" => dump_table_inner(&garage.version_table),
-		"block_ref" | "block_refs" => dump_table_inner(&garage.block_ref_table),
-		#[cfg(feature = "k2v")]
-		"k2v_item" | "k2v_items" => dump_table_inner(&garage.k2v.item_table),
-		//#[cfg(feature = "k2v")]
-		"k2v_counter" | "k2v_counters" => Err(Error::Message(
-			"k2v_counters cannot be JSON-serialized".into(),
-		)),
-		other => {
-			let mut stdout = std::io::stdout().lock();
-			match other {
-				"cluster_layout" => Err(Error::Message(
-					"cluster_layout cannot be JSON-serialized".into(),
-				)),
-				_ => Err(Error::Message(format!("invalid thing to dump: {}", what))),
-			}
-		}
-	}
-}
-
-#[derive(Serialize)]
-struct DumpEntry<'a, T: Serialize> {
-	#[serde(with = "serde_bytes")]
-	partition_key: &'a [u8],
-	#[serde(with = "serde_bytes")]
-	sort_key: &'a [u8],
-	entry: &'a T,
-}
-
-fn dump_table_inner<F, R>(table: &Table<F, R>) -> Result<(), Error>
-where
-	F: TableSchema,
-	R: TableReplication,
-{
-	eprintln!("Dumping table {}...", F::TABLE_NAME);
-
-	let mut stdout = std::io::stdout().lock();
-
-	for line in table.data.store.iter()? {
-		let (_k, v) = line?;
-		let v_dec = table.data.decode_entry(&v)?;
-		let pkh = v_dec.partition_key().hash();
-		let dump_entry = DumpEntry {
-			partition_key: pkh.as_slice(),
-			sort_key: v_dec.sort_key().sort_key(),
-			entry: &v_dec,
-		};
-		dump_line(&mut stdout, dump_entry)?;
-	}
-	stdout.flush()?;
-
-	Ok(())
-}
-
-fn dump_line<T: Serialize>(
-	mut stdout: &mut std::io::StdoutLock<'static>,
-	dump_entry: T,
-) -> Result<(), Error> {
-	let mut ser = serde_json::ser::Serializer::with_formatter(&mut stdout, DumpFormatter);
-	dump_entry.serialize(&mut ser)?;
-	stdout.write_all(b"\n")?;
-	Ok(())
-}
-
-struct DumpFormatter;
-impl serde_json::ser::Formatter for DumpFormatter {
-	fn write_byte_array<W>(&mut self, writer: &mut W, value: &[u8]) -> std::io::Result<()>
-	where
-		W: ?Sized + std::io::Write,
-	{
-		writer.write_all(b"\"")?;
-		writer.write_all(hex::encode(&value).as_bytes())?;
-		writer.write_all(b"\"")
-	}
-}

src/garage/repair/online.rs

@@ -92,6 +92,11 @@ pub async fn launch_online_repair(
 			info!("Repairing bucket aliases (foreground)");
 			garage.locked_helper().await.repair_aliases().await?;
 		}
+		RepairWhat::ClearResyncQueue => {
+			let garage = garage.clone();
+			tokio::task::spawn_blocking(move || garage.block_manager.resync.clear_resync_queue())
+				.await??
+		}
 	}
 	Ok(())
 }

src/model/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_model"
-version = "1.2.0"
+version = "1.3.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -44,4 +44,4 @@ default = [ "lmdb", "sqlite" ]
 k2v = [ "garage_util/k2v" ]
 lmdb = [ "garage_db/lmdb" ]
 sqlite = [ "garage_db/sqlite" ]
-fjall = [ "garage_db/fjall" ]
+fjall = [ "garage_db/fjall" ]

src/net/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_net"
-version = "1.2.0"
+version = "1.3.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/rpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_rpc"
-version = "1.2.0"
+version = "1.3.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/rpc/layout/manager.rs

@@ -229,13 +229,11 @@ impl LayoutManager {
 	}
 
 	/// Save cluster layout data to disk
-	async fn save_cluster_layout(&self) -> Result<(), Error> {
+	async fn save_cluster_layout(&self) {
 		let layout = self.layout.read().unwrap().inner().clone();
-		self.persist_cluster_layout
-			.save_async(&layout)
-			.await
-			.expect("Cannot save current cluster layout");
-		Ok(())
+		if let Err(e) = self.persist_cluster_layout.save_async(&layout).await {
+			error!("Failed to save cluster_layout: {}", e);
+		}
 	}
 
 	fn broadcast_update(self: &Arc<Self>, rpc: SystemRpc) {
@@ -313,7 +311,7 @@ impl LayoutManager {
 
 			self.change_notify.notify_waiters();
 			self.broadcast_update(SystemRpc::AdvertiseClusterLayout(new_layout));
-			self.save_cluster_layout().await?;
+			self.save_cluster_layout().await;
 		}
 
 		Ok(SystemRpc::Ok)
@@ -328,7 +326,7 @@ impl LayoutManager {
 		if let Some(new_trackers) = self.merge_layout_trackers(trackers) {
 			self.change_notify.notify_waiters();
 			self.broadcast_update(SystemRpc::AdvertiseClusterLayoutTrackers(new_trackers));
-			self.save_cluster_layout().await?;
+			self.save_cluster_layout().await;
 		}
 
 		Ok(SystemRpc::Ok)

src/table/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_table"
-version = "1.2.0"
+version = "1.3.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/util/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_util"
-version = "1.2.0"
+version = "1.3.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/util/config.rs

@@ -75,6 +75,10 @@ pub struct Config {
 	)]
 	pub block_ram_buffer_max: usize,
 
+	/// Maximum number of concurrent reads of block files on disk
+	#[serde(default = "default_block_max_concurrent_reads")]
+	pub block_max_concurrent_reads: usize,
+
 	/// Skip the permission check of secret files. Useful when
 	/// POSIX ACLs (or more complex chmods) are used.
 	#[serde(default)]
@@ -280,6 +284,9 @@ fn default_block_size() -> usize {
 fn default_block_ram_buffer_max() -> usize {
 	256 * 1024 * 1024
 }
+fn default_block_max_concurrent_reads() -> usize {
+	16
+}
 
 fn default_consistency_mode() -> String {
 	"consistent".into()

src/web/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_web"
-version = "1.2.0"
+version = "1.3.0"
 authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
 edition = "2018"
 license = "AGPL-3.0"