[SKIP CI] WIP merkle todo encapsulation

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/1015

Cargo.toml

@@ -132,8 +132,8 @@ opentelemetry-contrib = "0.9"
 prometheus = "0.13"
 
 # used by the k2v-client crate only
-aws-sigv4 = { version = "1.1" }
-hyper-rustls = { version = "0.26", features = ["http2"] }
+aws-sigv4 = { version = "1.1", default-features = false }
+hyper-rustls = { version = "0.26", default-features = false, features = ["http1", "http2", "ring", "rustls-native-certs"] }
 log = "0.4"
 thiserror = "1.0"
 
@@ -141,8 +141,9 @@ thiserror = "1.0"
 assert-json-diff = "2.0"
 rustc_version = "0.4.0"
 static_init = "1.0"
-aws-sdk-config = "1.62"
-aws-sdk-s3 = "=1.68"
+aws-smithy-runtime = { version = "1.8", default-features = false, features = ["tls-rustls"] }
+aws-sdk-config = { version = "1.62", default-features = false }
+aws-sdk-s3 = { version = "1.79", default-features = false, features = ["rt-tokio"] }
 
 [profile.dev]
 #lto = "thin"	# disabled for now, adds 2-4 min to each CI build

doc/book/connect/apps/index.md

@@ -69,7 +69,7 @@ $CONFIG = array(
         'hostname' => '127.0.0.1', // Can also be a domain name, eg. garage.example.com
         'port' => 3900,            // Put your reverse proxy port or your S3 API port
         'use_ssl' => false,        // Set it to true if you have a TLS enabled reverse proxy
-        'region' => 'garage',      // Garage has only one region named "garage"
+        'region' => 'garage',      // Garage default region is named "garage", edit according to your cluster config
         'use_path_style' => true   // Garage supports only path style, must be set to true
     ],
 ],
@@ -135,7 +135,7 @@ bucket but doesn't also know the secret encryption key.
 *Click on the picture to zoom*
 
 Add a new external storage. Put what you want in "folder name" (eg. "shared"). Select "Amazon S3". Keep "Access Key" for the Authentication field.
-In Configuration, put your bucket name (eg. nextcloud), the host (eg. 127.0.0.1), the port (eg. 3900 or 443), the region (garage). Tick the SSL box if you have put an HTTPS proxy in front of garage. You must tick the "Path access" box and you must leave the "Legacy authentication (v2)" box empty. Put your Key ID (eg. GK...) and your Secret Key in the last two input boxes. Finally click on the tick symbol on the right of your screen.
+In Configuration, put your bucket name (eg. nextcloud), the host (eg. 127.0.0.1), the port (eg. 3900 or 443), the region ("garage" if you use the default, or the one your configured in your `garage.toml`). Tick the SSL box if you have put an HTTPS proxy in front of garage. You must tick the "Path access" box and you must leave the "Legacy authentication (v2)" box empty. Put your Key ID (eg. GK...) and your Secret Key in the last two input boxes. Finally click on the tick symbol on the right of your screen.
 
 Now go to your "Files" app and a new "linked folder" has appeared with the name you chose earlier (eg. "shared").
 
@@ -238,7 +238,7 @@ object_storage:
   # Put localhost only if you have a garage instance running on that node
   endpoint: 'http://localhost:3900' # or "garage.example.com" if you have TLS on port 443
 
-  # Garage supports only one region for now, named garage
+  # Garage default region is named "garage", edit according to your config
   region: 'garage'
 
   credentials:
@@ -441,7 +441,7 @@ media_storage_providers:
   store_synchronous: True  # do we want to wait that the file has been written before returning?
   config:
     bucket: matrix       # the name of our bucket, we chose matrix earlier
-    region_name: garage  # only "garage" is supported for the region field
+    region_name: garage  # "garage" by default, edit according to your cluster config
     endpoint_url: http://localhost:3900 # the path to the S3 endpoint
     access_key_id: "GKxxx" # your Key ID
     secret_access_key: "xxxx" # your Secret Key

doc/book/cookbook/kubernetes.md

@@ -86,3 +86,62 @@ helm delete --namespace garage garage
 ```
 
 Note that this will leave behind custom CRD `garagenodes.deuxfleurs.fr`, which must be removed manually if desired.
+
+## Increase PVC size on running Garage instances
+
+Since the Garage Helm chart creates the data and meta PVC based on `StatefulSet` templates, increasing the PVC size can be a bit tricky.
+
+### Confirm the `StorageClass` used for Garage supports volume expansion
+
+Confirm the storage class used for garage.
+
+```bash
+kubectl -n garage get pvc 
+NAME            STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS     VOLUMEATTRIBUTESCLASS   AGE
+data-garage-0   Bound    pvc-080360c9-8ce3-4acf-8579-1701e57b7f3f   30Gi       RWO            longhorn-local   <unset>                 77d
+data-garage-1   Bound    pvc-ab8ba697-6030-4fc7-ab3c-0d6df9e3dbc0   30Gi       RWO            longhorn-local   <unset>                 5d8h
+data-garage-2   Bound    pvc-3ab37551-0231-4604-986d-136d0fd950ec   30Gi       RWO            longhorn-local   <unset>                 5d5h
+meta-garage-0   Bound    pvc-3b457302-3023-4169-846e-c928c5f2ea65   3Gi        RWO            longhorn-local   <unset>                 77d
+meta-garage-1   Bound    pvc-49ace2b9-5c85-42df-9247-51c4cf64b460   3Gi        RWO            longhorn-local   <unset>                 5d8h
+meta-garage-2   Bound    pvc-99e2e50f-42b4-4128-ae2f-b52629259723   3Gi        RWO            longhorn-local   <unset>                 5d5h
+```
+
+In this case, the storage class is `longhorn-local`. Now, check if `ALLOWVOLUMEEXPANSION` is true for the used `StorageClass`.
+
+```bash
+kubectl get storageclasses.storage.k8s.io longhorn-local
+NAME             PROVISIONER          RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
+longhorn-local   driver.longhorn.io   Delete          Immediate           true                   103d
+```
+
+If your `StorageClass` does not support volume expansion, double check if you can enable it. Otherwise, your only real option is to spin up a new Garage cluster with increased size and migrate all data over.
+
+If your `StorageClass` supports expansion, you are free to continue.
+
+### Increase the size of the PVCs
+
+Increase the size of all PVCs to your desired size.
+
+```bash
+kubectl -n garage edit pvc data-garage-0
+kubectl -n garage edit pvc data-garage-1
+kubectl -n garage edit pvc data-garage-2
+kubectl -n garage edit pvc meta-garage-0
+kubectl -n garage edit pvc meta-garage-1
+kubectl -n garage edit pvc meta-garage-2
+```
+
+### Increase the size of the `StatefulSet` PVC template
+
+This is an optional step, but if not done, future instances of Garage will be created with the original size from the template.
+
+```bash
+kubectl -n garage delete sts --cascade=orphan garage 
+statefulset.apps "garage" deleted
+```
+
+This will remove the Garage `StatefulSet` but leave the pods running. It may seem destructive but needs to be done this way since edits to the size of PVC templates are prohibited.
+
+### Redeploy the `StatefulSet`
+
+Now the size of future PVCs can be increased, and the Garage Helm chart can be upgraded. The new `StatefulSet` should take ownership of the orphaned pods again.

doc/book/quick-start/_index.md

@@ -129,9 +129,9 @@ docker run \
   -d \
   --name garaged \
   -p 3900:3900 -p 3901:3901 -p 3902:3902 -p 3903:3903 \
-  -v /etc/garage.toml:/path/to/garage.toml \
-  -v /var/lib/garage/meta:/path/to/garage/meta \
-  -v /var/lib/garage/data:/path/to/garage/data \
+  -v /path/to/garage.toml:/etc/garage.toml \
+  -v /path/to/garage/meta:/var/lib/garage/meta \
+  -v /path/to/garage/data:/var/lib/garage/data \
   dxflrs/garage:v1.1.0
 ```
 

script/helm/garage/Chart.yaml

@@ -1,24 +1,18 @@
 apiVersion: v2
 name: garage
 description: S3-compatible object store for small self-hosted geo-distributed deployments
-
-# A chart can be either an 'application' or a 'library' chart.
-#
-# Application charts are a collection of templates that can be packaged into versioned archives
-# to be deployed.
-#
-# Library charts provide useful utilities or functions for the chart developer. They're included as
-# a dependency of application charts to inject those utilities and functions into the rendering
-# pipeline. Library charts do not define any templates and therefore cannot be deployed.
 type: application
-
-# This is the chart version. This version number should be incremented each time you make changes
-# to the chart and its templates, including the app version.
-# Versions are expected to follow Semantic Versioning (https://semver.org/)
 version: 0.7.0
-
-# This is the version number of the application being deployed. This version number should be
-# incremented each time you make changes to the application. Versions are not expected to
-# follow Semantic Versioning. They should reflect the version the application is using.
-# It is recommended to use it with quotes.
 appVersion: "v1.1.0"
+home: https://garagehq.deuxfleurs.fr/
+icon: https://garagehq.deuxfleurs.fr/images/garage-logo.svg
+
+keywords:
+- geo-distributed
+- read-after-write-consistency
+- s3-compatible
+
+sources:
+- https://git.deuxfleurs.fr/Deuxfleurs/garage.git
+
+maintainers: []

script/helm/garage/README.md

@@ -1,9 +1,15 @@
 # garage
 
-![Version: 0.6.0](https://img.shields.io/badge/Version-0.6.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.0.1](https://img.shields.io/badge/AppVersion-v1.0.1-informational?style=flat-square)
+![Version: 0.7.0](https://img.shields.io/badge/Version-0.7.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.1.0](https://img.shields.io/badge/AppVersion-v1.1.0-informational?style=flat-square)
 
 S3-compatible object store for small self-hosted geo-distributed deployments
 
+**Homepage:** <https://garagehq.deuxfleurs.fr/>
+
+## Source Code
+
+* <https://git.deuxfleurs.fr/Deuxfleurs/garage.git>
+
 ## Values
 
 | Key | Type | Default | Description |
@@ -23,6 +29,7 @@ S3-compatible object store for small self-hosted geo-distributed deployments
 | garage.existingConfigMap | string | `""` | if not empty string, allow using an existing ConfigMap for the garage.toml, if set, ignores garage.toml |
 | garage.garageTomlString | string | `""` | String Template for the garage configuration if set, ignores above values. Values can be templated, see https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/ |
 | garage.kubernetesSkipCrd | bool | `false` | Set to true if you want to use k8s discovery but install the CRDs manually outside of the helm chart, for example if you operate at namespace level without cluster ressources |
+| garage.metadataAutoSnapshotInterval | string | `""` | If this value is set, Garage will automatically take a snapshot of the metadata DB file at a regular interval and save it in the metadata directory. https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#metadata_auto_snapshot_interval |
 | garage.replicationMode | string | `"3"` | Default to 3 replicas, see the replication_mode section at https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#replication-mode |
 | garage.rpcBindAddr | string | `"[::]:3901"` |  |
 | garage.rpcSecret | string | `""` | If not given, a random secret will be generated and stored in a Secret object |
@@ -49,6 +56,7 @@ S3-compatible object store for small self-hosted geo-distributed deployments
 | initImage.pullPolicy | string | `"IfNotPresent"` |  |
 | initImage.repository | string | `"busybox"` |  |
 | initImage.tag | string | `"stable"` |  |
+| livenessProbe | object | `{}` | Specifies a livenessProbe |
 | monitoring.metrics.enabled | bool | `false` | If true, a service for monitoring is created with a prometheus.io/scrape annotation |
 | monitoring.metrics.serviceMonitor.enabled | bool | `false` | If true, a ServiceMonitor CRD is created for a prometheus operator https://github.com/coreos/prometheus-operator |
 | monitoring.metrics.serviceMonitor.interval | string | `"15s"` |  |
@@ -71,6 +79,7 @@ S3-compatible object store for small self-hosted geo-distributed deployments
 | podSecurityContext.runAsGroup | int | `1000` |  |
 | podSecurityContext.runAsNonRoot | bool | `true` |  |
 | podSecurityContext.runAsUser | int | `1000` |  |
+| readinessProbe | object | `{}` | Specifies a readinessProbe |
 | resources | object | `{}` |  |
 | securityContext.capabilities | object | `{"drop":["ALL"]}` | The default security context is heavily restricted, feel free to tune it to your requirements |
 | securityContext.readOnlyRootFilesystem | bool | `true` |  |

script/helm/garage/templates/configmap.yaml

@@ -19,6 +19,10 @@ data:
 
     compression_level = {{ .Values.garage.compressionLevel }}
 
+    {{- if .Values.garage.metadataAutoSnapshotInterval }}
+    metadata_auto_snapshot_interval = {{ .Values.garage.metadataAutoSnapshotInterval | quote }}
+    {{- end }}
+
     rpc_bind_addr = "{{ .Values.garage.rpcBindAddr }}"
     # rpc_secret will be populated by the init container from a k8s secret object
     rpc_secret = "__RPC_SECRET_REPLACE__"

script/helm/garage/templates/workload.yaml

@@ -78,15 +78,14 @@ spec:
           {{- with .Values.extraVolumeMounts }}
             {{- toYaml . | nindent 12 }}
           {{- end }}
-          # TODO
-          # livenessProbe:
-          #   httpGet:
-          #     path: /
-          #     port: 3900
-          # readinessProbe:
-          #   httpGet:
-          #     path: /
-          #     port: 3900
+          {{- with .Values.livenessProbe }}
+          livenessProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.readinessProbe }}
+          readinessProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
       volumes:

script/helm/garage/values.yaml

@@ -21,6 +21,10 @@ garage:
   # https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#compression-level
   compressionLevel: "1"
 
+  # -- If this value is set, Garage will automatically take a snapshot of the metadata DB file at a regular interval and save it in the metadata directory.
+  # https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#metadata_auto_snapshot_interval
+  metadataAutoSnapshotInterval: ""
+
   rpcBindAddr: "[::]:3901"
   # -- If not given, a random secret will be generated and stored in a Secret object
   rpcSecret: ""
@@ -191,6 +195,21 @@ resources: {}
   #   cpu: 100m
   #   memory: 512Mi
 
+# -- Specifies a livenessProbe
+livenessProbe: {}
+  #httpGet:
+  #  path: /health
+  #  port: 3903
+  #initialDelaySeconds: 5
+  #periodSeconds: 30
+# -- Specifies a readinessProbe
+readinessProbe: {}
+  #httpGet:
+  #  path: /health
+  #  port: 3903
+  #initialDelaySeconds: 5
+  #periodSeconds: 30
+
 nodeSelector: {}
 
 tolerations: []

src/api/admin/bucket.rs

@@ -419,17 +419,11 @@ pub async fn handle_update_bucket(
 
 	if let Some(wa) = req.website_access {
 		if wa.enabled {
-			let (redirect_all, routing_rules) = match state.website_config.get() {
-				Some(wc) => (wc.redirect_all.clone(), wc.routing_rules.clone()),
-				None => (None, Vec::new()),
-			};
 			state.website_config.update(Some(WebsiteConfig {
 				index_document: wa.index_document.ok_or_bad_request(
 					"Please specify indexDocument when enabling website access.",
 				)?,
 				error_document: wa.error_document,
-				redirect_all,
-				routing_rules,
 			}));
 		} else {
 			if wa.index_document.is_some() || wa.error_document.is_some() {

src/api/s3/website.rs

@@ -3,7 +3,7 @@ use quick_xml::de::from_reader;
 use hyper::{header::HeaderName, Request, Response, StatusCode};
 use serde::{Deserialize, Serialize};
 
-use garage_model::bucket_table::{self, *};
+use garage_model::bucket_table::*;
 
 use garage_api_common::helpers::*;
 
@@ -26,28 +26,7 @@ pub async fn handle_get_website(ctx: ReqCtx) -> Result<Response<ResBody>, Error>
 				suffix: Value(website.index_document.to_string()),
 			}),
 			redirect_all_requests_to: None,
-			routing_rules: RoutingRules {
-				rules: website
-					.routing_rules
-					.clone()
-					.into_iter()
-					.map(|rule| RoutingRule {
-						condition: rule.condition.map(|cond| Condition {
-							http_error_code: cond.http_error_code.map(|c| IntValue(c as i64)),
-							prefix: cond.prefix.map(Value),
-						}),
-						redirect: Redirect {
-							hostname: rule.redirect.hostname.map(Value),
-							http_redirect_code: Some(IntValue(
-								rule.redirect.http_redirect_code as i64,
-							)),
-							protocol: rule.redirect.protocol.map(Value),
-							replace_full: rule.redirect.replace_key.map(Value),
-							replace_prefix: rule.redirect.replace_key_prefix.map(Value),
-						},
-					})
-					.collect(),
-			},
+			routing_rules: None,
 		};
 		let xml = to_xml_with_header(&wc)?;
 		Ok(Response::builder()
@@ -118,28 +97,18 @@ pub struct WebsiteConfiguration {
 	pub index_document: Option<Suffix>,
 	#[serde(rename = "RedirectAllRequestsTo")]
 	pub redirect_all_requests_to: Option<Target>,
-	#[serde(
-		rename = "RoutingRules",
-		default,
-		skip_serializing_if = "RoutingRules::is_empty"
-	)]
-	pub routing_rules: RoutingRules,
-}
-
-#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Default)]
-pub struct RoutingRules {
-	#[serde(rename = "RoutingRule")]
-	pub rules: Vec<RoutingRule>,
-}
-
-impl RoutingRules {
-	fn is_empty(&self) -> bool {
-		self.rules.is_empty()
-	}
+	#[serde(rename = "RoutingRules")]
+	pub routing_rules: Option<Vec<RoutingRule>>,
 }
 
 #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
 pub struct RoutingRule {
+	#[serde(rename = "RoutingRule")]
+	pub inner: RoutingRuleInner,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+pub struct RoutingRuleInner {
 	#[serde(rename = "Condition")]
 	pub condition: Option<Condition>,
 	#[serde(rename = "Redirect")]
@@ -193,7 +162,7 @@ impl WebsiteConfiguration {
 		if self.redirect_all_requests_to.is_some()
 			&& (self.error_document.is_some()
 				|| self.index_document.is_some()
-				|| !self.routing_rules.is_empty())
+				|| self.routing_rules.is_some())
 		{
 			return Err(Error::bad_request(
 				"Bad XML: can't have RedirectAllRequestsTo and other fields",
@@ -208,15 +177,10 @@ impl WebsiteConfiguration {
 		if let Some(ref rart) = self.redirect_all_requests_to {
 			rart.validate()?;
 		}
-		for rr in &self.routing_rules.rules {
-			rr.validate()?;
-		}
-		if self.routing_rules.rules.len() > 1000 {
-			// we will do linear scans, best to avoid overly long configuration. The
-			// limit was choosen arbitrarily
-			return Err(Error::bad_request(
-				"Bad XML: RoutingRules can't have more than 1000 child elements",
-			));
+		if let Some(ref rrs) = self.routing_rules {
+			for rr in rrs {
+				rr.inner.validate()?;
+			}
 		}
 
 		Ok(())
@@ -225,7 +189,11 @@ impl WebsiteConfiguration {
 	pub fn into_garage_website_config(self) -> Result<WebsiteConfig, Error> {
 		if self.redirect_all_requests_to.is_some() {
 			Err(Error::NotImplemented(
-				"RedirectAllRequestsTo is not currently implemented in Garage, however its effect can be emulated using a single inconditional RoutingRule.".into(),
+				"S3 website redirects are not currently implemented in Garage.".into(),
+			))
+		} else if self.routing_rules.map(|x| !x.is_empty()).unwrap_or(false) {
+			Err(Error::NotImplemented(
+				"S3 routing rules are not currently implemented in Garage.".into(),
 			))
 		} else {
 			Ok(WebsiteConfig {
@@ -234,36 +202,6 @@ impl WebsiteConfiguration {
 					.map(|x| x.suffix.0)
 					.unwrap_or_else(|| "index.html".to_string()),
 				error_document: self.error_document.map(|x| x.key.0),
-				redirect_all: None,
-				routing_rules: self
-					.routing_rules
-					.rules
-					.into_iter()
-					.map(|rule| {
-						bucket_table::RoutingRule {
-							condition: rule.condition.map(|condition| {
-								bucket_table::RedirectCondition {
-									http_error_code: condition.http_error_code.map(|c| c.0 as u16),
-									prefix: condition.prefix.map(|p| p.0),
-								}
-							}),
-							redirect: bucket_table::Redirect {
-								hostname: rule.redirect.hostname.map(|h| h.0),
-								protocol: rule.redirect.protocol.map(|p| p.0),
-								// aws default to 301, which i find punitive in case of
-								// missconfiguration (can be permanently cached on the
-								// user agent)
-								http_redirect_code: rule
-									.redirect
-									.http_redirect_code
-									.map(|c| c.0 as u16)
-									.unwrap_or(302),
-								replace_key_prefix: rule.redirect.replace_prefix.map(|k| k.0),
-								replace_key: rule.redirect.replace_full.map(|k| k.0),
-							},
-						}
-					})
-					.collect(),
 			})
 		}
 	}
@@ -304,69 +242,37 @@ impl Target {
 	}
 }
 
-impl RoutingRule {
+impl RoutingRuleInner {
 	pub fn validate(&self) -> Result<(), Error> {
-		if let Some(condition) = &self.condition {
-			condition.validate()?;
-		}
-		self.redirect.validate()
-	}
-}
-
-impl Condition {
-	pub fn validate(&self) -> Result<bool, Error> {
-		if let Some(ref error_code) = self.http_error_code {
-			// TODO do other error codes make sense? Aws only allows 4xx and 5xx
-			if error_code.0 != 404 {
-				return Err(Error::bad_request(
-					"Bad XML: HttpErrorCodeReturnedEquals must be 404 or absent",
-				));
-			}
-		}
-		Ok(self.prefix.is_some())
+		let has_prefix = self
+			.condition
+			.as_ref()
+			.and_then(|c| c.prefix.as_ref())
+			.is_some();
+		self.redirect.validate(has_prefix)
 	}
 }
 
 impl Redirect {
-	pub fn validate(&self) -> Result<(), Error> {
-		if self.replace_prefix.is_some() && self.replace_full.is_some() {
-			return Err(Error::bad_request(
-				"Bad XML: both ReplaceKeyPrefixWith and ReplaceKeyWith are set",
-			));
+	pub fn validate(&self, has_prefix: bool) -> Result<(), Error> {
+		if self.replace_prefix.is_some() {
+			if self.replace_full.is_some() {
+				return Err(Error::bad_request(
+					"Bad XML: both ReplaceKeyPrefixWith and ReplaceKeyWith are set",
+				));
+			}
+			if !has_prefix {
+				return Err(Error::bad_request(
+					"Bad XML: ReplaceKeyPrefixWith is set, but  KeyPrefixEquals isn't",
+				));
+			}
 		}
 		if let Some(ref protocol) = self.protocol {
 			if protocol.0 != "http" && protocol.0 != "https" {
 				return Err(Error::bad_request("Bad XML: invalid protocol"));
 			}
 		}
-		if let Some(ref http_redirect_code) = self.http_redirect_code {
-			match http_redirect_code.0 {
-				// aws allows all 3xx except 300, but some are non-sensical (not modified,
-				// use proxy...)
-				301 | 302 | 303 | 307 | 308 => {
-					if self.hostname.is_none() && self.protocol.is_some() {
-						return Err(Error::bad_request(
-							"Bad XML: HostName must be set if Protocol is set",
-						));
-					}
-				}
-				// aws doesn't allow these codes, but netlify does, and it seems like a
-				// cool feature (change the page seen without changing the url shown by the
-				// user agent)
-				200 | 404 => {
-					if self.hostname.is_some() || self.protocol.is_some() {
-						// hostname would mean different bucket, protocol doesn't make
-						// sense
-						return Err(Error::bad_request(
-                                        "Bad XML: an HttpRedirectCode of 200 is not acceptable alongside HostName or Protocol",
-                                ));
-					}
-				}
-				_ => {
-					return Err(Error::bad_request("Bad XML: invalid HttpRedirectCode"));
-				}
-			}
-		}
+		// TODO there are probably more invalid cases, but which ones?
 		Ok(())
 	}
 }
@@ -405,15 +311,6 @@ mod tests {
             <ReplaceKeyWith>fullkey</ReplaceKeyWith>
          </Redirect>
       </RoutingRule>
-      <RoutingRule>
-         <Condition>
-            <KeyPrefixEquals></KeyPrefixEquals>
-         </Condition>
-         <Redirect>
-            <HttpRedirectCode>404</HttpRedirectCode>
-            <ReplaceKeyWith>missing</ReplaceKeyWith>
-         </Redirect>
-      </RoutingRule>
    </RoutingRules>
 </WebsiteConfiguration>"#;
 		let conf: WebsiteConfiguration = from_str(message).unwrap();
@@ -429,36 +326,21 @@ mod tests {
 				hostname: Value("garage.tld".to_owned()),
 				protocol: Some(Value("https".to_owned())),
 			}),
-			routing_rules: RoutingRules {
-				rules: vec![
-					RoutingRule {
-						condition: Some(Condition {
-							http_error_code: Some(IntValue(404)),
-							prefix: Some(Value("prefix1".to_owned())),
-						}),
-						redirect: Redirect {
-							hostname: Some(Value("gara.ge".to_owned())),
-							protocol: Some(Value("http".to_owned())),
-							http_redirect_code: Some(IntValue(303)),
-							replace_prefix: Some(Value("prefix2".to_owned())),
-							replace_full: Some(Value("fullkey".to_owned())),
-						},
+			routing_rules: Some(vec![RoutingRule {
+				inner: RoutingRuleInner {
+					condition: Some(Condition {
+						http_error_code: Some(IntValue(404)),
+						prefix: Some(Value("prefix1".to_owned())),
+					}),
+					redirect: Redirect {
+						hostname: Some(Value("gara.ge".to_owned())),
+						protocol: Some(Value("http".to_owned())),
+						http_redirect_code: Some(IntValue(303)),
+						replace_prefix: Some(Value("prefix2".to_owned())),
+						replace_full: Some(Value("fullkey".to_owned())),
 					},
-					RoutingRule {
-						condition: Some(Condition {
-							http_error_code: None,
-							prefix: Some(Value("".to_owned())),
-						}),
-						redirect: Redirect {
-							hostname: None,
-							protocol: None,
-							http_redirect_code: Some(IntValue(404)),
-							replace_prefix: None,
-							replace_full: Some(Value("missing".to_owned())),
-						},
-					},
-				],
-			},
+				},
+			}]),
 		};
 		assert_eq! {
 			ref_value,

src/block/manager.rs

@@ -17,6 +17,7 @@ use opentelemetry::{
 	Context,
 };
 
+use garage_net::endpoint::RpcInFlightLimiter;
 use garage_net::stream::{read_stream_to_end, stream_asyncread, ByteStream};
 
 use garage_db as db;
@@ -295,6 +296,7 @@ impl BlockManager {
 				&node_id,
 				BlockRpc::GetBlock(*hash, order_tag),
 				priority,
+				RpcInFlightLimiter::TableWrite,
 			);
 			tokio::select! {
 				res = rpc => {

src/db/lmdb_adapter.rs

@@ -109,7 +109,7 @@ impl IDb for LmdbDb {
 		let mut path = to.clone();
 		path.push("data.mdb");
 		self.db
-			.copy_to_path(path, heed::CompactionOption::Disabled)?;
+			.copy_to_path(path, heed::CompactionOption::Enabled)?;
 		Ok(())
 	}
 

src/garage/Cargo.toml

@@ -62,6 +62,7 @@ syslog-tracing = { workspace = true, optional = true }
 garage_api_common.workspace = true
 
 aws-sdk-s3.workspace = true
+aws-smithy-runtime.workspace = true
 chrono.workspace = true
 http.workspace = true
 hmac.workspace = true

src/garage/admin/bucket.rs

@@ -390,15 +390,9 @@ impl AdminRpcHandler {
 		}
 
 		let website = if query.allow {
-			let (redirect_all, routing_rules) = match bucket_state.website_config.get() {
-				Some(wc) => (wc.redirect_all.clone(), wc.routing_rules.clone()),
-				None => (None, Vec::new()),
-			};
 			Some(WebsiteConfig {
 				index_document: query.index_document.clone(),
 				error_document: query.error_document.clone(),
-				redirect_all,
-				routing_rules,
 			})
 		} else {
 			None

src/garage/admin/mod.rs

@@ -13,6 +13,8 @@ use serde::{Deserialize, Serialize};
 
 use format_table::format_table_to_string;
 
+use garage_net::endpoint::RpcInFlightLimiter;
+
 use garage_util::background::BackgroundRunner;
 use garage_util::data::*;
 use garage_util::error::Error as GarageError;
@@ -118,6 +120,7 @@ impl AdminRpcHandler {
 						&node,
 						AdminRpc::LaunchRepair(opt_to_send.clone()),
 						PRIO_NORMAL,
+						RpcInFlightLimiter::NoLimit,
 					)
 					.await;
 				if !matches!(resp, Ok(Ok(_))) {
@@ -164,7 +167,12 @@ impl AdminRpcHandler {
 				let node_id = (*node).into();
 				match self
 					.endpoint
-					.call(&node_id, AdminRpc::Stats(opt), PRIO_NORMAL)
+					.call(
+						&node_id,
+						AdminRpc::Stats(opt),
+						PRIO_NORMAL,
+						RpcInFlightLimiter::NoLimit,
+					)
 					.await
 				{
 					Ok(Ok(AdminRpc::Ok(s))) => writeln!(&mut ret, "{}", s).unwrap(),
@@ -407,6 +415,7 @@ impl AdminRpcHandler {
 							variable: variable.clone(),
 						}),
 						PRIO_NORMAL,
+						RpcInFlightLimiter::NoLimit,
 					)
 					.await??
 				{
@@ -456,6 +465,7 @@ impl AdminRpcHandler {
 							value: value.to_string(),
 						}),
 						PRIO_NORMAL,
+						RpcInFlightLimiter::NoLimit,
 					)
 					.await??
 				{
@@ -488,6 +498,7 @@ impl AdminRpcHandler {
 							&to,
 							AdminRpc::MetaOperation(MetaOperation::Snapshot { all: false }),
 							PRIO_NORMAL,
+							RpcInFlightLimiter::NoLimit,
 						)
 						.await?
 				}))

src/garage/cli/cmd.rs

@@ -2,6 +2,7 @@ use std::collections::{HashMap, HashSet};
 use std::time::Duration;
 
 use format_table::format_table;
+use garage_net::endpoint::RpcInFlightLimiter;
 use garage_util::error::*;
 
 use garage_rpc::layout::*;
@@ -200,7 +201,12 @@ pub async fn cmd_connect(
 	args: ConnectNodeOpt,
 ) -> Result<(), Error> {
 	match rpc_cli
-		.call(&rpc_host, SystemRpc::Connect(args.node), PRIO_NORMAL)
+		.call(
+			&rpc_host,
+			SystemRpc::Connect(args.node),
+			PRIO_NORMAL,
+			RpcInFlightLimiter::NoLimit,
+		)
 		.await??
 	{
 		SystemRpc::Ok => {
@@ -216,7 +222,10 @@ pub async fn cmd_admin(
 	rpc_host: NodeID,
 	args: AdminRpc,
 ) -> Result<(), HelperError> {
-	match rpc_cli.call(&rpc_host, args, PRIO_NORMAL).await?? {
+	match rpc_cli
+		.call(&rpc_host, args, PRIO_NORMAL, RpcInFlightLimiter::NoLimit)
+		.await??
+	{
 		AdminRpc::Ok(msg) => {
 			println!("{}", msg);
 		}
@@ -271,7 +280,12 @@ pub async fn fetch_status(
 	rpc_host: NodeID,
 ) -> Result<Vec<KnownNodeInfo>, Error> {
 	match rpc_cli
-		.call(&rpc_host, SystemRpc::GetKnownNodes, PRIO_NORMAL)
+		.call(
+			&rpc_host,
+			SystemRpc::GetKnownNodes,
+			PRIO_NORMAL,
+			RpcInFlightLimiter::NoLimit,
+		)
 		.await??
 	{
 		SystemRpc::ReturnKnownNodes(nodes) => Ok(nodes),

src/garage/cli/layout.rs

@@ -1,6 +1,7 @@
 use bytesize::ByteSize;
 
 use format_table::format_table;
+use garage_net::endpoint::RpcInFlightLimiter;
 use garage_util::crdt::Crdt;
 use garage_util::error::*;
 
@@ -45,7 +46,12 @@ pub async fn cmd_assign_role(
 	args: AssignRoleOpt,
 ) -> Result<(), Error> {
 	let status = match rpc_cli
-		.call(&rpc_host, SystemRpc::GetKnownNodes, PRIO_NORMAL)
+		.call(
+			&rpc_host,
+			SystemRpc::GetKnownNodes,
+			PRIO_NORMAL,
+			RpcInFlightLimiter::NoLimit,
+		)
 		.await??
 	{
 		SystemRpc::ReturnKnownNodes(nodes) => nodes,
@@ -475,7 +481,12 @@ pub async fn fetch_layout(
 	rpc_host: NodeID,
 ) -> Result<LayoutHistory, Error> {
 	match rpc_cli
-		.call(&rpc_host, SystemRpc::PullClusterLayout, PRIO_NORMAL)
+		.call(
+			&rpc_host,
+			SystemRpc::PullClusterLayout,
+			PRIO_NORMAL,
+			RpcInFlightLimiter::NoLimit,
+		)
 		.await??
 	{
 		SystemRpc::AdvertiseClusterLayout(t) => Ok(t),
@@ -493,6 +504,7 @@ pub async fn send_layout(
 			&rpc_host,
 			SystemRpc::AdvertiseClusterLayout(layout),
 			PRIO_NORMAL,
+			RpcInFlightLimiter::NoLimit,
 		)
 		.await??;
 	Ok(())

src/garage/main.rs

@@ -244,7 +244,7 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
 	// Generate a temporary keypair for our RPC client
 	let (_pk, sk) = sodiumoxide::crypto::sign::ed25519::gen_keypair();
 
-	let netapp = NetApp::new(GARAGE_VERSION_TAG, network_key, sk, None);
+	let netapp = NetApp::new(GARAGE_VERSION_TAG, network_key, sk, None, None);
 
 	// Find and parse the address of the target host
 	let (id, addr, is_default_addr) = if let Some(h) = opt.rpc_host {

src/garage/tests/s3/website.rs

@@ -5,10 +5,7 @@ use crate::json_body;
 use assert_json_diff::assert_json_eq;
 use aws_sdk_s3::{
 	primitives::ByteStream,
-	types::{
-		Condition, CorsConfiguration, CorsRule, ErrorDocument, IndexDocument, Protocol, Redirect,
-		RoutingRule, WebsiteConfiguration,
-	},
+	types::{CorsConfiguration, CorsRule, ErrorDocument, IndexDocument, WebsiteConfiguration},
 };
 use http::{Request, StatusCode};
 use http_body_util::BodyExt;
@@ -536,444 +533,3 @@ async fn test_website_check_domain() {
 		})
 	);
 }
-
-#[tokio::test]
-async fn test_website_redirect_full_bucket() {
-	const BCKT_NAME: &str = "my-redirect-full";
-	let ctx = common::context();
-	let bucket = ctx.create_bucket(BCKT_NAME);
-
-	let conf = WebsiteConfiguration::builder()
-		.routing_rules(
-			RoutingRule::builder()
-				.condition(Condition::builder().key_prefix_equals("").build())
-				.redirect(
-					Redirect::builder()
-						.protocol(Protocol::Https)
-						.host_name("other.tld")
-						.replace_key_prefix_with("")
-						.build(),
-				)
-				.build(),
-		)
-		.build();
-
-	ctx.client
-		.put_bucket_website()
-		.bucket(&bucket)
-		.website_configuration(conf)
-		.send()
-		.await
-		.unwrap();
-
-	let req = Request::builder()
-		.method("GET")
-		.uri(format!("http://127.0.0.1:{}/my-path", ctx.garage.web_port))
-		.header("Host", format!("{}.web.garage", BCKT_NAME))
-		.body(Body::new(Bytes::new()))
-		.unwrap();
-
-	let client = Client::builder(TokioExecutor::new()).build_http();
-	let resp = client.request(req).await.unwrap();
-	assert_eq!(resp.status(), StatusCode::FOUND);
-	assert_eq!(
-		resp.headers()
-			.get(hyper::header::LOCATION)
-			.unwrap()
-			.to_str()
-			.unwrap(),
-		"https://other.tld/my-path"
-	);
-}
-
-#[tokio::test]
-async fn test_website_redirect() {
-	const BCKT_NAME: &str = "my-redirect";
-	let ctx = common::context();
-	let bucket = ctx.create_bucket(BCKT_NAME);
-
-	ctx.client
-		.put_object()
-		.bucket(&bucket)
-		.key("index.html")
-		.body(ByteStream::from_static(b"index"))
-		.send()
-		.await
-		.unwrap();
-	ctx.client
-		.put_object()
-		.bucket(&bucket)
-		.key("404.html")
-		.body(ByteStream::from_static(b"main 404"))
-		.send()
-		.await
-		.unwrap();
-	ctx.client
-		.put_object()
-		.bucket(&bucket)
-		.key("static-file")
-		.body(ByteStream::from_static(b"static file"))
-		.send()
-		.await
-		.unwrap();
-
-	let mut conf = WebsiteConfiguration::builder()
-		.index_document(
-			IndexDocument::builder()
-				.suffix("home.html")
-				.build()
-				.unwrap(),
-		)
-		.error_document(ErrorDocument::builder().key("404.html").build().unwrap());
-
-	for (prefix, condition) in [("unconditional", false), ("conditional", true)] {
-		let code = condition.then(|| "404".to_string());
-		conf = conf
-			// simple redirect
-			.routing_rules(
-				RoutingRule::builder()
-					.condition(
-						Condition::builder()
-							.set_http_error_code_returned_equals(code.clone())
-							.key_prefix_equals(format!("{prefix}/redirect-prefix/"))
-							.build(),
-					)
-					.redirect(
-						Redirect::builder()
-							.http_redirect_code("302")
-							.replace_key_prefix_with("other-prefix/")
-							.build(),
-					)
-					.build(),
-			)
-			.routing_rules(
-				RoutingRule::builder()
-					.condition(
-						Condition::builder()
-							.set_http_error_code_returned_equals(code.clone())
-							.key_prefix_equals(format!("{prefix}/redirect-prefix-307/"))
-							.build(),
-					)
-					.redirect(
-						Redirect::builder()
-							.http_redirect_code("307")
-							.replace_key_prefix_with("other-prefix/")
-							.build(),
-					)
-					.build(),
-			)
-			// simple redirect
-			.routing_rules(
-				RoutingRule::builder()
-					.condition(
-						Condition::builder()
-							.set_http_error_code_returned_equals(code.clone())
-							.key_prefix_equals(format!("{prefix}/redirect-fixed/"))
-							.build(),
-					)
-					.redirect(
-						Redirect::builder()
-							.http_redirect_code("302")
-							.replace_key_with("fixed_key")
-							.build(),
-					)
-					.build(),
-			)
-			// stream other file
-			.routing_rules(
-				RoutingRule::builder()
-					.condition(
-						Condition::builder()
-							.set_http_error_code_returned_equals(code.clone())
-							.key_prefix_equals(format!("{prefix}/stream-fixed/"))
-							.build(),
-					)
-					.redirect(
-						Redirect::builder()
-							.http_redirect_code("200")
-							.replace_key_with("static-file")
-							.build(),
-					)
-					.build(),
-			)
-			// stream other file as error
-			.routing_rules(
-				RoutingRule::builder()
-					.condition(
-						Condition::builder()
-							.set_http_error_code_returned_equals(code.clone())
-							.key_prefix_equals(format!("{prefix}/stream-404/"))
-							.build(),
-					)
-					.redirect(
-						Redirect::builder()
-							.http_redirect_code("404")
-							.replace_key_with("static-file")
-							.build(),
-					)
-					.build(),
-			)
-			// fail to stream other file
-			.routing_rules(
-				RoutingRule::builder()
-					.condition(
-						Condition::builder()
-							.set_http_error_code_returned_equals(code.clone())
-							.key_prefix_equals(format!("{prefix}/stream-missing/"))
-							.build(),
-					)
-					.redirect(
-						Redirect::builder()
-							.http_redirect_code("200")
-							.replace_key_with("missing-file")
-							.build(),
-					)
-					.build(),
-			);
-	}
-	let conf = conf.build();
-
-	ctx.client
-		.put_bucket_website()
-		.bucket(&bucket)
-		.website_configuration(conf.clone())
-		.send()
-		.await
-		.unwrap();
-
-	let stored_cfg = ctx
-		.client
-		.get_bucket_website()
-		.bucket(&bucket)
-		.send()
-		.await
-		.unwrap();
-	assert_eq!(stored_cfg.index_document, conf.index_document);
-	assert_eq!(stored_cfg.error_document, conf.error_document);
-	assert_eq!(stored_cfg.routing_rules, conf.routing_rules);
-
-	let req = |path| {
-		Request::builder()
-			.method("GET")
-			.uri(format!(
-				"http://127.0.0.1:{}/{}/path",
-				ctx.garage.web_port, path
-			))
-			.header("Host", format!("{}.web.garage", BCKT_NAME))
-			.body(Body::new(Bytes::new()))
-			.unwrap()
-	};
-
-	test_redirect_helper("unconditional", true, &req).await;
-	test_redirect_helper("conditional", true, &req).await;
-	for prefix in ["unconditional", "conditional"] {
-		for rule_path in [
-			"redirect-prefix",
-			"redirect-prefix-307",
-			"redirect-fixed",
-			"stream-fixed",
-			"stream-404",
-			"stream-missing",
-		] {
-			ctx.client
-				.put_object()
-				.bucket(&bucket)
-				.key(format!("{prefix}/{rule_path}/path"))
-				.body(ByteStream::from_static(b"i exist"))
-				.send()
-				.await
-				.unwrap();
-		}
-	}
-	test_redirect_helper("unconditional", true, &req).await;
-	test_redirect_helper("conditional", false, &req).await;
-}
-
-async fn test_redirect_helper(
-	prefix: &str,
-	should_see_redirect: bool,
-	req: impl Fn(String) -> Request<http_body_util::Full<Bytes>>,
-) {
-	use http::header;
-	let client = Client::builder(TokioExecutor::new()).build_http();
-	let expected_body = b"i exist".as_ref();
-
-	let resp = client
-		.request(req(format!("{prefix}/redirect-prefix")))
-		.await
-		.unwrap();
-	if should_see_redirect {
-		assert_eq!(resp.status(), StatusCode::FOUND);
-		assert_eq!(
-			resp.headers()
-				.get(header::LOCATION)
-				.unwrap()
-				.to_str()
-				.unwrap(),
-			"/other-prefix/path"
-		);
-		assert!(resp
-			.into_body()
-			.collect()
-			.await
-			.unwrap()
-			.to_bytes()
-			.is_empty());
-	} else {
-		assert_eq!(resp.status(), StatusCode::OK);
-		assert!(resp.headers().get(header::LOCATION).is_none());
-		assert_eq!(
-			resp.into_body().collect().await.unwrap().to_bytes(),
-			expected_body,
-		);
-	}
-
-	let resp = client
-		.request(req(format!("{prefix}/redirect-prefix-307")))
-		.await
-		.unwrap();
-	if should_see_redirect {
-		assert_eq!(resp.status(), StatusCode::TEMPORARY_REDIRECT);
-		assert_eq!(
-			resp.headers()
-				.get(header::LOCATION)
-				.unwrap()
-				.to_str()
-				.unwrap(),
-			"/other-prefix/path"
-		);
-		assert!(resp
-			.into_body()
-			.collect()
-			.await
-			.unwrap()
-			.to_bytes()
-			.is_empty());
-	} else {
-		assert_eq!(resp.status(), StatusCode::OK);
-		assert!(resp.headers().get(header::LOCATION).is_none());
-		assert_eq!(
-			resp.into_body().collect().await.unwrap().to_bytes(),
-			expected_body,
-		);
-	}
-
-	let resp = client
-		.request(req(format!("{prefix}/redirect-fixed")))
-		.await
-		.unwrap();
-	if should_see_redirect {
-		assert_eq!(resp.status(), StatusCode::FOUND);
-		assert_eq!(
-			resp.headers()
-				.get(header::LOCATION)
-				.unwrap()
-				.to_str()
-				.unwrap(),
-			"/fixed_key"
-		);
-		assert!(resp
-			.into_body()
-			.collect()
-			.await
-			.unwrap()
-			.to_bytes()
-			.is_empty());
-	} else {
-		assert_eq!(resp.status(), StatusCode::OK);
-		assert!(resp.headers().get(header::LOCATION).is_none());
-		assert_eq!(
-			resp.into_body().collect().await.unwrap().to_bytes(),
-			expected_body,
-		);
-	}
-	let resp = client
-		.request(req(format!("{prefix}/stream-fixed")))
-		.await
-		.unwrap();
-	if should_see_redirect {
-		assert_eq!(resp.status(), StatusCode::OK);
-		assert!(resp.headers().get(header::LOCATION).is_none());
-		assert_eq!(
-			resp.into_body().collect().await.unwrap().to_bytes(),
-			b"static file".as_ref(),
-		);
-	} else {
-		assert_eq!(resp.status(), StatusCode::OK);
-		assert!(resp.headers().get(header::LOCATION).is_none());
-		assert_eq!(
-			resp.into_body().collect().await.unwrap().to_bytes(),
-			expected_body,
-		);
-	}
-	let resp = client
-		.request(req(format!("{prefix}/stream-404")))
-		.await
-		.unwrap();
-	if should_see_redirect {
-		assert_eq!(resp.status(), StatusCode::NOT_FOUND);
-		assert!(resp.headers().get(header::LOCATION).is_none());
-		assert_eq!(
-			resp.into_body().collect().await.unwrap().to_bytes(),
-			b"static file".as_ref(),
-		);
-	} else {
-		assert_eq!(resp.status(), StatusCode::OK);
-		assert!(resp.headers().get(header::LOCATION).is_none());
-		assert_eq!(
-			resp.into_body().collect().await.unwrap().to_bytes(),
-			expected_body,
-		);
-	}
-	let resp = client
-		.request(req(format!("{prefix}/stream-404")))
-		.await
-		.unwrap();
-	if should_see_redirect {
-		assert_eq!(resp.status(), StatusCode::NOT_FOUND);
-		assert!(resp.headers().get(header::LOCATION).is_none());
-		assert_eq!(
-			resp.into_body().collect().await.unwrap().to_bytes(),
-			b"static file".as_ref(),
-		);
-	} else {
-		assert_eq!(resp.status(), StatusCode::OK);
-		assert!(resp.headers().get(header::LOCATION).is_none());
-		assert_eq!(
-			resp.into_body().collect().await.unwrap().to_bytes(),
-			expected_body,
-		);
-	}
-}
-
-#[tokio::test]
-async fn test_website_invalid_redirect() {
-	const BCKT_NAME: &str = "my-invalid-redirect";
-	let ctx = common::context();
-	let bucket = ctx.create_bucket(BCKT_NAME);
-
-	let conf = WebsiteConfiguration::builder()
-		.routing_rules(
-			RoutingRule::builder()
-				.condition(Condition::builder().key_prefix_equals("").build())
-				.redirect(
-					Redirect::builder()
-						.protocol(Protocol::Https)
-						.host_name("other.tld")
-						.replace_key_prefix_with("")
-						// we don't allow 200 with hostname
-						.http_redirect_code("200")
-						.build(),
-				)
-				.build(),
-		)
-		.build();
-
-	ctx.client
-		.put_bucket_website()
-		.bucket(&bucket)
-		.website_configuration(conf)
-		.send()
-		.await
-		.unwrap_err();
-}

src/model/bucket_table.rs

@@ -119,122 +119,7 @@ mod v08 {
 	impl garage_util::migrate::InitialFormat for Bucket {}
 }
 
-mod v2 {
-	use crate::permission::BucketKeyPerm;
-	use garage_util::crdt;
-	use garage_util::data::Uuid;
-	use serde::{Deserialize, Serialize};
-
-	use super::v08;
-
-	pub use v08::{BucketQuotas, CorsRule, LifecycleExpiration, LifecycleFilter, LifecycleRule};
-
-	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-	pub struct Bucket {
-		/// ID of the bucket
-		pub id: Uuid,
-		/// State, and configuration if not deleted, of the bucket
-		pub state: crdt::Deletable<BucketParams>,
-	}
-
-	/// Configuration for a bucket
-	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-	pub struct BucketParams {
-		/// Bucket's creation date
-		pub creation_date: u64,
-		/// Map of key with access to the bucket, and what kind of access they give
-		pub authorized_keys: crdt::Map<String, BucketKeyPerm>,
-
-		/// Map of aliases that are or have been given to this bucket
-		/// in the global namespace
-		/// (not authoritative: this is just used as an indication to
-		/// map back to aliases when doing ListBuckets)
-		pub aliases: crdt::LwwMap<String, bool>,
-		/// Map of aliases that are or have been given to this bucket
-		/// in namespaces local to keys
-		/// key = (access key id, alias name)
-		pub local_aliases: crdt::LwwMap<(String, String), bool>,
-
-		/// Whether this bucket is allowed for website access
-		/// (under all of its global alias names),
-		/// and if so, the website configuration XML document
-		pub website_config: crdt::Lww<Option<WebsiteConfig>>,
-		/// CORS rules
-		pub cors_config: crdt::Lww<Option<Vec<CorsRule>>>,
-		/// Lifecycle configuration
-		pub lifecycle_config: crdt::Lww<Option<Vec<LifecycleRule>>>,
-		/// Bucket quotas
-		pub quotas: crdt::Lww<BucketQuotas>,
-	}
-
-	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-	pub struct WebsiteConfig {
-		pub index_document: String,
-		pub error_document: Option<String>,
-		// this field is currently unused, but present so adding it in the future doesn't
-		// need a new migration
-		pub redirect_all: Option<RedirectAll>,
-		pub routing_rules: Vec<RoutingRule>,
-	}
-
-	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-	pub struct RedirectAll {
-		pub hostname: String,
-		pub protocol: String,
-	}
-
-	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-	pub struct RoutingRule {
-		pub condition: Option<RedirectCondition>,
-		pub redirect: Redirect,
-	}
-
-	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-	pub struct RedirectCondition {
-		pub http_error_code: Option<u16>,
-		pub prefix: Option<String>,
-	}
-
-	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-	pub struct Redirect {
-		pub hostname: Option<String>,
-		pub http_redirect_code: u16,
-		pub protocol: Option<String>,
-		pub replace_key_prefix: Option<String>,
-		pub replace_key: Option<String>,
-	}
-
-	impl garage_util::migrate::Migrate for Bucket {
-		const VERSION_MARKER: &'static [u8] = b"G2bkt";
-
-		type Previous = v08::Bucket;
-
-		fn migrate(old: v08::Bucket) -> Bucket {
-			Bucket {
-				id: old.id,
-				state: old.state.map(|x| BucketParams {
-					creation_date: x.creation_date,
-					authorized_keys: x.authorized_keys,
-					aliases: x.aliases,
-					local_aliases: x.local_aliases,
-					website_config: x.website_config.map(|wc_opt| {
-						wc_opt.map(|wc| WebsiteConfig {
-							index_document: wc.index_document,
-							error_document: wc.error_document,
-							redirect_all: None,
-							routing_rules: vec![],
-						})
-					}),
-					cors_config: x.cors_config,
-					lifecycle_config: x.lifecycle_config,
-					quotas: x.quotas,
-				}),
-			}
-		}
-	}
-}
-
-pub use v2::*;
+pub use v08::*;
 
 impl AutoCrdt for BucketQuotas {
 	const WARN_IF_DIFFERENT: bool = true;

src/model/garage.rs

@@ -175,7 +175,13 @@ impl Garage {
 
 		// ---- admin tables ----
 		info!("Initialize bucket_table...");
-		let bucket_table = Table::new(BucketTable, control_rep_param.clone(), system.clone(), &db);
+		let bucket_table = Table::new(
+			BucketTable,
+			control_rep_param.clone(),
+			system.clone(),
+			&db,
+			&config.experimental.merkle_backpressure,
+		);
 
 		info!("Initialize bucket_alias_table...");
 		let bucket_alias_table = Table::new(
@@ -183,9 +189,16 @@ impl Garage {
 			control_rep_param.clone(),
 			system.clone(),
 			&db,
+			&config.experimental.merkle_backpressure,
 		);
 		info!("Initialize key_table_table...");
-		let key_table = Table::new(KeyTable, control_rep_param, system.clone(), &db);
+		let key_table = Table::new(
+			KeyTable,
+			control_rep_param,
+			system.clone(),
+			&db,
+			&config.experimental.merkle_backpressure,
+		);
 
 		// ---- S3 tables ----
 		info!("Initialize block_ref_table...");
@@ -196,6 +209,7 @@ impl Garage {
 			meta_rep_param.clone(),
 			system.clone(),
 			&db,
+			&config.experimental.merkle_backpressure,
 		);
 
 		info!("Initialize version_table...");
@@ -206,10 +220,12 @@ impl Garage {
 			meta_rep_param.clone(),
 			system.clone(),
 			&db,
+			&config.experimental.merkle_backpressure,
 		);
 
 		info!("Initialize multipart upload counter table...");
-		let mpu_counter_table = IndexCounter::new(system.clone(), meta_rep_param.clone(), &db);
+		let mpu_counter_table =
+			IndexCounter::new(system.clone(), meta_rep_param.clone(), &db, &config);
 
 		info!("Initialize multipart upload table...");
 		let mpu_table = Table::new(
@@ -220,10 +236,12 @@ impl Garage {
 			meta_rep_param.clone(),
 			system.clone(),
 			&db,
+			&config.experimental.merkle_backpressure,
 		);
 
 		info!("Initialize object counter table...");
-		let object_counter_table = IndexCounter::new(system.clone(), meta_rep_param.clone(), &db);
+		let object_counter_table =
+			IndexCounter::new(system.clone(), meta_rep_param.clone(), &db, &config);
 
 		info!("Initialize object_table...");
 		#[allow(clippy::redundant_clone)]
@@ -236,6 +254,7 @@ impl Garage {
 			meta_rep_param.clone(),
 			system.clone(),
 			&db,
+			&config.experimental.merkle_backpressure,
 		);
 
 		info!("Load lifecycle worker state...");
@@ -245,7 +264,7 @@ impl Garage {
 
 		// ---- K2V ----
 		#[cfg(feature = "k2v")]
-		let k2v = GarageK2V::new(system.clone(), &db, meta_rep_param);
+		let k2v = GarageK2V::new(system.clone(), &db, meta_rep_param, &config);
 
 		// ---- setup block refcount recalculation ----
 		// this function can be used to fix inconsistencies in the RC table
@@ -335,9 +354,14 @@ impl Garage {
 
 #[cfg(feature = "k2v")]
 impl GarageK2V {
-	fn new(system: Arc<System>, db: &db::Db, meta_rep_param: TableShardedReplication) -> Self {
+	fn new(
+		system: Arc<System>,
+		db: &db::Db,
+		meta_rep_param: TableShardedReplication,
+		config: &Config,
+	) -> Self {
 		info!("Initialize K2V counter table...");
-		let counter_table = IndexCounter::new(system.clone(), meta_rep_param.clone(), db);
+		let counter_table = IndexCounter::new(system.clone(), meta_rep_param.clone(), db, config);
 
 		info!("Initialize K2V subscription manager...");
 		let subscriptions = Arc::new(SubscriptionManager::new());
@@ -351,6 +375,7 @@ impl GarageK2V {
 			meta_rep_param,
 			system.clone(),
 			db,
+			&config.experimental.merkle_backpressure,
 		);
 
 		info!("Initialize K2V RPC handler...");

src/model/index_counter.rs

@@ -10,6 +10,7 @@ use garage_db as db;
 use garage_rpc::layout::LayoutHelper;
 use garage_rpc::system::System;
 use garage_util::background::BackgroundRunner;
+use garage_util::config::Config;
 use garage_util::data::*;
 use garage_util::error::*;
 use garage_util::migrate::Migrate;
@@ -173,6 +174,7 @@ impl<T: CountedItem> IndexCounter<T> {
 		system: Arc<System>,
 		replication: TableShardedReplication,
 		db: &db::Db,
+		config: &Config,
 	) -> Arc<Self> {
 		Arc::new(Self {
 			this_node: system.id,
@@ -186,6 +188,7 @@ impl<T: CountedItem> IndexCounter<T> {
 				replication,
 				system,
 				db,
+				&config.experimental.merkle_backpressure,
 			),
 		})
 	}

src/net/Cargo.toml

@@ -39,6 +39,7 @@ kuska-handshake.workspace = true
 
 opentelemetry = { workspace = true, optional = true }
 opentelemetry-contrib = { workspace = true, optional = true }
+tracing.workspace = true
 
 [dev-dependencies]
 pretty_env_logger.workspace = true

src/net/client.rs

@@ -4,6 +4,7 @@ use std::pin::Pin;
 use std::sync::atomic::{self, AtomicU32};
 use std::sync::{Arc, Mutex};
 use std::task::Poll;
+use tracing::*;
 
 use arc_swap::ArcSwapOption;
 use bytes::Bytes;
@@ -14,7 +15,7 @@ use futures::Stream;
 use kuska_handshake::async_std::{handshake_client, BoxStream};
 use tokio::net::TcpStream;
 use tokio::select;
-use tokio::sync::{mpsc, oneshot, watch};
+use tokio::sync::{mpsc, oneshot, watch, Semaphore};
 use tokio_util::compat::*;
 
 #[cfg(feature = "telemetry")]
@@ -25,6 +26,7 @@ use opentelemetry::{
 #[cfg(feature = "telemetry")]
 use opentelemetry_contrib::trace::propagator::binary::*;
 
+use crate::endpoint::RpcInFlightLimiter;
 use crate::error::*;
 use crate::message::*;
 use crate::netapp::*;
@@ -41,6 +43,7 @@ pub(crate) struct ClientConn {
 
 	next_query_number: AtomicU32,
 	inflight: Mutex<HashMap<RequestID, oneshot::Sender<ByteStream>>>,
+	rpc_table_write_inflight_limiter: Option<Semaphore>,
 }
 
 impl ClientConn {
@@ -98,8 +101,14 @@ impl ClientConn {
 			next_query_number: AtomicU32::from(RequestID::default()),
 			query_send: ArcSwapOption::new(Some(Arc::new(query_send))),
 			inflight: Mutex::new(HashMap::new()),
+			rpc_table_write_inflight_limiter: netapp.max_in_flight_table_write.map(Semaphore::new),
 		});
 
+		info!(
+			"Created conn with table write limit set to {}",
+			netapp.max_in_flight_table_write.unwrap_or(0)
+		);
+
 		netapp.connected_as_client(peer_id, conn.clone());
 
 		let debug_name = format!("CLI {}", hex::encode(&peer_id[..8]));
@@ -144,10 +153,21 @@ impl ClientConn {
 		req: Req<T>,
 		path: &str,
 		prio: RequestPriority,
+		limiter: RpcInFlightLimiter,
 	) -> Result<Resp<T>, Error>
 	where
 		T: Message,
 	{
+		let _permit = match (limiter, &self.rpc_table_write_inflight_limiter) {
+			(RpcInFlightLimiter::TableWrite, Some(sem)) => {
+				info!(
+					"Available RPC table write slots: {}",
+					sem.available_permits()
+				);
+				Some(sem.acquire().await.unwrap())
+			}
+			_ => None,
+		};
 		let query_send = self.query_send.load_full().ok_or(Error::ConnectionClosed)?;
 
 		let id = self
@@ -212,6 +232,7 @@ impl ClientConn {
 		let stream = Box::pin(canceller.for_stream(stream));
 
 		let resp_enc = RespEnc::decode(stream).await?;
+		drop(_permit);
 		debug!("client: got response to request {} (path {})", id, path);
 		Resp::from_enc(resp_enc)
 	}

src/net/endpoint.rs

@@ -57,6 +57,13 @@ where
 	}
 }
 
+#[derive(Debug, Copy, Clone, Default)]
+pub enum RpcInFlightLimiter {
+	#[default]
+	NoLimit,
+	TableWrite,
+}
+
 // ----
 
 /// This struct represents an endpoint for message of type `M`.
@@ -114,6 +121,7 @@ where
 		target: &NodeID,
 		req: T,
 		prio: RequestPriority,
+		limiter: RpcInFlightLimiter,
 	) -> Result<Resp<M>, Error>
 	where
 		T: IntoReq<M>,
@@ -136,7 +144,10 @@ where
 					"Not connected: {}",
 					hex::encode(&target[..8])
 				))),
-				Some(c) => c.call(req.into_req()?, self.path.as_str(), prio).await,
+				Some(c) => {
+					c.call(req.into_req()?, self.path.as_str(), prio, limiter)
+						.await
+				}
 			}
 		}
 	}
@@ -149,8 +160,12 @@ where
 		target: &NodeID,
 		req: M,
 		prio: RequestPriority,
+		limiter: RpcInFlightLimiter,
 	) -> Result<<M as Message>::Response, Error> {
-		Ok(self.call_streaming(target, req, prio).await?.into_msg())
+		Ok(self
+			.call_streaming(target, req, prio, limiter)
+			.await?
+			.into_msg())
 	}
 }
 

src/net/netapp.rs

@@ -74,6 +74,8 @@ pub struct NetApp {
 	pub id: NodeID,
 	/// Private key associated with our peer ID
 	pub privkey: ed25519::SecretKey,
+	/// Config related to netapp
+	pub(crate) max_in_flight_table_write: Option<usize>,
 
 	pub(crate) server_conns: RwLock<HashMap<NodeID, Arc<ServerConn>>>,
 	pub(crate) client_conns: RwLock<HashMap<NodeID, Arc<ClientConn>>>,
@@ -101,6 +103,7 @@ impl NetApp {
 		netid: auth::Key,
 		privkey: ed25519::SecretKey,
 		bind_outgoing_to: Option<IpAddr>,
+		max_in_flight_table_write: Option<usize>,
 	) -> Arc<Self> {
 		let mut version_tag = [0u8; 16];
 		version_tag[0..8].copy_from_slice(&u64::to_be_bytes(NETAPP_VERSION_TAG)[..]);
@@ -114,6 +117,7 @@ impl NetApp {
 			netid,
 			id,
 			privkey,
+			max_in_flight_table_write,
 			server_conns: RwLock::new(HashMap::new()),
 			client_conns: RwLock::new(HashMap::new()),
 			endpoints: RwLock::new(HashMap::new()),
@@ -427,6 +431,7 @@ impl NetApp {
 							server_port,
 						},
 						PRIO_NORMAL,
+						RpcInFlightLimiter::NoLimit,
 					)
 					.await
 					.map(|_| ())

src/net/peering.rs

@@ -406,7 +406,7 @@ impl PeeringManager {
 			ping_time
 		);
 		let ping_response = select! {
-			r = self.ping_endpoint.call(&id, ping_msg, PRIO_HIGH) => r,
+			r = self.ping_endpoint.call(&id, ping_msg, PRIO_HIGH, RpcInFlightLimiter::NoLimit) => r,
 			_ = tokio::time::sleep(ping_timeout) => Err(Error::Message("Ping timeout".into())),
 		};
 
@@ -458,7 +458,12 @@ impl PeeringManager {
 		let pex_message = PeerListMessage { list: peer_list };
 		match self
 			.peer_list_endpoint
-			.call(id, pex_message, PRIO_BACKGROUND)
+			.call(
+				id,
+				pex_message,
+				PRIO_BACKGROUND,
+				RpcInFlightLimiter::NoLimit,
+			)
 			.await
 		{
 			Err(e) => warn!("Error doing peer exchange: {}", e),

src/rpc/rpc_helper.rs

@@ -6,6 +6,7 @@ use std::time::Duration;
 use futures::future::join_all;
 use futures::stream::futures_unordered::FuturesUnordered;
 use futures::stream::StreamExt;
+use garage_net::endpoint::RpcInFlightLimiter;
 use tokio::select;
 
 use opentelemetry::KeyValue;
@@ -44,6 +45,8 @@ pub struct RequestStrategy<T> {
 	rs_timeout: Timeout,
 	/// Data to drop when everything completes
 	rs_drop_on_complete: T,
+	/// RPC In Flight Limiter
+	rs_inflight_limiter: RpcInFlightLimiter,
 }
 
 #[derive(Copy, Clone)]
@@ -61,6 +64,7 @@ impl Clone for RequestStrategy<()> {
 			rs_priority: self.rs_priority,
 			rs_timeout: self.rs_timeout,
 			rs_drop_on_complete: (),
+			rs_inflight_limiter: self.rs_inflight_limiter,
 		}
 	}
 }
@@ -74,6 +78,7 @@ impl RequestStrategy<()> {
 			rs_priority: prio,
 			rs_timeout: Timeout::Default,
 			rs_drop_on_complete: (),
+			rs_inflight_limiter: RpcInFlightLimiter::NoLimit,
 		}
 	}
 	/// Add an item to be dropped on completion
@@ -84,6 +89,7 @@ impl RequestStrategy<()> {
 			rs_priority: self.rs_priority,
 			rs_timeout: self.rs_timeout,
 			rs_drop_on_complete: drop_on_complete,
+			rs_inflight_limiter: RpcInFlightLimiter::NoLimit,
 		}
 	}
 }
@@ -109,6 +115,10 @@ impl<T> RequestStrategy<T> {
 		self.rs_timeout = Timeout::Custom(timeout);
 		self
 	}
+	pub fn with_write_limiter(mut self) -> Self {
+		self.rs_inflight_limiter = RpcInFlightLimiter::TableWrite;
+		self
+	}
 	/// Extract drop_on_complete item
 	fn extract_drop_on_complete(self) -> (RequestStrategy<()>, T) {
 		(
@@ -118,6 +128,7 @@ impl<T> RequestStrategy<T> {
 				rs_priority: self.rs_priority,
 				rs_timeout: self.rs_timeout,
 				rs_drop_on_complete: (),
+				rs_inflight_limiter: self.rs_inflight_limiter,
 			},
 			self.rs_drop_on_complete,
 		)
@@ -185,7 +196,7 @@ impl RpcHelper {
 
 		let node_id = to.into();
 		let rpc_call = endpoint
-			.call_streaming(&node_id, msg, strat.rs_priority)
+			.call_streaming(&node_id, msg, strat.rs_priority, strat.rs_inflight_limiter)
 			.with_context(Context::current_with_span(span))
 			.record_duration(&self.0.metrics.rpc_duration, &metric_tags);
 

src/rpc/system.rs

@@ -21,7 +21,7 @@ use garage_net::{NetApp, NetworkKey, NodeID, NodeKey};
 
 #[cfg(feature = "kubernetes-discovery")]
 use garage_util::config::KubernetesDiscoveryConfig;
-use garage_util::config::{Config, DataDirEnum};
+use garage_util::config::{Config, DataDirEnum, RpcInFlightLimiterEnum};
 use garage_util::data::*;
 use garage_util::error::*;
 use garage_util::persister::Persister;
@@ -256,7 +256,17 @@ impl System {
 		let bind_outgoing_to = Some(config)
 			.filter(|x| x.rpc_bind_outgoing)
 			.map(|x| x.rpc_bind_addr.ip());
-		let netapp = NetApp::new(GARAGE_VERSION_TAG, network_key, node_key, bind_outgoing_to);
+		let maybe_max_table_write = match &config.experimental.rpc_in_flight_limiters {
+			RpcInFlightLimiterEnum::None => None,
+			RpcInFlightLimiterEnum::FixedSize(v) => Some(v.max_table_write),
+		};
+		let netapp = NetApp::new(
+			GARAGE_VERSION_TAG,
+			network_key,
+			node_key,
+			bind_outgoing_to,
+			maybe_max_table_write,
+		);
 		let system_endpoint = netapp.endpoint(SYSTEM_RPC_PATH.into());
 
 		// ---- setup netapp public listener and full mesh peering strategy ----

src/table/data.rs

@@ -3,10 +3,12 @@ use std::convert::TryInto;
 use std::sync::Arc;
 
 use serde_bytes::ByteBuf;
-use tokio::sync::Notify;
+use tokio::sync::SemaphorePermit;
+use tokio::sync::{Notify, Semaphore};
 
 use garage_db as db;
 
+use garage_util::config::MerkleBackpressureEnum;
 use garage_util::data::*;
 use garage_util::error::*;
 use garage_util::migrate::Migrate;
@@ -20,6 +22,67 @@ use crate::replication::*;
 use crate::schema::*;
 use crate::util::*;
 
+pub(crate) struct MerkleTodo {
+	merkle_todo: db::Tree,
+	merkle_todo_notify: Notify,
+	merkle_todo_bounded_queue: Option<Arc<Semaphore>>,
+}
+impl Clone for MerkleTodo {
+	fn clone(&self) -> Self {
+		Self {
+			merkle_todo: self.merkle_todo.clone(),
+			merkle_todo_notify: Notify::new(),
+			merkle_todo_bounded_queue: self.merkle_todo_bounded_queue.clone(),
+		}
+	}
+}
+impl MerkleTodo {
+	fn new<F: TableSchema>(db: &db::Db, config: &MerkleBackpressureEnum) -> Self {
+		let merkle_todo = db
+			.open_tree(format!("{}:merkle_todo", F::TABLE_NAME))
+			.expect("Unable to open DB Merkle TODO tree");
+
+		let merkle_todo_bounded_queue = match config {
+			MerkleBackpressureEnum::None => None,
+			MerkleBackpressureEnum::FixedQueue(p) => {
+				Some(Arc::new(Semaphore::new(p.max_queue_size)))
+			}
+		};
+
+		Self {
+			merkle_todo,
+			merkle_todo_notify: Notify::new(),
+			merkle_todo_bounded_queue,
+		}
+	}
+
+	pub(crate) fn len(&self) -> Result<usize, db::Error> {
+		self.merkle_todo.len()
+	}
+
+	pub(crate) async fn with_db<F: FnOnce(&db::Tree, SemaphorePermit)>(&self, f: F) {
+		let bounded = self
+			.merkle_todo_bounded_queue
+			.clone()
+			.unwrap_or(Arc::new(Semaphore::new(1)));
+		let permit = bounded.acquire().await.unwrap();
+		f(&self.merkle_todo, permit);
+	}
+
+	pub(crate) fn appended(&self, permit: SemaphorePermit) {
+		permit.forget();
+		self.merkle_todo_notify.notify_one();
+	}
+
+	pub(crate) fn processed(&self) {
+		let bounded = self
+			.merkle_todo_bounded_queue
+			.clone()
+			.unwrap_or(Arc::new(Semaphore::new(1)));
+		bounded.add_permits(1);
+	}
+}
+
 pub struct TableData<F: TableSchema, R: TableReplication> {
 	system: Arc<System>,
 
@@ -29,8 +92,7 @@ pub struct TableData<F: TableSchema, R: TableReplication> {
 	pub store: db::Tree,
 
 	pub(crate) merkle_tree: db::Tree,
-	pub(crate) merkle_todo: db::Tree,
-	pub(crate) merkle_todo_notify: Notify,
+	pub(crate) merkle_todo: MerkleTodo,
 
 	pub(crate) insert_queue: db::Tree,
 	pub(crate) insert_queue_notify: Arc<Notify>,
@@ -38,10 +100,18 @@ pub struct TableData<F: TableSchema, R: TableReplication> {
 	pub(crate) gc_todo: db::Tree,
 
 	pub(crate) metrics: TableMetrics,
+
+	pub(crate) config: MerkleBackpressureEnum,
 }
 
 impl<F: TableSchema, R: TableReplication> TableData<F, R> {
-	pub fn new(system: Arc<System>, instance: F, replication: R, db: &db::Db) -> Arc<Self> {
+	pub fn new(
+		system: Arc<System>,
+		instance: F,
+		replication: R,
+		db: &db::Db,
+		config: &MerkleBackpressureEnum,
+	) -> Arc<Self> {
 		let store = db
 			.open_tree(format!("{}:table", F::TABLE_NAME))
 			.expect("Unable to open DB tree");
@@ -49,9 +119,8 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
 		let merkle_tree = db
 			.open_tree(format!("{}:merkle_tree", F::TABLE_NAME))
 			.expect("Unable to open DB Merkle tree tree");
-		let merkle_todo = db
-			.open_tree(format!("{}:merkle_todo", F::TABLE_NAME))
-			.expect("Unable to open DB Merkle TODO tree");
+
+		let merkle_todo = MerkleTodo::new::<F>(db, config);
 
 		let insert_queue = db
 			.open_tree(format!("{}:insert_queue", F::TABLE_NAME))
@@ -76,11 +145,11 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
 			store,
 			merkle_tree,
 			merkle_todo,
-			merkle_todo_notify: Notify::new(),
 			insert_queue,
 			insert_queue_notify: Arc::new(Notify::new()),
 			gc_todo,
 			metrics,
+			config: config.clone(),
 		})
 	}
 
@@ -167,6 +236,8 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
 	// - When an entry is modified or deleted, add it to the merkle updater's todo list.
 	//   This has to be done atomically with the modification for the merkle updater
 	//   to maintain consistency. The merkle updater must then be notified with todo_notify.
+	//   Also to avoid overloading the merkle updater, you need to sleep a given amount of
+	//   time to enable backpressure (ie. slow down clients).
 	// - When an entry is updated to be a tombstone, add it to the gc_todo tree
 
 	pub(crate) fn update_many<T: Borrow<ByteBuf>>(&self, entries: &[T]) -> Result<(), Error> {
@@ -201,6 +272,7 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
 	) -> Result<Option<F::E>, Error> {
 		let tree_key = self.tree_key(partition_key, sort_key);
 
+		// transaction begins
 		let changed = self.store.db().transaction(|tx| {
 			let (old_entry, old_bytes, new_entry) = match tx.get(&self.store, &tree_key)? {
 				Some(old_bytes) => {
@@ -238,31 +310,44 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
 				Ok(None)
 			}
 		})?;
+		// transaction ends
 
-		if let Some((new_entry, new_bytes_hash)) = changed {
-			self.metrics.internal_update_counter.add(1);
-
-			let is_tombstone = new_entry.is_tombstone();
-			self.merkle_todo_notify.notify_one();
-			if is_tombstone {
-				// We are only responsible for GC'ing this item if we are the
-				// "leader" of the partition, i.e. the first node in the
-				// set of nodes that replicates this partition.
-				// This avoids GC loops and does not change the termination properties
-				// of the GC algorithm, as in all cases GC is suspended if
-				// any node of the partition is unavailable.
-				let pk_hash = Hash::try_from(&tree_key[..32]).unwrap();
-				// TODO: this probably breaks when the layout changes
-				let nodes = self.replication.storage_nodes(&pk_hash);
-				if nodes.first() == Some(&self.system.id) {
-					GcTodoEntry::new(tree_key, new_bytes_hash).save(&self.gc_todo)?;
+		// early return if nothing changed
+		let (new_entry, new_bytes_hash) = match changed {
+			Some((e, b)) => (e, b),
+			None => {
+				let maybe_bound = self.merkle_todo_bounded_queue.clone();
+				if let Some(b) = &maybe_bound {
+					b.add_permits(1);
 				}
+				return Ok(None);
 			}
+		};
 
-			Ok(Some(new_entry))
-		} else {
-			Ok(None)
+		// Handle GC in case of tombstone
+		let is_tombstone = new_entry.is_tombstone();
+		if is_tombstone {
+			// We are only responsible for GC'ing this item if we are the
+			// "leader" of the partition, i.e. the first node in the
+			// set of nodes that replicates this partition.
+			// This avoids GC loops and does not change the termination properties
+			// of the GC algorithm, as in all cases GC is suspended if
+			// any node of the partition is unavailable.
+			let pk_hash = Hash::try_from(&tree_key[..32]).unwrap();
+			// TODO: this probably breaks when the layout changes
+			let nodes = self.replication.storage_nodes(&pk_hash);
+			if nodes.first() == Some(&self.system.id) {
+				GcTodoEntry::new(tree_key, new_bytes_hash).save(&self.gc_todo)?;
+			}
 		}
+
+		// Collect metrics
+		self.metrics.internal_update_counter.add(1);
+
+		// Synchronize with the Merkle Worker
+		self.merkle_todo_notify.notify_one(); // Wake-up it
+
+		Ok(Some(new_entry))
 	}
 
 	pub(crate) fn delete_if_equal(self: &Arc<Self>, k: &[u8], v: &[u8]) -> Result<bool, Error> {
@@ -282,10 +367,16 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
 				_ => Ok(false),
 			})?;
 
-		if removed {
-			self.metrics.internal_delete_counter.add(1);
-			self.merkle_todo_notify.notify_one();
+		if !removed {
+			let maybe_bound = self.merkle_todo_bounded_queue.clone();
+			if let Some(b) = &maybe_bound {
+				b.add_permits(1);
+			}
+			return Ok(false);
 		}
+
+		self.metrics.internal_delete_counter.add(1);
+		self.merkle_todo_notify.notify_one();
 		Ok(removed)
 	}
 
@@ -310,11 +401,18 @@ impl<F: TableSchema, R: TableReplication> TableData<F, R> {
 				_ => Ok(false),
 			})?;
 
-		if removed {
-			self.metrics.internal_delete_counter.add(1);
-			self.merkle_todo_notify.notify_one();
+		if !removed {
+			let maybe_bound = self.merkle_todo_bounded_queue.clone();
+			if let Some(b) = &maybe_bound {
+				b.add_permits(1);
+			}
+			return Ok(false);
 		}
-		Ok(removed)
+
+		self.metrics.internal_delete_counter.add(1);
+		self.merkle_todo_notify.notify_one();
+
+		Ok(true)
 	}
 
 	// ---- Insert queue functions ----

src/table/gc.rs

@@ -262,7 +262,8 @@ impl<F: TableSchema, R: TableReplication> TableGc<F, R> {
 		// GC has been successful for all of these entries.
 		// We now remove them all from our local table and from the GC todo list.
 		for item in items {
-			self.data
+			let _is_removed = self
+				.data
 				.delete_if_equal_hash(&item.key[..], item.value_hash)
 				.err_context("GC: local delete tombstones")?;
 			item.remove_if_equal(&self.data.gc_todo)
@@ -275,14 +276,21 @@ impl<F: TableSchema, R: TableReplication> TableGc<F, R> {
 
 impl<F: TableSchema, R: TableReplication> EndpointHandler<GcRpc> for TableGc<F, R> {
 	async fn handle(self: &Arc<Self>, message: &GcRpc, _from: NodeID) -> Result<GcRpc, Error> {
+		let maybe_bounded = self.data.merkle_todo_bounded_queue.clone();
 		match message {
 			GcRpc::Update(items) => {
+				if let Some(b) = maybe_bounded {
+					b.acquire_many(items.len() as u32).await.unwrap().forget();
+				}
 				self.data.update_many(items)?;
 				Ok(GcRpc::Ok)
 			}
 			GcRpc::DeleteIfEqualHash(items) => {
+				if let Some(b) = maybe_bounded {
+					b.acquire_many(items.len() as u32).await.unwrap().forget();
+				}
 				for (key, vhash) in items.iter() {
-					self.data.delete_if_equal_hash(&key[..], *vhash)?;
+					let _is_removed = self.data.delete_if_equal_hash(&key[..], *vhash)?;
 				}
 				Ok(GcRpc::Ok)
 			}
@@ -329,7 +337,6 @@ impl<F: TableSchema, R: TableReplication> Worker for GcWorker<F, R> {
 	}
 
 	async fn wait_for_work(&mut self) -> WorkerState {
-		tokio::time::sleep(self.wait_delay).await;
 		WorkerState::Busy
 	}
 }

src/table/merkle.rs

@@ -9,6 +9,7 @@ use tokio::sync::watch;
 use garage_db as db;
 
 use garage_util::background::*;
+use garage_util::config::MerkleBackpressureEnum;
 use garage_util::data::*;
 use garage_util::encode::{nonversioned_decode, nonversioned_encode};
 use garage_util::error::Error;
@@ -70,6 +71,15 @@ impl<F: TableSchema, R: TableReplication> MerkleUpdater<F, R> {
 	pub(crate) fn new(data: Arc<TableData<F, R>>) -> Arc<Self> {
 		let empty_node_hash = blake2sum(&nonversioned_encode(&MerkleNode::Empty).unwrap()[..]);
 
+		// @FIXME: move in worker
+		match &data.config {
+			MerkleBackpressureEnum::None => info!("Merkle Backpressure is not activated"),
+			MerkleBackpressureEnum::FixedQueue(v) => info!(
+				"Merkle backpressure with a fixed queue size (qlen={}) is activated.",
+				v.max_queue_size
+			),
+		}
+
 		Arc::new(Self {
 			data,
 			empty_node_hash,
@@ -125,6 +135,11 @@ impl<F: TableSchema, R: TableReplication> MerkleUpdater<F, R> {
 				k
 			);
 		}
+
+		let maybe_bound = self.data.merkle_todo_bounded_queue.clone();
+		if let Some(b) = &maybe_bound {
+			b.add_permits(1);
+		}
 		Ok(())
 	}
 

src/table/metrics.rs

@@ -1,12 +1,16 @@
 use opentelemetry::{global, metrics::*, KeyValue};
+use std::convert::TryInto;
 
 use garage_db as db;
 
+use crate::data::MerkleTodo;
+
 /// TableMetrics reference all counter used for metrics
 pub struct TableMetrics {
 	pub(crate) _table_size: ValueObserver<u64>,
 	pub(crate) _merkle_tree_size: ValueObserver<u64>,
 	pub(crate) _merkle_todo_len: ValueObserver<u64>,
+	pub(crate) _merkle_todo_bounded_queue_free: ValueObserver<u64>,
 	pub(crate) _gc_todo_len: ValueObserver<u64>,
 
 	pub(crate) get_request_counter: BoundCounter<u64>,
@@ -25,7 +29,7 @@ impl TableMetrics {
 		table_name: &'static str,
 		store: db::Tree,
 		merkle_tree: db::Tree,
-		merkle_todo: db::Tree,
+		merkle_todo: MerkleTodo,
 		gc_todo: db::Tree,
 	) -> Self {
 		let meter = global::meter(table_name);
@@ -72,6 +76,20 @@ impl TableMetrics {
 				)
 				.with_description("Merkle tree updater TODO queue length")
 				.init(),
+                        _merkle_todo_bounded_queue_free: meter
+                            .u64_value_observer(
+                                "table.merkle_todo_bounded_queue_free",
+                                move |observer| {
+                                    let maybe_bounded = merkle_todo_bounded_queue.clone();
+                                    let free: u64 = match &maybe_bounded {
+                                        Some(v) => v.available_permits().try_into().unwrap(),
+                                        None => 0,
+                                    };
+                                    observer.observe(free, &[KeyValue::new("table_name", table_name)])
+                                }
+                            )
+			    .with_description("Merkle TODO queue free slots")
+			    .init(),
 			_gc_todo_len: meter
 				.u64_value_observer(
 					"table.gc_todo_queue_length",

src/table/replication/fullcopy.rs

@@ -43,10 +43,13 @@ impl TableReplication for TableFullReplication {
 	}
 	fn write_quorum(&self) -> usize {
 		let nmembers = self.system.cluster_layout().current().all_nodes().len();
-		if nmembers < 3 {
-			1
+
+		let max_faults = if nmembers > 1 { 1 } else { 0 };
+
+		if nmembers > max_faults {
+			nmembers - max_faults
 		} else {
-			nmembers.div_euclid(2) + 1
+			1
 		}
 	}
 

src/table/sync.rs

@@ -244,8 +244,14 @@ impl<F: TableSchema, R: TableReplication> TableSyncer<F, R> {
 
 		// All remote nodes have written those items, now we can delete them locally
 		let mut not_removed = 0;
+		let maybe_bounded = self.data.merkle_todo_bounded_queue.clone();
+		if let Some(b) = maybe_bounded {
+			b.acquire_many(items.len() as u32).await.unwrap().forget();
+		}
+
 		for (k, v) in items.iter() {
-			if !self.data.delete_if_equal(&k[..], &v[..])? {
+			let removed = self.data.delete_if_equal(&k[..], &v[..])?;
+			if !removed {
 				not_removed += 1;
 			}
 		}

src/table/table.rs

@@ -14,6 +14,7 @@ use opentelemetry::{
 use garage_db as db;
 
 use garage_util::background::BackgroundRunner;
+use garage_util::config::MerkleBackpressureEnum;
 use garage_util::data::*;
 use garage_util::error::Error;
 use garage_util::metrics::RecordDuration;
@@ -68,12 +69,18 @@ impl<F: TableSchema> Rpc for TableRpc<F> {
 impl<F: TableSchema, R: TableReplication> Table<F, R> {
 	// =============== PUBLIC INTERFACE FUNCTIONS (new, insert, get, etc) ===============
 
-	pub fn new(instance: F, replication: R, system: Arc<System>, db: &db::Db) -> Arc<Self> {
+	pub fn new(
+		instance: F,
+		replication: R,
+		system: Arc<System>,
+		db: &db::Db,
+		config: &MerkleBackpressureEnum,
+	) -> Arc<Self> {
 		let endpoint = system
 			.netapp
 			.endpoint(format!("garage_table/table.rs/Rpc:{}", F::TABLE_NAME));
 
-		let data = TableData::new(system.clone(), instance, replication, db);
+		let data = TableData::new(system.clone(), instance, replication, db, config);
 
 		let merkle_updater = MerkleUpdater::new(data.clone());
 
@@ -131,7 +138,8 @@ impl<F: TableSchema, R: TableReplication> Table<F, R> {
 				who.as_ref(),
 				rpc,
 				RequestStrategy::with_priority(PRIO_NORMAL)
-					.with_quorum(self.data.replication.write_quorum()),
+					.with_quorum(self.data.replication.write_quorum())
+					.with_write_limiter(),
 			)
 			.await?;
 
@@ -527,6 +535,10 @@ impl<F: TableSchema, R: TableReplication> EndpointHandler<TableRpc<F>> for Table
 				Ok(TableRpc::Update(values))
 			}
 			TableRpc::Update(pairs) => {
+				let maybe_bounded = self.data.merkle_todo_bounded_queue.clone();
+				if let Some(b) = maybe_bounded {
+					b.acquire_many(pairs.len() as u32).await.unwrap().forget();
+				}
 				self.data.update_many(pairs)?;
 				Ok(TableRpc::Ok)
 			}

src/util/config.rs

@@ -135,6 +135,10 @@ pub struct Config {
 	/// Configuration for the admin API endpoint
 	#[serde(default = "Default::default")]
 	pub admin: AdminConfig,
+
+	/// --- Experimental
+	#[serde(default = "Default::default")]
+	pub experimental: ExperimentalConfig,
 }
 
 /// Value for data_dir: either a single directory or a list of dirs with attributes
@@ -255,6 +259,40 @@ pub struct KubernetesDiscoveryConfig {
 	pub skip_crd: bool,
 }
 
+#[derive(Deserialize, Debug, Clone, Default)]
+pub struct ExperimentalConfig {
+	pub merkle_backpressure: MerkleBackpressureEnum,
+	pub rpc_in_flight_limiters: RpcInFlightLimiterEnum,
+}
+
+#[derive(Deserialize, Debug, Clone, Default)]
+#[serde(rename_all = "lowercase", tag = "kind")]
+pub enum MerkleBackpressureEnum {
+	#[default]
+	None,
+	FixedQueue(MerkleFixedQueue),
+}
+
+#[derive(Deserialize, Debug, Clone, Default)]
+#[serde(rename_all = "lowercase", tag = "kind")]
+pub enum RpcInFlightLimiterEnum {
+	#[default]
+	None,
+	FixedSize(InFlightFixedSize),
+}
+
+#[derive(Deserialize, Debug, Clone, Default)]
+pub struct InFlightFixedSize {
+	#[serde(default = "default_max_table_write")]
+	pub max_table_write: usize,
+}
+
+#[derive(Deserialize, Debug, Clone, Default)]
+pub struct MerkleFixedQueue {
+	#[serde(default = "default_max_queue_size")]
+	pub max_queue_size: usize,
+}
+
 /// Read and parse configuration
 pub fn read_config(config_file: PathBuf) -> Result<Config, Error> {
 	let config = std::fs::read_to_string(config_file)?;
@@ -281,6 +319,14 @@ fn default_compression() -> Option<i32> {
 	Some(1)
 }
 
+fn default_max_table_write() -> usize {
+	64
+}
+
+fn default_max_queue_size() -> usize {
+	256
+}
+
 fn deserialize_compression<'de, D>(deserializer: D) -> Result<Option<i32>, D::Error>
 where
 	D: de::Deserializer<'de>,

src/util/crdt/deletable.rs

@@ -9,16 +9,6 @@ pub enum Deletable<T> {
 	Deleted,
 }
 
-impl<T> Deletable<T> {
-	/// Map value, used for migrations
-	pub fn map<U, F: FnOnce(T) -> U>(self, f: F) -> Deletable<U> {
-		match self {
-			Self::Present(x) => Deletable::<U>::Present(f(x)),
-			Self::Deleted => Deletable::<U>::Deleted,
-		}
-	}
-}
-
 impl<T: Crdt> Deletable<T> {
 	/// Create a new deletable object that isn't deleted
 	pub fn present(v: T) -> Self {

src/util/crdt/lww.rs

@@ -43,16 +43,6 @@ pub struct Lww<T> {
 	v: T,
 }
 
-impl<T> Lww<T> {
-	/// Map value, used for migrations
-	pub fn map<U, F: FnOnce(T) -> U>(self, f: F) -> Lww<U> {
-		Lww::<U> {
-			ts: self.ts,
-			v: f(self.v),
-		}
-	}
-}
-
 impl<T> Lww<T>
 where
 	T: Crdt,

src/web/web_server.rs

@@ -25,14 +25,12 @@ use garage_api_common::cors::{
 };
 use garage_api_common::generic_server::{server_loop, UnixListenerOn};
 use garage_api_common::helpers::*;
-use garage_api_s3::api_server::ResBody;
 use garage_api_s3::error::{
 	CommonErrorDerivative, Error as ApiError, OkOrBadRequest, OkOrInternalError,
 };
 use garage_api_s3::get::{handle_get_without_ctx, handle_head_without_ctx};
 use garage_api_s3::website::X_AMZ_WEBSITE_REDIRECT_LOCATION;
 
-use garage_model::bucket_table::{self, RoutingRule};
 use garage_model::garage::Garage;
 
 use garage_table::*;
@@ -262,71 +260,45 @@ impl WebServer {
 		// Get path
 		let path = req.uri().path().to_string();
 		let index = &website_config.index_document;
-		let routing_result = path_to_keys(&path, index, &website_config.routing_rules)?;
+		let (key, may_redirect) = path_to_keys(&path, index)?;
 
 		debug!(
-			"Selected bucket: \"{}\" {:?}, routing to {:?}",
-			bucket_name, bucket_id, routing_result,
+			"Selected bucket: \"{}\" {:?}, target key: \"{}\", may redirect to: {:?}",
+			bucket_name, bucket_id, key, may_redirect
 		);
 
-		let ret_doc = match (req.method(), routing_result.main_target()) {
-			(&Method::OPTIONS, _) => handle_options_for_bucket(req, &bucket_params)
+		let ret_doc = match *req.method() {
+			Method::OPTIONS => handle_options_for_bucket(req, &bucket_params)
 				.map_err(ApiError::from)
 				.map(|res| res.map(|_empty_body: EmptyBody| empty_body())),
-			(_, Err((url, code))) => Ok(Response::builder()
-				.status(code)
-				.header("Location", url)
-				.body(empty_body())
-				.unwrap()),
-			(_, Ok((key, code))) => {
-				handle_inner(self.garage.clone(), req, bucket_id, key, code).await
+			Method::HEAD => {
+				handle_head_without_ctx(self.garage.clone(), req, bucket_id, &key, None).await
 			}
-		};
-
-		// Try handling errors if bucket configuration provided fallbacks
-		let ret_doc_with_redir = match (&ret_doc, &routing_result) {
-			(
-				Err(ApiError::NoSuchKey),
-				RoutingResult::LoadOrRedirect {
-					redirect_if_exists,
-					redirect_url,
-					redirect_code,
-					..
-				},
-			) => {
-				let redirect = if let Some(redirect_key) = redirect_if_exists {
-					self.check_key_exists(bucket_id, redirect_key.as_str())
-						.await?
-				} else {
-					true
-				};
-				if redirect {
-					Ok(Response::builder()
-						.status(redirect_code)
-						.header("Location", redirect_url)
-						.body(empty_body())
-						.unwrap())
-				} else {
-					ret_doc
-				}
-			}
-			(
-				Err(ApiError::NoSuchKey),
-				RoutingResult::LoadOrAlternativeError {
-					redirect_key,
-					redirect_code,
-					..
-				},
-			) => {
-				handle_inner(
+			Method::GET => {
+				handle_get_without_ctx(
 					self.garage.clone(),
 					req,
 					bucket_id,
-					redirect_key,
-					*redirect_code,
+					&key,
+					None,
+					Default::default(),
 				)
 				.await
 			}
+			_ => Err(ApiError::bad_request("HTTP method not supported")),
+		};
+
+		// Try implicit redirect on error
+		let ret_doc_with_redir = match (&ret_doc, may_redirect) {
+			(Err(ApiError::NoSuchKey), ImplicitRedirect::To { key, url })
+				if self.check_key_exists(bucket_id, key.as_str()).await? =>
+			{
+				Ok(Response::builder()
+					.status(StatusCode::FOUND)
+					.header(LOCATION, url)
+					.body(empty_body())
+					.unwrap())
+			}
 			(Ok(ret), _) if ret.headers().contains_key(X_AMZ_WEBSITE_REDIRECT_LOCATION) => {
 				let redirect_location = ret.headers().get(X_AMZ_WEBSITE_REDIRECT_LOCATION).unwrap();
 				Ok(Response::builder()
@@ -360,17 +332,17 @@ impl WebServer {
 				// We want to return the error document
 				// Create a fake HTTP request with path = the error document
 				let req2 = Request::builder()
-					.method("GET")
 					.uri(format!("http://{}/{}", host, &error_document))
 					.body(())
 					.unwrap();
 
-				match handle_inner(
+				match handle_get_without_ctx(
 					self.garage.clone(),
 					&req2,
 					bucket_id,
 					&error_document,
-					error.http_status_code(),
+					None,
+					Default::default(),
 				)
 				.await
 				{
@@ -385,6 +357,8 @@ impl WebServer {
 							error
 						);
 
+						*error_doc.status_mut() = error.http_status_code();
+
 						// Preserve error message in a special header
 						for error_line in error.to_string().split('\n') {
 							if let Ok(v) = HeaderValue::from_bytes(error_line.as_bytes()) {
@@ -415,52 +389,6 @@ impl WebServer {
 	}
 }
 
-async fn handle_inner(
-	garage: Arc<Garage>,
-	req: &Request<()>,
-	bucket_id: Uuid,
-	key: &str,
-	status_code: StatusCode,
-) -> Result<Response<ResBody>, ApiError> {
-	if status_code != StatusCode::OK {
-		// If we are returning an error document, discard all headers from
-		// the original request that would have influenced the result:
-		// - Range header, we don't want to return a subrange of the error document
-		// - Caching directives such as If-None-Match, etc, which are not relevant
-		let cleaned_req = Request::builder().uri(req.uri()).body(()).unwrap();
-
-		let mut ret = match req.method() {
-			&Method::HEAD => {
-				handle_head_without_ctx(garage, &cleaned_req, bucket_id, key, None).await?
-			}
-			&Method::GET => {
-				handle_get_without_ctx(
-					garage,
-					&cleaned_req,
-					bucket_id,
-					key,
-					None,
-					Default::default(),
-				)
-				.await?
-			}
-			_ => return Err(ApiError::bad_request("HTTP method not supported")),
-		};
-
-		*ret.status_mut() = status_code;
-
-		Ok(ret)
-	} else {
-		match req.method() {
-			&Method::HEAD => handle_head_without_ctx(garage, req, bucket_id, key, None).await,
-			&Method::GET => {
-				handle_get_without_ctx(garage, req, bucket_id, key, None, Default::default()).await
-			}
-			_ => Err(ApiError::bad_request("HTTP method not supported")),
-		}
-	}
-}
-
 fn error_to_res(e: Error) -> Response<BoxBody<Error>> {
 	// If we are here, it is either that:
 	// - there was an error before trying to get the requested URL
@@ -477,44 +405,9 @@ fn error_to_res(e: Error) -> Response<BoxBody<Error>> {
 }
 
 #[derive(Debug, PartialEq)]
-enum RoutingResult {
-	// Load a key and use `code` as status, or fallback to normal 404 handler if not found
-	LoadKey {
-		key: String,
-		code: StatusCode,
-	},
-	// Load a key and use `200` as status, or fallback with a redirection using `redirect_code`
-	// as status
-	LoadOrRedirect {
-		key: String,
-		redirect_if_exists: Option<String>,
-		redirect_url: String,
-		redirect_code: StatusCode,
-	},
-	// Load a key and use `200` as status, or fallback by loading a different key and use
-	// `redirect_code` as status
-	LoadOrAlternativeError {
-		key: String,
-		redirect_key: String,
-		redirect_code: StatusCode,
-	},
-	// Send an http redirect with `code` as status
-	Redirect {
-		url: String,
-		code: StatusCode,
-	},
-}
-
-impl RoutingResult {
-	// return Ok((key_to_deref, status_code)) or Err((redirect_target, status_code))
-	fn main_target(&self) -> Result<(&str, StatusCode), (&str, StatusCode)> {
-		match self {
-			RoutingResult::LoadKey { key, code } => Ok((key, *code)),
-			RoutingResult::LoadOrRedirect { key, .. } => Ok((key, StatusCode::OK)),
-			RoutingResult::LoadOrAlternativeError { key, .. } => Ok((key, StatusCode::OK)),
-			RoutingResult::Redirect { url, code } => Err((url, *code)),
-		}
-	}
+enum ImplicitRedirect {
+	No,
+	To { key: String, url: String },
 }
 
 /// Path to key
@@ -524,154 +417,35 @@ impl RoutingResult {
 /// which is also AWS S3 behavior.
 ///
 /// Check: https://docs.aws.amazon.com/AmazonS3/latest/userguide/IndexDocumentSupport.html
-fn path_to_keys(
-	path: &str,
-	index: &str,
-	routing_rules: &[RoutingRule],
-) -> Result<RoutingResult, Error> {
+fn path_to_keys<'a>(path: &'a str, index: &str) -> Result<(String, ImplicitRedirect), Error> {
 	let path_utf8 = percent_encoding::percent_decode_str(path).decode_utf8()?;
 
 	let base_key = match path_utf8.strip_prefix("/") {
 		Some(bk) => bk,
 		None => return Err(Error::BadRequest("Path must start with a / (slash)".into())),
 	};
-
-	let is_bucket_root = base_key.is_empty();
+	let is_bucket_root = base_key.len() == 0;
 	let is_trailing_slash = path_utf8.ends_with("/");
 
-	let key = if is_bucket_root || is_trailing_slash {
-		// we can't store anything at the root, so we need to query the index
-		// if the key end with a slash, we always query the index
-		format!("{base_key}{index}")
-	} else {
-		// if the key doesn't end with `/`, leave it unmodified
-		base_key.to_string()
-	};
+	match (is_bucket_root, is_trailing_slash) {
+		// It is not possible to store something at the root of the bucket (ie. empty key),
+		// the only option is to fetch the index
+		(true, _) => Ok((index.to_string(), ImplicitRedirect::No)),
 
-	let mut routing_rules_iter = routing_rules.iter();
-	let key = loop {
-		let Some(routing_rule) = routing_rules_iter.next() else {
-			break key;
-		};
+		// "If you create a folder structure in your bucket, you must have an index document at each level. In each folder, the index document must have the same name, for example, index.html. When a user specifies a URL that resembles a folder lookup, the presence or absence of a trailing slash determines the behavior of the website. For example, the following URL, with a trailing slash, returns the photos/index.html index document."
+		(false, true) => Ok((format!("{base_key}{index}"), ImplicitRedirect::No)),
 
-		let Ok(status_code) = StatusCode::from_u16(routing_rule.redirect.http_redirect_code) else {
-			continue;
-		};
-		if let Some(condition) = &routing_rule.condition {
-			let suffix = if let Some(prefix) = &condition.prefix {
-				let Some(suffix) = key.strip_prefix(prefix) else {
-					continue;
-				};
-				Some(suffix)
-			} else {
-				None
-			};
-			let mut target = compute_redirect_target(&routing_rule.redirect, suffix);
-			let query_alternative_key =
-				status_code == StatusCode::OK || status_code == StatusCode::NOT_FOUND;
-			let redirect_on_error =
-				condition.http_error_code == Some(StatusCode::NOT_FOUND.as_u16());
-			match (query_alternative_key, redirect_on_error) {
-				(false, false) => {
-					return Ok(RoutingResult::Redirect {
-						url: target,
-						code: status_code,
-					})
-				}
-				(true, false) => {
-					// we need to remove the leading /
-					target.remove(0);
-					if status_code == StatusCode::OK {
-						break target;
-					} else {
-						return Ok(RoutingResult::LoadKey {
-							key: target,
-							code: status_code,
-						});
-					}
-				}
-				(false, true) => {
-					return Ok(RoutingResult::LoadOrRedirect {
-						key,
-						redirect_if_exists: None,
-						redirect_url: target,
-						redirect_code: status_code,
-					});
-				}
-				(true, true) => {
-					target.remove(0);
-					return Ok(RoutingResult::LoadOrAlternativeError {
-						key,
-						redirect_key: target,
-						redirect_code: status_code,
-					});
-				}
-			}
-		} else {
-			let target = compute_redirect_target(&routing_rule.redirect, None);
-			return Ok(RoutingResult::Redirect {
-				url: target,
-				code: status_code,
-			});
-		}
-	};
-
-	if is_bucket_root || is_trailing_slash {
-		Ok(RoutingResult::LoadKey {
-			key,
-			code: StatusCode::OK,
-		})
-	} else {
-		Ok(RoutingResult::LoadOrRedirect {
-			redirect_if_exists: Some(format!("{key}/{index}")),
-			// we can't use `path` because key might have changed substentially in case of
-			// routing rules
-			redirect_url: percent_encoding::percent_encode(
-				format!("/{key}/").as_bytes(),
-				PATH_ENCODING_SET,
-			)
-			.to_string(),
-			key,
-			redirect_code: StatusCode::FOUND,
-		})
+		// "However, if you exclude the trailing slash from the preceding URL, Amazon S3 first looks for an object photos in the bucket. If the photos object is not found, it searches for an index document, photos/index.html. If that document is found, Amazon S3 returns a 302 Found message and points to the photos/ key. For subsequent requests to photos/, Amazon S3 returns photos/index.html. If the index document is not found, Amazon S3 returns an error."
+		(false, false) => Ok((
+			base_key.to_string(),
+			ImplicitRedirect::To {
+				key: format!("{base_key}/{index}"),
+				url: format!("{path}/"),
+			},
+		)),
 	}
 }
 
-// per https://url.spec.whatwg.org/#path-percent-encode-set
-const PATH_ENCODING_SET: &percent_encoding::AsciiSet = &percent_encoding::CONTROLS
-	.add(b' ')
-	.add(b'"')
-	.add(b'#')
-	.add(b'<')
-	.add(b'>')
-	.add(b'?')
-	.add(b'`')
-	.add(b'{')
-	.add(b'}');
-
-fn compute_redirect_target(redirect: &bucket_table::Redirect, suffix: Option<&str>) -> String {
-	let mut res = String::new();
-	if let Some(hostname) = &redirect.hostname {
-		if let Some(protocol) = &redirect.protocol {
-			res.push_str(protocol);
-			res.push_str("://");
-		} else {
-			res.push_str("//");
-		}
-		res.push_str(hostname);
-	}
-	res.push('/');
-	if let Some(replace_key_prefix) = &redirect.replace_key_prefix {
-		res.push_str(replace_key_prefix);
-		if let Some(suffix) = suffix {
-			res.push_str(suffix)
-		}
-	} else if let Some(replace_key) = &redirect.replace_key {
-		res.push_str(replace_key)
-	}
-	res
-}
-
 #[cfg(test)]
 mod tests {
 	use super::*;
@@ -679,39 +453,35 @@ mod tests {
 	#[test]
 	fn path_to_keys_test() -> Result<(), Error> {
 		assert_eq!(
-			path_to_keys("/file%20.jpg", "index.html", &[])?,
-			RoutingResult::LoadOrRedirect {
-				key: "file .jpg".to_string(),
-				redirect_url: "/file%20.jpg/".to_string(),
-				redirect_if_exists: Some("file .jpg/index.html".to_string()),
-				redirect_code: StatusCode::FOUND,
-			}
+			path_to_keys("/file%20.jpg", "index.html")?,
+			(
+				"file .jpg".to_string(),
+				ImplicitRedirect::To {
+					key: "file .jpg/index.html".to_string(),
+					url: "/file%20.jpg/".to_string()
+				}
+			)
 		);
 		assert_eq!(
-			path_to_keys("/%20t/", "index.html", &[])?,
-			RoutingResult::LoadKey {
-				key: " t/index.html".to_string(),
-				code: StatusCode::OK
-			}
+			path_to_keys("/%20t/", "index.html")?,
+			(" t/index.html".to_string(), ImplicitRedirect::No)
 		);
 		assert_eq!(
-			path_to_keys("/", "index.html", &[])?,
-			RoutingResult::LoadKey {
-				key: "index.html".to_string(),
-				code: StatusCode::OK
-			}
+			path_to_keys("/", "index.html")?,
+			("index.html".to_string(), ImplicitRedirect::No)
 		);
 		assert_eq!(
-			path_to_keys("/hello", "index.html", &[])?,
-			RoutingResult::LoadOrRedirect {
-				key: "hello".to_string(),
-				redirect_url: "/hello/".to_string(),
-				redirect_if_exists: Some("hello/index.html".to_string()),
-				redirect_code: StatusCode::FOUND,
-			}
+			path_to_keys("/hello", "index.html")?,
+			(
+				"hello".to_string(),
+				ImplicitRedirect::To {
+					key: "hello/index.html".to_string(),
+					url: "/hello/".to_string()
+				}
+			)
 		);
-		assert!(path_to_keys("", "index.html", &[]).is_err());
-		assert!(path_to_keys("i/am/relative", "index.html", &[]).is_err());
+		assert!(path_to_keys("", "index.html").is_err());
+		assert!(path_to_keys("i/am/relative", "index.html").is_err());
 		Ok(())
 	}
 }