Merge branch 'main' into next-0.10

2026-08-13 15:46:53 +00:00 · 2024-03-04 15:56:10 +01:00
parent d0d95fd53f 3168bb34a0
commit bbde9bc912
54 changed files with 6525 additions and 763 deletions
@@ -55,8 +55,8 @@ Create your key and bucket:

 ```bash
 garage key create my-key
-garage bucket create backup
-garage bucket allow backup --read --write --key my-key
+garage bucket create backups
+garage bucket allow backups --read --write --key my-key
 ```

 Then register your Key ID and Secret key in your environment:
@@ -85,14 +85,14 @@ to store 2 TB of data in total.
 ## Get a Docker image

 Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
-We encourage you to use a fixed tag (eg. `v0.9.1`) and not the `latest` tag.
-For this example, we will use the latest published version at the time of the writing which is `v0.9.1` but it's up to you
+We encourage you to use a fixed tag (eg. `v0.9.3`) and not the `latest` tag.
+For this example, we will use the latest published version at the time of the writing which is `v0.9.3` but it's up to you
 to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).

 For example:

 ```
-sudo docker pull dxflrs/garage:v0.9.1
+sudo docker pull dxflrs/garage:v0.9.3
 ```

 ## Deploying and configuring Garage
@@ -157,7 +157,7 @@ docker run \
  -v /etc/garage.toml:/etc/garage.toml \
  -v /var/lib/garage/meta:/var/lib/garage/meta \
  -v /var/lib/garage/data:/var/lib/garage/data \
-  dxflrs/garage:v0.9.1
+  dxflrs/garage:v0.9.3
 ```

 With this command line, Garage should be started automatically at each boot.
@@ -171,7 +171,7 @@ If you want to use `docker-compose`, you may use the following `docker-compose.y
 version: "3"
 services:
  garage:
-    image: dxflrs/garage:v0.9.1
+    image: dxflrs/garage:v0.9.3
    network_mode: "host"
    restart: unless-stopped
    volumes:
@@ -79,8 +79,9 @@ index = "index.html"
 api_bind_addr = "[::]:3904"

 [admin]
-api_bind_addr = "0.0.0.0:3903"
+api_bind_addr = "[::]:3903"
 admin_token = "$(openssl rand -base64 32)"
+metrics_token = "$(openssl rand -base64 32)"
 EOF
 ```

@@ -69,8 +69,8 @@ root_domain = ".web.garage"

 [admin]
 api_bind_addr = "0.0.0.0:3903"
-metrics_token = "cacce0b2de4bc2d9f5b5fdff551e01ac1496055aed248202d415398987e35f81"
-admin_token = "ae8cb40ea7368bbdbb6430af11cca7da833d3458a5f52086f4e805a570fb5c2a"
+metrics_token = "BCAdFjoa9G0KJR0WXnHHm7fs1ZAbfpI8iIZ+Z/a2NgI="
+admin_token = "UkLeGWEvHnXBqnueR3ISEMWpOnm40jH2tM2HnnL/0F4="
 trace_sink = "http://localhost:4317"
 ```

@@ -417,7 +417,7 @@ the node, even in the case of a NAT: the NAT should be configured to forward the
 port number to the same internal port nubmer. This means that if you have several nodes running
 behind a NAT, they should each use a different RPC port number.

-#### `rpc_bind_outgoing` {#rpc_bind_outgoing} (since v0.9.2)
+#### `rpc_bind_outgoing`(since v0.9.2) {#rpc_bind_outgoing}

 If enabled, pre-bind all sockets for outgoing connections to the same IP address
 used for listening (the IP address specified in `rpc_bind_addr`) before
@@ -612,7 +612,7 @@ the socket will have 0220 mode. Make sure to set user and group permissions acco
 The token for accessing the Metrics endpoint. If this token is not set, the
 Metrics endpoint can be accessed without access control.

-You can use any random string for this value. We recommend generating a random token with `openssl rand -hex 32`.
+You can use any random string for this value. We recommend generating a random token with `openssl rand -base64 32`.

 `metrics_token` was introduced in Garage `v0.7.2`.
 `metrics_token_file` and the `GARAGE_METRICS_TOKEN` environment variable are supported since Garage `v0.8.2`.
@@ -624,7 +624,7 @@ You can use any random string for this value. We recommend generating a random t
 The token for accessing all of the other administration endpoints.  If this
 token is not set, access to these endpoints is disabled entirely.

-You can use any random string for this value. We recommend generating a random token with `openssl rand -hex 32`.
+You can use any random string for this value. We recommend generating a random token with `openssl rand -base64 32`.

 `admin_token` was introduced in Garage `v0.7.2`.
 `admin_token_file` and the `GARAGE_ADMIN_TOKEN` environment variable are supported since Garage `v0.8.2`.
@@ -37,6 +37,21 @@ A Garage cluster can very easily evolve over time, as storage nodes are added or
 Garage will automatically rebalance data between nodes as needed to ensure the desired number of copies.
 Read about cluster layout management [here](@/documentation/operations/layout.md).

+### Several replication modes
+
+Garage supports a variety of replication modes, with 1 copy, 2 copies or 3 copies of your data,
+and with various levels of consistency, in order to adapt to a variety of usage scenarios.
+Read our reference page on [supported replication modes](@/documentation/reference-manual/configuration.md#replication_mode)
+to select the replication mode best suited to your use case (hint: in most cases, `replication_mode = "3"` is what you want).
+
+### Compression and deduplication
+
+All data stored in Garage is deduplicated, and optionnally compressed using
+Zstd.  Objects uploaded to Garage are chunked in blocks of constant sizes (see
+[`block_size`](@/documentation/reference-manual/configuration.md#block_size)),
+and the hashes of individual blocks are used to dispatch them to storage nodes
+and to deduplicate them.
+
 ### No RAFT slowing you down

 It might seem strange to tout the absence of something as a desirable feature,
@@ -48,13 +63,6 @@ As a consequence, requests can be handled much faster, even in cases where laten
 between cluster nodes is important (see our [benchmarks](@/documentation/design/benchmarks/index.md) for data on this).
 This is particularly usefull when nodes are far from one another and talk to one other through standard Internet connections.

-### Several replication modes
-
-Garage supports a variety of replication modes, with 1 copy, 2 copies or 3 copies of your data,
-and with various levels of consistency, in order to adapt to a variety of usage scenarios.
-Read our reference page on [supported replication modes](@/documentation/reference-manual/configuration.md#replication_mode)
-to select the replication mode best suited to your use case (hint: in most cases, `replication_mode = "3"` is what you want).
-
 ### Web server for static websites

 A storage bucket can easily be configured to be served directly by Garage as a static web site.
@@ -0,0 +1,17 @@
+*
+
+!*.txt
+!*.md
+
+!assets
+
+!.gitignore
+!*.svg
+!*.png
+!*.jpg
+!*.tex
+!Makefile
+!.gitignore
+!assets/*.drawio.pdf
+
+!talk.pdf
@@ -0,0 +1,10 @@
+ASSETS=../assets/logos/deuxfleurs.pdf
+
+talk.pdf: talk.tex $(ASSETS)
+	pdflatex talk.tex
+
+%.pdf: %.svg
+	inkscape -D -z --file=$^ --export-pdf=$@
+
+%.pdf_tex: %.svg
+	inkscape -D -z --file=$^ --export-pdf=$@ --export-latex
@@ -0,0 +1,543 @@
+\nonstopmode
+\documentclass[aspectratio=169,xcolor={svgnames}]{beamer}
+\usepackage[utf8]{inputenc}
+% \usepackage[frenchb]{babel}
+\usepackage{amsmath}
+\usepackage{mathtools}
+\usepackage{breqn}
+\usepackage{multirow}
+\usetheme{boxes}
+\usepackage{graphicx}
+\usepackage{import}
+\usepackage{adjustbox}
+\usepackage[absolute,overlay]{textpos}
+%\useoutertheme[footline=authortitle,subsection=false]{miniframes}
+%\useoutertheme[footline=authorinstitute,subsection=false]{miniframes}
+\useoutertheme{infolines}
+\setbeamertemplate{headline}{}
+
+\beamertemplatenavigationsymbolsempty
+
+\definecolor{TitleOrange}{RGB}{255,137,0}
+\setbeamercolor{title}{fg=TitleOrange}
+\setbeamercolor{frametitle}{fg=TitleOrange}
+
+\definecolor{ListOrange}{RGB}{255,145,5}
+\setbeamertemplate{itemize item}{\color{ListOrange}$\blacktriangleright$}
+
+\definecolor{verygrey}{RGB}{70,70,70}
+\setbeamercolor{normal text}{fg=verygrey}
+
+
+\usepackage{tabu}
+\usepackage{multicol}
+\usepackage{vwcol}
+\usepackage{stmaryrd}
+\usepackage{graphicx}
+
+\usepackage[normalem]{ulem}
+
+\AtBeginSection[]{
+  \begin{frame}
+  \vfill
+  \centering
+  \begin{beamercolorbox}[sep=8pt,center,shadow=true,rounded=true]{title}
+    \usebeamerfont{title}\insertsectionhead\par%
+  \end{beamercolorbox}
+  \vfill
+  \end{frame}
+}
+
+\title{Garage}
+\author{Alex Auvolat, Deuxfleurs}
+\date{Capitoul, 2024-02-29}
+
+\begin{document}
+
+\begin{frame}
+	\centering
+	\includegraphics[width=.3\linewidth]{../../sticker/Garage.png}
+	\vspace{1em}
+
+	{\large\bf Alex Auvolat, Deuxfleurs Association}
+	\vspace{1em}
+
+	\url{https://garagehq.deuxfleurs.fr/}
+
+	Matrix channel: \texttt{\#garage:deuxfleurs.fr}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Who I am}
+	\begin{columns}[t]
+		\begin{column}{.2\textwidth}
+			\centering
+			\adjincludegraphics[width=.4\linewidth, valign=t]{../assets/alex.jpg}
+		\end{column}
+		\begin{column}{.6\textwidth}
+			\textbf{Alex Auvolat}\\
+			PhD; co-founder of Deuxfleurs
+		\end{column}
+		\begin{column}{.2\textwidth}
+			~
+		\end{column}
+	\end{columns}
+	\vspace{2em}
+
+	\begin{columns}[t]
+		\begin{column}{.2\textwidth}
+			\centering
+			\adjincludegraphics[width=.5\linewidth, valign=t]{../assets/logos/deuxfleurs.pdf}
+		\end{column}
+		\begin{column}{.6\textwidth}
+			\textbf{Deuxfleurs}\\
+			A non-profit self-hosting collective,\\
+			member of the CHATONS network
+		\end{column}
+		\begin{column}{.2\textwidth}
+			\centering
+			\adjincludegraphics[width=.7\linewidth, valign=t]{../assets/logos/logo_chatons.png}
+		\end{column}
+	\end{columns}
+
+\end{frame}
+
+\begin{frame}
+	\frametitle{Our objective at Deuxfleurs}
+	
+	\begin{center}
+		\textbf{Promote self-hosting and small-scale hosting\\
+			as an alternative to large cloud providers}
+	\end{center}
+	\vspace{2em}
+	\visible<2->{
+		Why is it hard?
+		\vspace{2em}
+		\begin{center}
+			\textbf{\underline{Resilience}}\\
+			{\footnotesize we want good uptime/availability with low supervision}
+		\end{center}
+	}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Our very low-tech infrastructure}
+
+	\only<1,3-6>{
+		\begin{itemize}
+			\item \textcolor<4->{gray}{Commodity hardware (e.g. old desktop PCs)\\
+				\vspace{.5em}
+				\visible<3->{{\footnotesize (can die at any time)}}}
+				\vspace{1.5em}
+			\item<4-> \textcolor<6->{gray}{Regular Internet (e.g. FTTB, FTTH) and power grid connections\\
+				\vspace{.5em}
+				\visible<5->{{\footnotesize (can be unavailable randomly)}}}
+				\vspace{1.5em}
+			\item<6-> \textbf{Geographical redundancy} (multi-site replication)
+		\end{itemize}
+	}
+	\only<2>{
+		\begin{center}
+			\includegraphics[width=.8\linewidth]{../assets/neptune.jpg}
+		\end{center}
+	}
+	\only<7>{
+		\begin{center}
+			\includegraphics[width=.8\linewidth]{../assets/inframap_jdll2023.pdf}
+		\end{center}
+	}
+\end{frame}
+
+\begin{frame}
+	\frametitle{How to make this happen}
+	\begin{center}
+		\only<1>{\includegraphics[width=.8\linewidth]{../assets/intro/slide1.png}}%
+		\only<2>{\includegraphics[width=.8\linewidth]{../assets/intro/slide2.png}}%
+		\only<3>{\includegraphics[width=.8\linewidth]{../assets/intro/slide3.png}}%
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Distributed file systems are slow}
+	File systems are complex, for example:
+	\vspace{1em}
+	\begin{itemize}
+		\item Concurrent modification by several processes
+			\vspace{1em}
+		\item Folder hierarchies
+			\vspace{1em}
+		\item Other requirements of the POSIX spec (e.g.~locks)
+	\end{itemize}
+	\vspace{1em}
+	Coordination in a distributed system is costly
+
+	\vspace{1em}
+	Costs explode with commodity hardware / Internet connections\\
+	{\small (we experienced this!)}
+\end{frame}
+
+\begin{frame}
+	\frametitle{A simpler solution: object storage}
+	Only two operations:
+	\vspace{1em}
+	\begin{itemize}
+		\item Put an object at a key
+			\vspace{1em}
+		\item Retrieve an object from its key
+	\end{itemize}
+	\vspace{1em}
+	{\footnotesize (and a few others)}
+
+	\vspace{1em}
+	Sufficient for many applications!
+\end{frame}
+
+\begin{frame}
+	\frametitle{A simpler solution: object storage}
+		\begin{center}
+			\includegraphics[height=6em]{../assets/logos/Amazon-S3.jpg}
+			\hspace{3em}
+			\visible<2->{\includegraphics[height=5em]{../assets/logos/minio.png}}
+			\hspace{3em}
+			\visible<3>{\includegraphics[height=6em]{../../logo/garage_hires_crop.png}}
+		\end{center}
+		\vspace{1em}
+	S3: a de-facto standard, many compatible applications
+
+	\vspace{1em}
+	\visible<2->{MinIO is self-hostable but not suited for geo-distributed deployments}
+
+	\vspace{1em}
+	\visible<3->{\textbf{Garage is a self-hosted drop-in replacement for the Amazon S3 object store}}
+\end{frame}
+
+% --------- BASED ON CRDTS ----------
+
+\section{Principle 1: based on CRDTs}
+
+\begin{frame}
+	\frametitle{CRDTs / weak consistency instead of consensus}
+
+	\underline{Internally, Garage uses only CRDTs} (conflict-free replicated data types)
+
+	\vspace{2em}
+	Why not Raft, Paxos, ...? Issues of consensus algorithms:
+
+	\vspace{1em}
+	\begin{itemize}
+		\item<2-> \textbf{Software complexity}
+			\vspace{1em}
+		\item<3-> \textbf{Performance issues:}
+			\vspace{.5em}
+			\begin{itemize}
+				\item<4-> The leader is a \textbf{bottleneck} for all requests\\
+					\vspace{.5em}
+				\item<5-> \textbf{Sensitive to higher latency} between nodes
+					\vspace{.5em}
+				\item<6-> \textbf{Takes time to reconverge} when disrupted (e.g. node going down)
+			\end{itemize}
+	\end{itemize}
+\end{frame}
+
+\begin{frame}
+	\frametitle{The data model of object storage}
+	Object storage is basically a \textbf{key-value store}:
+	\vspace{.5em}
+
+	{\scriptsize
+		\begin{center}
+		\begin{tabular}{|l|p{7cm}|}
+			\hline
+			\textbf{Key: file path + name} & \textbf{Value: file data + metadata} \\
+			\hline
+			\hline
+			\texttt{index.html} &
+				\texttt{Content-Type: text/html; charset=utf-8} \newline
+				\texttt{Content-Length: 24929} \newline
+				\texttt{<binary blob>} \\ 
+			\hline
+			\texttt{img/logo.svg} &
+				\texttt{Content-Type: text/svg+xml} \newline
+				\texttt{Content-Length: 13429} \newline
+				\texttt{<binary blob>} \\ 
+			\hline
+			\texttt{download/index.html} &
+				\texttt{Content-Type: text/html; charset=utf-8} \newline
+				\texttt{Content-Length: 26563} \newline
+				\texttt{<binary blob>} \\ 
+			\hline
+		\end{tabular}
+		\end{center}
+		}
+
+	\vspace{.5em}
+	\begin{itemize}
+		\item<2-> Maps well to CRDT data types
+		\item<3> Read-after-write consistency with quorums
+	\end{itemize}
+\end{frame}
+
+
+\begin{frame}
+	\frametitle{Performance gains in practice}
+	\begin{center}
+		\includegraphics[width=.8\linewidth]{../assets/perf/endpoint_latency_0.7_0.8_minio.png}
+	\end{center}
+\end{frame}
+
+% --------- GEO-DISTRIBUTED MODEL ----------
+
+\section{Principle 2: geo-distributed data model}
+
+\begin{frame}
+	\frametitle{Key-value stores, upgraded: the Dynamo model}
+	\textbf{Two keys:}
+	\begin{itemize}
+		\item Partition key: used to divide data into partitions {\small (a.k.a.~shards)}
+		\item Sort key: used to identify items inside a partition
+	\end{itemize}
+
+	\vspace{1em}
+
+	\begin{center}
+	\begin{tabular}{|l|l|p{3cm}|}
+		\hline
+		\textbf{Partition key: bucket} & \textbf{Sort key: filename} & \textbf{Value} \\
+		\hline
+		\hline
+		\texttt{website} & \texttt{index.html} & (file data) \\
+		\hline
+		\texttt{website} & \texttt{img/logo.svg} & (file data) \\
+		\hline
+		\texttt{website} & \texttt{download/index.html} & (file data) \\
+		\hline
+		\hline
+		\texttt{backup} & \texttt{borg/index.2822} & (file data) \\
+		\hline
+		\texttt{backup} & \texttt{borg/data/2/2329} & (file data) \\
+		\hline
+		\texttt{backup} & \texttt{borg/data/2/2680} & (file data) \\
+		\hline
+		\hline
+		\texttt{private} & \texttt{qq3a2nbe1qjq0ebbvo6ocsp6co} & (file data) \\
+		\hline
+	\end{tabular}
+	\end{center}
+\end{frame}
+
+
+\begin{frame}
+	\frametitle{Layout computation}
+	\begin{overprint}
+		\onslide<1>
+		\begin{center}
+			\includegraphics[width=\linewidth, trim=0 0 0 -4cm]{../assets/screenshots/garage_status_0.9_prod_zonehl.png}
+		\end{center}
+		\onslide<2>
+		\begin{center}
+			\includegraphics[width=.7\linewidth]{../assets/map.png}
+		\end{center}
+	\end{overprint}
+	\vspace{1em}
+	Garage stores replicas on different zones when possible
+\end{frame}
+
+\begin{frame}
+	\frametitle{What a "layout" is}
+		\textbf{A layout is a precomputed index table:}
+		\vspace{1em}
+
+		{\footnotesize
+			\begin{center}
+			\begin{tabular}{|l|l|l|l|}
+				\hline
+				\textbf{Partition} & \textbf{Node 1} & \textbf{Node 2} & \textbf{Node 3} \\
+				\hline
+				\hline
+				Partition 0 & df-ymk (bespin) & Abricot (scorpio) & Courgette (neptune) \\
+				\hline
+				Partition 1 & Ananas (scorpio) & Courgette (neptune) & df-ykl (bespin) \\
+				\hline
+				Partition 2 & df-ymf (bespin) & Celeri (neptune) & Abricot (scorpio) \\
+				\hline
+				\hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ & \hspace{1em}$\vdots$ \\
+				\hline
+				Partition 255 & Concombre (neptune) & df-ykl (bespin) & Abricot (scorpio) \\
+				\hline
+			\end{tabular}
+			\end{center}
+		}
+
+	\vspace{2em}
+	\visible<2->{
+		The index table is built centrally using an optimal algorithm,\\
+		then propagated to all nodes
+	}
+
+	\vspace{1em}
+	\visible<3->{
+		\footnotesize
+		Oulamara, M., \& Auvolat, A. (2023). \emph{An algorithm for geo-distributed and redundant storage in Garage}.\\ arXiv preprint arXiv:2302.13798.
+	}
+\end{frame}
+
+
+\begin{frame}
+	\frametitle{The relationship between \emph{partition} and \emph{partition key}}
+	\begin{center}
+	\begin{tabular}{|l|l|l|l|}
+		\hline
+		\textbf{Partition key} & \textbf{Partition} & \textbf{Sort key} & \textbf{Value} \\
+		\hline
+		\hline
+		\texttt{website} & Partition 12 & \texttt{index.html} & (file data) \\
+		\hline
+		\texttt{website} & Partition 12 & \texttt{img/logo.svg} & (file data) \\
+		\hline
+		\texttt{website} & Partition 12 &\texttt{download/index.html} & (file data) \\
+		\hline
+		\hline
+		\texttt{backup} & Partition 42 & \texttt{borg/index.2822} & (file data) \\
+		\hline
+		\texttt{backup} & Partition 42 & \texttt{borg/data/2/2329} & (file data) \\
+		\hline
+		\texttt{backup} & Partition 42 & \texttt{borg/data/2/2680} & (file data) \\
+		\hline
+		\hline
+		\texttt{private} & Partition 42 & \texttt{qq3a2nbe1qjq0ebbvo6ocsp6co} & (file data) \\
+		\hline
+	\end{tabular}
+	\end{center}
+	\vspace{1em}
+	\textbf{To read or write an item:} hash partition key
+	\\ \hspace{5cm} $\to$ determine partition number (first 8 bits)
+	\\ \hspace{5cm} $\to$ find associated nodes
+\end{frame}
+
+\begin{frame}
+	\frametitle{Garage's internal data structures}
+	\centering
+	\includegraphics[width=.75\columnwidth]{../assets/garage_tables.pdf}
+\end{frame}
+
+% ---------- OPERATING GARAGE ---------
+
+\section{Operating Garage clusters}
+
+\begin{frame}
+	\frametitle{Operating Garage}
+	\begin{center}
+		\only<1-2>{
+			\includegraphics[width=.9\linewidth]{../assets/screenshots/garage_status_0.10.png}
+			\\\vspace{1em}
+			\visible<2>{\includegraphics[width=.9\linewidth]{../assets/screenshots/garage_status_unhealthy_0.10.png}}
+		}
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Background synchronization}
+	\begin{center}
+		\includegraphics[width=.6\linewidth]{../assets/garage_sync.drawio.pdf}
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Digging deeper}
+	\begin{center}
+		\only<1>{\includegraphics[width=.9\linewidth]{../assets/screenshots/garage_stats_0.10.png}}
+		\only<2>{\includegraphics[width=.5\linewidth]{../assets/screenshots/garage_worker_list_0.10.png}}
+		\only<3>{\includegraphics[width=.6\linewidth]{../assets/screenshots/garage_worker_param_0.10.png}}
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Monitoring with Prometheus + Grafana}
+	\begin{center}
+		\includegraphics[width=.9\linewidth]{../assets/screenshots/grafana_dashboard.png}
+	\end{center}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Debugging with traces}
+	\begin{center}
+		\includegraphics[width=.8\linewidth]{../assets/screenshots/jaeger_listobjects.png}
+	\end{center}
+\end{frame}
+
+% ---------- SCALING GARAGE ---------
+
+\section{Scaling Garage clusters}
+
+\begin{frame}
+	\frametitle{Potential limitations and bottlenecks}
+	\begin{itemize}
+		\item Global:
+			\begin{itemize}
+				\item Max. $\sim$100 nodes per cluster (excluding gateways)
+			\end{itemize}
+			\vspace{1em}
+		\item Metadata:
+			\begin{itemize}
+				\item One big bucket = bottleneck, object list on 3 nodes only
+			\end{itemize}
+			\vspace{1em}
+		\item Block manager:
+			\begin{itemize}
+				\item Lots of small files on disk
+				\item Processing the resync queue can be slow
+			\end{itemize}
+	\end{itemize}
+\end{frame}
+
+\begin{frame}
+	\frametitle{Deployment advice for very large clusters}
+	\begin{itemize}
+		\item Metadata storage:
+			\begin{itemize}
+				\item ZFS mirror (x2) on fast NVMe
+				\item Use LMDB storage engine
+			\end{itemize}
+			\vspace{.5em}
+		\item Data block storage:
+			\begin{itemize}
+				\item Use Garage's native multi-HDD support
+				\item XFS on individual drives
+				\item Increase block size (1MB $\to$ 10MB, requires more RAM and good networking)
+				\item Tune \texttt{resync-tranquility} and \texttt{resync-worker-count} dynamically
+			\end{itemize}
+			\vspace{.5em}
+		\item Other :
+			\begin{itemize}
+				\item Split data over several buckets
+				\item Use less than 100 storage nodes
+				\item Use gateway nodes
+			\end{itemize}
+			\vspace{.5em}
+	\end{itemize}
+	Our deployments: $< 10$ TB. Some people have done more!
+\end{frame}
+
+
+% ======================================== END
+% ======================================== END
+% ======================================== END
+
+\begin{frame}
+	\frametitle{Where to find us}
+	\begin{center}
+			\includegraphics[width=.25\linewidth]{../../logo/garage_hires.png}\\
+			\vspace{-1em}
+		\url{https://garagehq.deuxfleurs.fr/}\\
+		\url{mailto:garagehq@deuxfleurs.fr}\\
+		\texttt{\#garage:deuxfleurs.fr} on Matrix
+
+		\vspace{1.5em}
+			\includegraphics[width=.06\linewidth]{../assets/logos/rust_logo.png}
+			\includegraphics[width=.13\linewidth]{../assets/logos/AGPLv3_Logo.png}
+	\end{center}
+\end{frame}
+
+\end{document}
+
+%% vim: set ts=4 sw=4 tw=0 noet spelllang=en :