admin api: increased compatibility for v0/ endpoints

admin api: remove broken GET /v0/key router rule
upgrade doc: fixes and precisions
2026-08-14 16:13:11 +00:00 · 2023-10-05 16:56:13 +02:00 · 2023-10-05 16:27:29 +02:00 · 2023-10-05 15:29:55 +02:00 · 2023-10-05 15:20:48 +02:00 · 2023-10-05 10:42:10 +02:00
29 changed files with 822 additions and 230 deletions
@@ -65,11 +65,16 @@ steps:
      - nix-build --no-build-output --attr pkgs.amd64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
      - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage"

-  - name: integration
+  - name: integration tests
    image: nixpkgs/nix:nixos-22.05
    commands:
      - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)

+  - name: upgrade tests
+    image: nixpkgs/nix:nixos-22.05
+    commands:
+      - nix-shell --attr integration --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
+
  - name: push static binary
    image: nixpkgs/nix:nixos-22.05
    environment:
@@ -116,11 +121,16 @@ steps:
      - nix-build --no-build-output --attr pkgs.i386.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
      - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage"

-  - name: integration
+  - name: integration tests
    image: nixpkgs/nix:nixos-22.05
    commands:
      - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)

+  - name: upgrade tests
+    image: nixpkgs/nix:nixos-22.05
+    commands:
+      - nix-shell --attr integration --run "./script/test-upgrade.sh v0.8.4 i686-unknown-linux-musl" || (cat /tmp/garage.log; false)
+
  - name: push static binary
    image: nixpkgs/nix:nixos-22.05
    environment:
@@ -285,6 +295,6 @@ trigger:

 ---
 kind: signature
-hmac: 24aebbcdba84fd0cdf963061d7bb72ae5b915bfdd0f50c7b019001126fb7fa56
+hmac: 0c4b57eb4b27b7c6a6ff21ab87f0767fe3eb90f5d95d5cbcdccf794e9d2a5d86

 ...
@@ -1271,6 +1271,7 @@ dependencies = [
 "http-range",
 "httpdate",
 "hyper",
+ "hyperlocal",
 "idna",
 "md-5",
 "multer",
@@ -1464,8 +1465,10 @@ dependencies = [
 "garage_util",
 "http",
 "hyper",
+ "hyperlocal",
 "opentelemetry",
 "percent-encoding",
+ "tokio",
 "tracing",
 ]

@@ -1775,6 +1778,19 @@ dependencies = [
 "tokio-io-timeout",
 ]

+[[package]]
+name = "hyperlocal"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fafdf7b2b2de7c9784f76e02c0935e65a8117ec3b768644379983ab333ac98c"
+dependencies = [
+ "futures-util",
+ "hex",
+ "hyper",
+ "pin-project",
+ "tokio",
+]
+
 [[package]]
 name = "iana-time-zone"
 version = "0.1.57"
@@ -33,7 +33,7 @@ args@{
  ignoreLockHash,
 }:
 let
-  nixifiedLockHash = "8ff415a3cc93dd7330ffcc18ee0b3a76c2863e1108be1c88d8e37f29182651f2";
+  nixifiedLockHash = "3f325a8a549c43a788ff702e65f6de2d42ad19a46067248e29108e90212ca2f5";
  workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc;
  currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock);
  lockHashIgnored = if ignoreLockHash
@@ -1809,6 +1809,7 @@ in
      http_range = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-range."0.1.5" { inherit profileName; }).out;
      httpdate = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".httpdate."1.0.3" { inherit profileName; }).out;
      hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.27" { inherit profileName; }).out;
+      hyperlocal = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyperlocal."0.8.0" { inherit profileName; }).out;
      idna = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".idna."0.4.0" { inherit profileName; }).out;
      md5 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".md-5."0.10.5" { inherit profileName; }).out;
      multer = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".multer."2.1.0" { inherit profileName; }).out;
@@ -2064,8 +2065,10 @@ in
      garage_util = (rustPackages."unknown".garage_util."0.8.4" { inherit profileName; }).out;
      http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out;
      hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.27" { inherit profileName; }).out;
+      hyperlocal = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyperlocal."0.8.0" { inherit profileName; }).out;
      opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
      percent_encoding = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.3.0" { inherit profileName; }).out;
+      tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out;
      tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.37" { inherit profileName; }).out;
    };
  });
@@ -2491,6 +2494,23 @@ in
    };
  });
  
+  "registry+https://github.com/rust-lang/crates.io-index".hyperlocal."0.8.0" = overridableMkRustCrate (profileName: rec {
+    name = "hyperlocal";
+    version = "0.8.0";
+    registry = "registry+https://github.com/rust-lang/crates.io-index";
+    src = fetchCratesIo { inherit name version; sha256 = "0fafdf7b2b2de7c9784f76e02c0935e65a8117ec3b768644379983ab333ac98c"; };
+    features = builtins.concatLists [
+      [ "server" ]
+    ];
+    dependencies = {
+      futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out;
+      hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
+      hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.27" { inherit profileName; }).out;
+      pin_project = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.1.3" { inherit profileName; }).out;
+      tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.32.0" { inherit profileName; }).out;
+    };
+  });
+  
  "registry+https://github.com/rust-lang/crates.io-index".iana-time-zone."0.1.57" = overridableMkRustCrate (profileName: rec {
    name = "iana-time-zone";
    version = "0.1.57";
@@ -421,7 +421,7 @@ Now we can write a simple script (eg `~/.local/bin/matrix-cache-gc`):
 ## CONFIGURATION ##
 AWS_ACCESS_KEY_ID=GKxxx
 AWS_SECRET_ACCESS_KEY=xxxx
-S3_ENDPOINT=http://localhost:3900
+AWS_ENDPOINT_URL=http://localhost:3900
 S3_BUCKET=matrix
 MEDIA_STORE=/var/lib/matrix-synapse/media
 PG_USER=matrix
@@ -442,7 +442,7 @@ EOF

 s3_media_upload update-db 1d
 s3_media_upload --no-progress check-deleted $MEDIA_STORE
-s3_media_upload --no-progress upload $MEDIA_STORE $S3_BUCKET --delete --endpoint-url $S3_ENDPOINT
+s3_media_upload --no-progress upload $MEDIA_STORE $S3_BUCKET --delete --endpoint-url $AWS_ENDPOINT_URL
 ```

 This script will list all the medias that were not accessed in the 24 hours according to your database.
@@ -70,16 +70,17 @@ Then a file named `~/.aws/config` and put:
 ```toml
 [default]
 region=garage
+endpoint_url=http://127.0.0.1:3900
 ```

 Now, supposing Garage is listening on `http://127.0.0.1:3900`, you can list your buckets with:

 ```bash
-aws --endpoint-url http://127.0.0.1:3900 s3 ls
+aws s3 ls
 ```

-Passing the `--endpoint-url` parameter to each command is annoying but AWS developers do not provide a corresponding configuration entry.
-As a workaround, you can redefine the aws command by editing the file `~/.bashrc`:
+If you're using awscli `<1.29.0` or `<2.13.0`, you need to pass `--endpoint-url` to each CLI invocation explicitly.
+As a workaround, you can redefine the aws command by editing the file `~/.bashrc` in this case:

 ```
 function aws { command aws --endpoint-url http://127.0.0.1:3900 $@ ; }
@@ -269,12 +269,14 @@ named `~/.awsrc` with this content:
 export AWS_ACCESS_KEY_ID=xxxx      # put your Key ID here
 export AWS_SECRET_ACCESS_KEY=xxxx  # put your Secret key here
 export AWS_DEFAULT_REGION='garage'
-export AWS_ENDPOINT='http://localhost:3900'
+export AWS_ENDPOINT_URL='http://localhost:3900'

-function aws { command aws --endpoint-url $AWS_ENDPOINT $@ ; }
 aws --version
 ```

+Note you need to have at least `awscli` `>=1.29.0` or `>=2.13.0`, otherwise you
+need to specify `--endpoint-url` explicitly on each `awscli` invocation.
+
 Now, each time you want to use `awscli` on this target, run:

 ```bash
@@ -468,6 +468,8 @@ manually.
 The IP and port on which to bind for accepting S3 API calls.
 This endpoint does not suport TLS: a reverse proxy should be used to provide it.

+Alternatively, since `v0.8.5`, a path can be used to create a unix socket with 0222 mode.
+
 ### `s3_region`

 Garage will accept S3 API calls that are targetted to the S3 region defined here.
@@ -497,6 +499,8 @@ The IP and port on which to bind for accepting HTTP requests to buckets configur
 for website access.
 This endpoint does not suport TLS: a reverse proxy should be used to provide it.

+Alternatively, since `v0.8.5`, a path can be used to create a unix socket with 0222 mode.
+
 ### `root_domain`

 The optional suffix appended to bucket names for the corresponding HTTP Host.
@@ -516,6 +520,9 @@ If specified, Garage will bind an HTTP server to this port and address, on
 which it will listen to requests for administration features.
 See [administration API reference](@/documentation/reference-manual/admin-api.md) to learn more about these features.

+Alternatively, since `v0.8.5`, a path can be used to create a unix socket. Note that for security reasons,
+the socket will have 0220 mode. Make sure to set user and group permissions accordingly.
+
 ### `metrics_token`, `metrics_token_file` or `GARAGE_METRICS_TOKEN` (env)

 The token for accessing the Metrics endpoint. If this token is not set, the
@@ -0,0 +1,69 @@
+++
+title = "Migrating from 0.8 to 0.9"
+weight = 14
+++
+
+**This guide explains how to migrate to 0.9 if you have an existing 0.8 cluster.
+We don't recommend trying to migrate to 0.9 directly from 0.7 or older.**
+
+**We make no guarantee that this migration will work perfectly:
+back up all your data before attempting it!**
+
+The following are **breaking changes** in Garage v0.9 that require your attention when migrating:
+
+- LMDB is now the default metadata db engine and Sled is deprecated. If you were using Sled, make sure to specify `db_engine = "sled"` in your configuration file, or take the time to [convert your database](https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#db-engine-since-v0-8-0).
+
+- Capacity values are now in actual byte units. The translation from the old layout will assign 1 capacity = 1Gb by default, which might be wrong for your cluster. This does not cause any data to be moved around, but you might want to re-assign correct capacity values post-migration.
+
+- Multipart uploads that were started in Garage v0.8 will not be visible in Garage v0.9 and will have to be restarted from scratch.
+
+- Changes to the admin API: some `v0/` endpoints have been replaced by `v1/` counterparts with updated/uniformized syntax. All other endpoints have also moved to `v1/` by default, without syntax changes, but are still available under `v0/` for compatibility.
+
+
+## Simple migration procedure (takes cluster offline for a while)
+
+The migration steps are as follows:
+
+1. Disable API and web access. You may do this by stopping your reverse proxy or by commenting out
+   the `api_bind_addr` values in your `config.toml` file and restarting Garage.
+2. Do `garage repair --all-nodes --yes tables` and `garage repair --all-nodes --yes blocks`,
+   check the logs and check that all data seems to be synced correctly between
+   nodes. If you have time, do additional checks (`versions`, `block_refs`, etc.)
+3. Check that the block resync queue and Merkle queue are empty:
+   run `garage stats -a` to query them or inspect metrics in the Grafana dashboard.
+4. Turn off Garage v0.8
+5. **Backup the metadata folder of all your nodes!** For instance, use the following command
+	if your metadata directory is `/var/lib/garage/meta`: `cd /var/lib/garage ; tar -acf meta-v0.8.tar.zst meta/`
+6. Install Garage v0.9
+7. Update your configuration file if necessary.
+8. Turn on Garage v0.9
+9. Do `garage repair --all-nodes --yes tables` and `garage repair --all-nodes --yes blocks`.
+   Wait for a full table sync to run.
+10. Your upgraded cluster should be in a working state. Re-enable API and Web
+    access and check that everything went well.
+11. Monitor your cluster in the next hours to see if it works well under your production load, report any issue.
+12. You might want to assign correct capacity values to all your nodes. Doing so might cause data to be moved
+    in your cluster, which should also be monitored carefully.
+
+## Minimal downtime migration procedure
+
+The migration to Garage v0.9 can be done with almost no downtime,
+by restarting all nodes at once in the new version.
+
+The migration steps are as follows:
+
+1. Do `garage repair --all-nodes --yes tables` and `garage repair --all-nodes --yes blocks`,
+   check the logs and check that all data seems to be synced correctly between
+   nodes. If you have time, do additional checks (`versions`, `block_refs`, etc.)
+
+2. Turn off each node individually; back up its metadata folder (see above); turn it back on again.
+   This will allow you to take a backup of all nodes without impacting global cluster availability.
+   You can do all nodes of a single zone at once as this does not impact the availability of Garage.
+
+3. Prepare your binaries and configuration files for Garage v0.9
+
+4. Shut down all v0.8 nodes simultaneously, and restart them all simultaneously in v0.9.
+   Use your favorite deployment tool (Ansible, Kubernetes, Nomad) to achieve this as fast as possible.
+   Garage v0.9 should be in a working state as soon as it starts.
+
+5. Proceed with repair and monitoring as described in steps 9-12 above.
@@ -9,11 +9,22 @@ GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
 NIX_RELEASE="${REPO_FOLDER}/result/bin/"
 PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH"

-garage -c /tmp/config.1.toml bucket create eprouvette
-KEY_INFO=$(garage -c /tmp/config.1.toml key create opérateur)
+if [ -z "$GARAGE_BIN" ]; then
+	GARAGE_BIN=$(which garage || exit 1)
+	echo -en "Found garage at: ${GARAGE_BIN}\n"
+else
+	echo -en "Using garage binary at: ${GARAGE_BIN}\n"
+fi
+
+$GARAGE_BIN -c /tmp/config.1.toml bucket create eprouvette
+if [ "$GARAGE_08" = "1" ]; then
+	KEY_INFO=$($GARAGE_BIN -c /tmp/config.1.toml key new --name opérateur)
+else
+	KEY_INFO=$($GARAGE_BIN -c /tmp/config.1.toml key create opérateur)
+fi
 ACCESS_KEY=`echo $KEY_INFO|grep -Po 'GK[a-f0-9]+'`
 SECRET_KEY=`echo $KEY_INFO|grep -Po 'Secret key: [a-f0-9]+'|grep -Po '[a-f0-9]+$'`
-garage -c /tmp/config.1.toml bucket allow eprouvette --read --write --owner --key $ACCESS_KEY
+$GARAGE_BIN -c /tmp/config.1.toml bucket allow eprouvette --read --write --owner --key $ACCESS_KEY
 echo "$ACCESS_KEY $SECRET_KEY" > /tmp/garage.s3

 echo "Bucket s3://eprouvette created. Credentials stored in /tmp/garage.s3."
@@ -14,8 +14,13 @@ export RUST_BACKTRACE=1
 export RUST_LOG=garage=info,garage_api=debug
 MAIN_LABEL="\e[${FANCYCOLORS[0]}[main]\e[49m"

-WHICH_GARAGE=$(which garage || exit 1)
-echo -en "${MAIN_LABEL} Found garage at: ${WHICH_GARAGE}\n"
+if [ -z "$GARAGE_BIN" ]; then
+	GARAGE_BIN=$(which garage || exit 1)
+	echo -en "${MAIN_LABEL} Found garage at: ${GARAGE_BIN}\n"
+else
+	echo -en "${MAIN_LABEL} Using garage binary at: ${GARAGE_BIN}\n"
+fi
+$GARAGE_BIN --version

 NETWORK_SECRET="$(openssl rand -hex 32)"

@@ -28,6 +33,7 @@ LABEL="\e[${FANCYCOLORS[$count]}[$count]\e[49m"
 cat > $CONF_PATH <<EOF
 block_size = 1048576			# objects are split in blocks of maximum this number of bytes
 metadata_dir = "/tmp/garage-meta-$count"
+db_engine = "lmdb"
 data_dir = "/tmp/garage-data-$count"
 rpc_bind_addr = "0.0.0.0:$((3900+$count))"		# the port other Garage nodes will use to talk to this node
 rpc_public_addr = "127.0.0.1:$((3900+$count))"
@@ -51,7 +57,7 @@ EOF

 echo -en "$LABEL configuration written to $CONF_PATH\n"

-(garage -c /tmp/config.$count.toml server 2>&1|while read r; do echo -en "$LABEL $r\n"; done) &
+($GARAGE_BIN -c /tmp/config.$count.toml server 2>&1|while read r; do echo -en "$LABEL $r\n"; done) &
 done
 # >>>>>>>>>>>>>>>> END FOR LOOP ON NODES

@@ -73,14 +79,14 @@ fi
 sleep 3
 # Establish connections between nodes
 for count in $(seq 1 3); do
-	NODE=$(garage -c /tmp/config.$count.toml node id -q)
+	NODE=$($GARAGE_BIN -c /tmp/config.$count.toml node id -q)
 	for count2 in $(seq 1 3); do
-		garage -c /tmp/config.$count2.toml node connect $NODE
+		$GARAGE_BIN -c /tmp/config.$count2.toml node connect $NODE
 	done
 done

 RETRY=120
-until garage -c /tmp/config.1.toml status 2>&1|grep -q HEALTHY ; do 
+until $GARAGE_BIN -c /tmp/config.1.toml status 2>&1|grep -q HEALTHY ; do 
  (( RETRY-- ))
  if (( RETRY <= 0 )); then 
    echo -en "${MAIN_LABEL} Garage did not start"
@@ -9,9 +9,17 @@ GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
 NIX_RELEASE="${REPO_FOLDER}/result/bin/"
 PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH"

+if [ -z "$GARAGE_BIN" ]; then
+	GARAGE_BIN=$(which garage || exit 1)
+	echo -en "Found garage at: ${GARAGE_BIN}\n"
+else
+	echo -en "Using garage binary at: ${GARAGE_BIN}\n"
+fi
+$GARAGE_BIN --version
+
 sleep 5
 RETRY=120
-until garage -c /tmp/config.1.toml status 2>&1|grep -q HEALTHY ; do 
+until $GARAGE_BIN -c /tmp/config.1.toml status 2>&1|grep -q HEALTHY ; do
  (( RETRY-- ))
  if (( RETRY <= 0 )); then
    echo "garage did not start in time, failing."
@@ -21,12 +29,20 @@ until garage -c /tmp/config.1.toml status 2>&1|grep -q HEALTHY ; do
 	sleep 1
 done

-garage -c /tmp/config.1.toml status \
-	| grep 'NO ROLE' \
-	| grep -Po '^[0-9a-f]+' \
-	| while read id; do 
-	  garage -c /tmp/config.1.toml layout assign $id -z dc1 -c 1G
-	done
+if [ "$GARAGE_08" = "1" ]; then
+	$GARAGE_BIN -c /tmp/config.1.toml status \
+		| grep 'NO ROLE' \
+		| grep -Po '^[0-9a-f]+' \
+		| while read id; do
+		  $GARAGE_BIN -c /tmp/config.1.toml layout assign $id -z dc1 -c 1
+		done
+else
+	$GARAGE_BIN -c /tmp/config.1.toml status \
+		| grep 'NO ROLE' \
+		| grep -Po '^[0-9a-f]+' \
+		| while read id; do
+		  $GARAGE_BIN -c /tmp/config.1.toml layout assign $id -z dc1 -c 1G
+		done
+fi

-garage -c /tmp/config.1.toml layout config -r 1
-garage -c /tmp/config.1.toml layout apply --version 1
+$GARAGE_BIN -c /tmp/config.1.toml layout apply --version 1
@@ -1,7 +1,7 @@
 export AWS_ACCESS_KEY_ID=`cat /tmp/garage.s3 |cut -d' ' -f1`
 export AWS_SECRET_ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
 export AWS_DEFAULT_REGION='garage'
-
+# FUTUREWORK: set AWS_ENDPOINT_URL instead, once nixpkgs bumps awscli to >=2.13.0.
 function aws { command aws --endpoint-url http://127.0.0.1:3911 $@ ; }

 aws --version
@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+
+set -ex
+
+export LC_ALL=C.UTF-8
+export LANG=C.UTF-8
+SCRIPT_FOLDER="`dirname \"$0\"`"
+REPO_FOLDER="${SCRIPT_FOLDER}/../"
+GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
+GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
+NIX_RELEASE="${REPO_FOLDER}/result/bin/:${REPO_FOLDER}/result-bin/bin/"
+PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH"
+
+OLD_VERSION="$1"
+ARCH="$2"
+
+
+echo "Downloading old garage binary..."
+curl https://garagehq.deuxfleurs.fr/_releases/$OLD_VERSION/$ARCH/garage > /tmp/old_garage
+chmod +x /tmp/old_garage
+
+echo "============= insert data into old version cluster ================="
+
+export GARAGE_BIN=/tmp/old_garage
+if echo $OLD_VERSION | grep 'v0\.8\.'; then
+	echo "Detected Garage v0.8.x"
+	export GARAGE_08=1
+fi
+
+echo "⏳ Setup cluster using old version"
+$GARAGE_BIN --version
+${SCRIPT_FOLDER}/dev-clean.sh
+${SCRIPT_FOLDER}/dev-cluster.sh > /tmp/garage.log 2>&1 &
+sleep 6
+${SCRIPT_FOLDER}/dev-configure.sh
+${SCRIPT_FOLDER}/dev-bucket.sh
+
+echo "🛠️ Inserting data in old cluster"
+source ${SCRIPT_FOLDER}/dev-env-rclone.sh
+rclone copy "${SCRIPT_FOLDER}/../.git/" garage:eprouvette/test_dotgit --stats=1s --stats-log-level=NOTICE --stats-one-line
+
+echo "🏁 Stopping old cluster"
+killall -INT old_garage
+sleep 2
+killall -9 old_garage || true
+
+echo "🏁 Removing old garage version"
+rm -rv $GARAGE_BIN
+export -n GARAGE_BIN
+export -n GARAGE_08
+
+echo "================ read data from new cluster ==================="
+
+echo "⏳ Setup cluster using new version"
+pwd
+ls
+export GARAGE_BIN=$(which garage)
+$GARAGE_BIN --version
+${SCRIPT_FOLDER}/dev-cluster.sh >> /tmp/garage.log 2>&1 &
+sleep 3
+
+echo "🛠️ Retrieving data from old cluster"
+rclone copy garage:eprouvette/test_dotgit /tmp/test_dotgit --stats=1s --stats-log-level=NOTICE --stats-one-line --fast-list
+
+if ! diff <(find "${SCRIPT_FOLDER}/../.git" -type f | xargs md5sum | cut -d ' ' -f 1 | sort) <(find /tmp/test_dotgit -type f | xargs md5sum | cut -d ' ' -f 1 | sort); then
+	echo "TEST FAILURE: directories are different"
+	exit 1
+fi
+rm -r /tmp/test_dotgit
+
+echo "🏁 Teardown"
+rm -rf /tmp/garage-{data,meta}-*
+rm -rf /tmp/config.*.toml
+
+echo "✅ Success"
@@ -45,6 +45,7 @@ http = "0.2"
 httpdate = "1.0"
 http-range = "0.1"
 hyper = { version = "0.14", features = ["server", "http1", "runtime", "tcp", "stream"] }
+hyperlocal = { version = "0.8.0", default-features = false, features = ["server"] }
 multer = "2.0"
 percent-encoding = "2.1.0"
 roxmltree = "0.18"
@@ -1,5 +1,4 @@
 use std::collections::HashMap;
-use std::net::SocketAddr;
 use std::sync::Arc;

 use async_trait::async_trait;
@@ -18,6 +17,7 @@ use prometheus::{Encoder, TextEncoder};
 use garage_model::garage::Garage;
 use garage_rpc::system::ClusterHealthStatus;
 use garage_util::error::Error as GarageError;
+use garage_util::socket_address::UnixOrTCPSocketAddress;

 use crate::generic_server::*;

@@ -25,7 +25,8 @@ use crate::admin::bucket::*;
 use crate::admin::cluster::*;
 use crate::admin::error::*;
 use crate::admin::key::*;
-use crate::admin::router::{Authorization, Endpoint};
+use crate::admin::router_v0;
+use crate::admin::router_v1::{Authorization, Endpoint};
 use crate::helpers::host_to_bucket;

 pub struct AdminApiServer {
@@ -61,12 +62,12 @@ impl AdminApiServer {

 	pub async fn run(
 		self,
-		bind_addr: SocketAddr,
+		bind_addr: UnixOrTCPSocketAddress,
 		shutdown_signal: impl Future<Output = ()>,
 	) -> Result<(), GarageError> {
 		let region = self.garage.config.s3_api.s3_region.clone();
 		ApiServer::new(region, self)
-			.run_server(bind_addr, shutdown_signal)
+			.run_server(bind_addr, Some(0o220), shutdown_signal)
 			.await
 	}

@@ -229,7 +230,12 @@ impl ApiHandler for AdminApiServer {
 	type Error = Error;

 	fn parse_endpoint(&self, req: &Request<Body>) -> Result<Endpoint, Error> {
-		Endpoint::from_request(req)
+		if req.uri().path().starts_with("/v0/") {
+			let endpoint_v0 = router_v0::Endpoint::from_request(req)?;
+			Endpoint::from_v0(endpoint_v0)
+		} else {
+			Endpoint::from_request(req)
+		}
 	}

 	async fn handle(
@@ -1,6 +1,7 @@
 pub mod api_server;
 mod error;
-mod router;
+mod router_v0;
+mod router_v1;

 mod bucket;
 mod cluster;
@@ -1,160 +0,0 @@
-use std::borrow::Cow;
-
-use hyper::{Method, Request};
-
-use crate::admin::error::*;
-use crate::router_macros::*;
-
-pub enum Authorization {
-	None,
-	MetricsToken,
-	AdminToken,
-}
-
-router_match! {@func
-
-/// List of all Admin API endpoints.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub enum Endpoint {
-	Options,
-	CheckDomain,
-	Health,
-	Metrics,
-	GetClusterStatus,
-	GetClusterHealth,
-	ConnectClusterNodes,
-	// Layout
-	GetClusterLayout,
-	UpdateClusterLayout,
-	ApplyClusterLayout,
-	RevertClusterLayout,
-	// Keys
-	ListKeys,
-	CreateKey,
-	ImportKey,
-	GetKeyInfo {
-		id: Option<String>,
-		search: Option<String>,
-		show_secret_key: Option<String>,
-	},
-	DeleteKey {
-		id: String,
-	},
-	UpdateKey {
-		id: String,
-	},
-	// Buckets
-	ListBuckets,
-	CreateBucket,
-	GetBucketInfo {
-		id: Option<String>,
-		global_alias: Option<String>,
-	},
-	DeleteBucket {
-		id: String,
-	},
-	UpdateBucket {
-		id: String,
-	},
-	// Bucket-Key Permissions
-	BucketAllowKey,
-	BucketDenyKey,
-	// Bucket aliases
-	GlobalAliasBucket {
-		id: String,
-		alias: String,
-	},
-	GlobalUnaliasBucket {
-		id: String,
-		alias: String,
-	},
-	LocalAliasBucket {
-		id: String,
-		access_key_id: String,
-		alias: String,
-	},
-	LocalUnaliasBucket {
-		id: String,
-		access_key_id: String,
-		alias: String,
-	},
-}}
-
-impl Endpoint {
-	/// Determine which S3 endpoint a request is for using the request, and a bucket which was
-	/// possibly extracted from the Host header.
-	/// Returns Self plus bucket name, if endpoint is not Endpoint::ListBuckets
-	pub fn from_request<T>(req: &Request<T>) -> Result<Self, Error> {
-		let uri = req.uri();
-		let path = uri.path();
-		let query = uri.query();
-
-		let mut query = QueryParameters::from_query(query.unwrap_or_default())?;
-
-		let res = router_match!(@gen_path_parser (req.method(), path, query) [
-			OPTIONS _ => Options,
-			GET "/check" => CheckDomain,
-			GET "/health" => Health,
-			GET "/metrics" => Metrics,
-			GET "/v1/status" => GetClusterStatus,
-			GET "/v1/health" => GetClusterHealth,
-			POST ("/v0/connect" | "/v1/connect") => ConnectClusterNodes,
-			// Layout endpoints
-			GET "/v1/layout" => GetClusterLayout,
-			POST "/v1/layout" => UpdateClusterLayout,
-			POST "/v1/layout/apply" => ApplyClusterLayout,
-			POST ("/v0/layout/revert" | "/v1/layout/revert") => RevertClusterLayout,
-			// API key endpoints
-			GET "/v1/key" if id => GetKeyInfo (query_opt::id, query_opt::search, query_opt::show_secret_key),
-			GET "/v1/key" if search => GetKeyInfo (query_opt::id, query_opt::search, query_opt::show_secret_key),
-			POST "/v1/key" if id => UpdateKey (query::id),
-			POST "/v1/key" => CreateKey,
-			POST "/v1/key/import" => ImportKey,
-			DELETE ("/v0/key" | "/v1/key") if id => DeleteKey (query::id),
-			GET ("/v0/key" | "/v1/key") => ListKeys,
-			// Bucket endpoints
-			GET ("/v0/bucket" | "/v1/bucket") if id => GetBucketInfo (query_opt::id, query_opt::global_alias),
-			GET ("/v0/bucket" | "/v1/bucket") if global_alias => GetBucketInfo (query_opt::id, query_opt::global_alias),
-			GET ("/v0/bucket" | "/v1/bucket") => ListBuckets,
-			POST ("/v0/bucket" | "/v1/bucket") => CreateBucket,
-			DELETE ("/v0/bucket" | "/v1/bucket") if id => DeleteBucket (query::id),
-			PUT ("/v0/bucket" | "/v1/bucket") if id => UpdateBucket (query::id),
-			// Bucket-key permissions
-			POST ("/v0/bucket/allow" | "/v1/bucket/allow") => BucketAllowKey,
-			POST ("/v0/bucket/deny" | "/v1/bucket/deny") => BucketDenyKey,
-			// Bucket aliases
-			PUT ("/v0/bucket/alias/global" | "/v1/bucket/alias/global") => GlobalAliasBucket (query::id, query::alias),
-			DELETE ("/v0/bucket/alias/global" | "/v1/bucket/alias/global") => GlobalUnaliasBucket (query::id, query::alias),
-			PUT ("/v0/bucket/alias/local" | "/v1/bucket/alias/local") => LocalAliasBucket (query::id, query::access_key_id, query::alias),
-			DELETE ("/v0/bucket/alias/local" | "/v1/bucket/alias/local") => LocalUnaliasBucket (query::id, query::access_key_id, query::alias),
-		]);
-
-		if let Some(message) = query.nonempty_message() {
-			debug!("Unused query parameter: {}", message)
-		}
-
-		Ok(res)
-	}
-	/// Get the kind of authorization which is required to perform the operation.
-	pub fn authorization_type(&self) -> Authorization {
-		match self {
-			Self::Health => Authorization::None,
-			Self::CheckDomain => Authorization::None,
-			Self::Metrics => Authorization::MetricsToken,
-			_ => Authorization::AdminToken,
-		}
-	}
-}
-
-generateQueryParameters! {
-	keywords: [],
-	fields: [
-		"format" => format,
-		"id" => id,
-		"search" => search,
-		"globalAlias" => global_alias,
-		"alias" => alias,
-		"accessKeyId" => access_key_id,
-		"showSecretKey" => show_secret_key
-	]
-}
@@ -0,0 +1,143 @@
+use std::borrow::Cow;
+
+use hyper::{Method, Request};
+
+use crate::admin::error::*;
+use crate::router_macros::*;
+
+router_match! {@func
+
+/// List of all Admin API endpoints.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum Endpoint {
+	Options,
+	CheckDomain,
+	Health,
+	Metrics,
+	GetClusterStatus,
+	GetClusterHealth,
+	ConnectClusterNodes,
+	// Layout
+	GetClusterLayout,
+	UpdateClusterLayout,
+	ApplyClusterLayout,
+	RevertClusterLayout,
+	// Keys
+	ListKeys,
+	CreateKey,
+	ImportKey,
+	GetKeyInfo {
+		id: Option<String>,
+		search: Option<String>,
+	},
+	DeleteKey {
+		id: String,
+	},
+	UpdateKey {
+		id: String,
+	},
+	// Buckets
+	ListBuckets,
+	CreateBucket,
+	GetBucketInfo {
+		id: Option<String>,
+		global_alias: Option<String>,
+	},
+	DeleteBucket {
+		id: String,
+	},
+	UpdateBucket {
+		id: String,
+	},
+	// Bucket-Key Permissions
+	BucketAllowKey,
+	BucketDenyKey,
+	// Bucket aliases
+	GlobalAliasBucket {
+		id: String,
+		alias: String,
+	},
+	GlobalUnaliasBucket {
+		id: String,
+		alias: String,
+	},
+	LocalAliasBucket {
+		id: String,
+		access_key_id: String,
+		alias: String,
+	},
+	LocalUnaliasBucket {
+		id: String,
+		access_key_id: String,
+		alias: String,
+	},
+}}
+
+impl Endpoint {
+	/// Determine which S3 endpoint a request is for using the request, and a bucket which was
+	/// possibly extracted from the Host header.
+	/// Returns Self plus bucket name, if endpoint is not Endpoint::ListBuckets
+	pub fn from_request<T>(req: &Request<T>) -> Result<Self, Error> {
+		let uri = req.uri();
+		let path = uri.path();
+		let query = uri.query();
+
+		let mut query = QueryParameters::from_query(query.unwrap_or_default())?;
+
+		let res = router_match!(@gen_path_parser (req.method(), path, query) [
+			OPTIONS _ => Options,
+			GET "/check" => CheckDomain,
+			GET "/health" => Health,
+			GET "/metrics" => Metrics,
+			GET "/v0/status" => GetClusterStatus,
+			GET "/v0/health" => GetClusterHealth,
+			POST "/v0/connect" => ConnectClusterNodes,
+			// Layout endpoints
+			GET "/v0/layout" => GetClusterLayout,
+			POST "/v0/layout" => UpdateClusterLayout,
+			POST "/v0/layout/apply" => ApplyClusterLayout,
+			POST "/v0/layout/revert" => RevertClusterLayout,
+			// API key endpoints
+			GET "/v0/key" if id => GetKeyInfo (query_opt::id, query_opt::search),
+			GET "/v0/key" if search => GetKeyInfo (query_opt::id, query_opt::search),
+			POST "/v0/key" if id => UpdateKey (query::id),
+			POST "/v0/key" => CreateKey,
+			POST "/v0/key/import" => ImportKey,
+			DELETE "/v0/key" if id => DeleteKey (query::id),
+			GET "/v0/key" => ListKeys,
+			// Bucket endpoints
+			GET "/v0/bucket" if id => GetBucketInfo (query_opt::id, query_opt::global_alias),
+			GET "/v0/bucket" if global_alias => GetBucketInfo (query_opt::id, query_opt::global_alias),
+			GET "/v0/bucket" => ListBuckets,
+			POST "/v0/bucket" => CreateBucket,
+			DELETE "/v0/bucket" if id => DeleteBucket (query::id),
+			PUT "/v0/bucket" if id => UpdateBucket (query::id),
+			// Bucket-key permissions
+			POST "/v0/bucket/allow" => BucketAllowKey,
+			POST "/v0/bucket/deny" => BucketDenyKey,
+			// Bucket aliases
+			PUT "/v0/bucket/alias/global" => GlobalAliasBucket (query::id, query::alias),
+			DELETE "/v0/bucket/alias/global" => GlobalUnaliasBucket (query::id, query::alias),
+			PUT "/v0/bucket/alias/local" => LocalAliasBucket (query::id, query::access_key_id, query::alias),
+			DELETE "/v0/bucket/alias/local" => LocalUnaliasBucket (query::id, query::access_key_id, query::alias),
+		]);
+
+		if let Some(message) = query.nonempty_message() {
+			debug!("Unused query parameter: {}", message)
+		}
+
+		Ok(res)
+	}
+}
+
+generateQueryParameters! {
+	keywords: [],
+	fields: [
+		"format" => format,
+		"id" => id,
+		"search" => search,
+		"globalAlias" => global_alias,
+		"alias" => alias,
+		"accessKeyId" => access_key_id
+	]
+}
@@ -0,0 +1,235 @@
+use std::borrow::Cow;
+
+use hyper::{Method, Request};
+
+use crate::admin::error::*;
+use crate::admin::router_v0;
+use crate::router_macros::*;
+
+pub enum Authorization {
+	None,
+	MetricsToken,
+	AdminToken,
+}
+
+router_match! {@func
+
+/// List of all Admin API endpoints.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum Endpoint {
+	Options,
+	CheckDomain,
+	Health,
+	Metrics,
+	GetClusterStatus,
+	GetClusterHealth,
+	ConnectClusterNodes,
+	// Layout
+	GetClusterLayout,
+	UpdateClusterLayout,
+	ApplyClusterLayout,
+	RevertClusterLayout,
+	// Keys
+	ListKeys,
+	CreateKey,
+	ImportKey,
+	GetKeyInfo {
+		id: Option<String>,
+		search: Option<String>,
+		show_secret_key: Option<String>,
+	},
+	DeleteKey {
+		id: String,
+	},
+	UpdateKey {
+		id: String,
+	},
+	// Buckets
+	ListBuckets,
+	CreateBucket,
+	GetBucketInfo {
+		id: Option<String>,
+		global_alias: Option<String>,
+	},
+	DeleteBucket {
+		id: String,
+	},
+	UpdateBucket {
+		id: String,
+	},
+	// Bucket-Key Permissions
+	BucketAllowKey,
+	BucketDenyKey,
+	// Bucket aliases
+	GlobalAliasBucket {
+		id: String,
+		alias: String,
+	},
+	GlobalUnaliasBucket {
+		id: String,
+		alias: String,
+	},
+	LocalAliasBucket {
+		id: String,
+		access_key_id: String,
+		alias: String,
+	},
+	LocalUnaliasBucket {
+		id: String,
+		access_key_id: String,
+		alias: String,
+	},
+}}
+
+impl Endpoint {
+	/// Determine which S3 endpoint a request is for using the request, and a bucket which was
+	/// possibly extracted from the Host header.
+	/// Returns Self plus bucket name, if endpoint is not Endpoint::ListBuckets
+	pub fn from_request<T>(req: &Request<T>) -> Result<Self, Error> {
+		let uri = req.uri();
+		let path = uri.path();
+		let query = uri.query();
+
+		let mut query = QueryParameters::from_query(query.unwrap_or_default())?;
+
+		let res = router_match!(@gen_path_parser (req.method(), path, query) [
+			OPTIONS _ => Options,
+			GET "/check" => CheckDomain,
+			GET "/health" => Health,
+			GET "/metrics" => Metrics,
+			GET "/v1/status" => GetClusterStatus,
+			GET "/v1/health" => GetClusterHealth,
+			POST "/v1/connect" => ConnectClusterNodes,
+			// Layout endpoints
+			GET "/v1/layout" => GetClusterLayout,
+			POST "/v1/layout" => UpdateClusterLayout,
+			POST "/v1/layout/apply" => ApplyClusterLayout,
+			POST "/v1/layout/revert" => RevertClusterLayout,
+			// API key endpoints
+			GET "/v1/key" if id => GetKeyInfo (query_opt::id, query_opt::search, query_opt::show_secret_key),
+			GET "/v1/key" if search => GetKeyInfo (query_opt::id, query_opt::search, query_opt::show_secret_key),
+			POST "/v1/key" if id => UpdateKey (query::id),
+			POST "/v1/key" => CreateKey,
+			POST "/v1/key/import" => ImportKey,
+			DELETE "/v1/key" if id => DeleteKey (query::id),
+			GET "/v1/key" => ListKeys,
+			// Bucket endpoints
+			GET "/v1/bucket" if id => GetBucketInfo (query_opt::id, query_opt::global_alias),
+			GET "/v1/bucket" if global_alias => GetBucketInfo (query_opt::id, query_opt::global_alias),
+			GET "/v1/bucket" => ListBuckets,
+			POST "/v1/bucket" => CreateBucket,
+			DELETE "/v1/bucket" if id => DeleteBucket (query::id),
+			PUT "/v1/bucket" if id => UpdateBucket (query::id),
+			// Bucket-key permissions
+			POST "/v1/bucket/allow" => BucketAllowKey,
+			POST "/v1/bucket/deny" => BucketDenyKey,
+			// Bucket aliases
+			PUT "/v1/bucket/alias/global" => GlobalAliasBucket (query::id, query::alias),
+			DELETE "/v1/bucket/alias/global" => GlobalUnaliasBucket (query::id, query::alias),
+			PUT "/v1/bucket/alias/local" => LocalAliasBucket (query::id, query::access_key_id, query::alias),
+			DELETE "/v1/bucket/alias/local" => LocalUnaliasBucket (query::id, query::access_key_id, query::alias),
+		]);
+
+		if let Some(message) = query.nonempty_message() {
+			debug!("Unused query parameter: {}", message)
+		}
+
+		Ok(res)
+	}
+	/// Some endpoints work exactly the same in their v1/ version as they did in their v0/ version.
+	/// For these endpoints, we can convert a v0/ call to its equivalent as if it was made using
+	/// its v1/ URL.
+	pub fn from_v0(v0_endpoint: router_v0::Endpoint) -> Result<Self, Error> {
+		match v0_endpoint {
+			// Cluster endpoints
+			router_v0::Endpoint::ConnectClusterNodes => Ok(Self::ConnectClusterNodes),
+			// - GetClusterStatus: response format changed
+			// - GetClusterHealth: response format changed
+
+			// Layout endpoints
+			router_v0::Endpoint::RevertClusterLayout => Ok(Self::RevertClusterLayout),
+			// - GetClusterLayout: response format changed
+			// - UpdateClusterLayout: query format changed
+			// - ApplyCusterLayout: response format changed
+
+			// Key endpoints
+			router_v0::Endpoint::ListKeys => Ok(Self::ListKeys),
+			router_v0::Endpoint::CreateKey => Ok(Self::CreateKey),
+			router_v0::Endpoint::GetKeyInfo { id, search } => Ok(Self::GetKeyInfo {
+				id,
+				search,
+				show_secret_key: Some("true".into()),
+			}),
+			router_v0::Endpoint::DeleteKey { id } => Ok(Self::DeleteKey { id }),
+			// - UpdateKey: response format changed (secret key no longer returned)
+
+			// Bucket endpoints
+			router_v0::Endpoint::GetBucketInfo { id, global_alias } => {
+				Ok(Self::GetBucketInfo { id, global_alias })
+			}
+			router_v0::Endpoint::ListBuckets => Ok(Self::ListBuckets),
+			router_v0::Endpoint::CreateBucket => Ok(Self::CreateBucket),
+			router_v0::Endpoint::DeleteBucket { id } => Ok(Self::DeleteBucket { id }),
+			router_v0::Endpoint::UpdateBucket { id } => Ok(Self::UpdateBucket { id }),
+
+			// Bucket-key permissions
+			router_v0::Endpoint::BucketAllowKey => Ok(Self::BucketAllowKey),
+			router_v0::Endpoint::BucketDenyKey => Ok(Self::BucketDenyKey),
+
+			// Bucket alias endpoints
+			router_v0::Endpoint::GlobalAliasBucket { id, alias } => {
+				Ok(Self::GlobalAliasBucket { id, alias })
+			}
+			router_v0::Endpoint::GlobalUnaliasBucket { id, alias } => {
+				Ok(Self::GlobalUnaliasBucket { id, alias })
+			}
+			router_v0::Endpoint::LocalAliasBucket {
+				id,
+				access_key_id,
+				alias,
+			} => Ok(Self::LocalAliasBucket {
+				id,
+				access_key_id,
+				alias,
+			}),
+			router_v0::Endpoint::LocalUnaliasBucket {
+				id,
+				access_key_id,
+				alias,
+			} => Ok(Self::LocalUnaliasBucket {
+				id,
+				access_key_id,
+				alias,
+			}),
+
+			// For endpoints that have different body content syntax, issue
+			// deprecation warning
+			_ => Err(Error::bad_request(format!(
+				"v0/ endpoint is no longer supported: {}",
+				v0_endpoint.name()
+			))),
+		}
+	}
+	/// Get the kind of authorization which is required to perform the operation.
+	pub fn authorization_type(&self) -> Authorization {
+		match self {
+			Self::Health => Authorization::None,
+			Self::CheckDomain => Authorization::None,
+			Self::Metrics => Authorization::MetricsToken,
+			_ => Authorization::AdminToken,
+		}
+	}
+}
+
+generateQueryParameters! {
+	keywords: [],
+	fields: [
+		"format" => format,
+		"id" => id,
+		"search" => search,
+		"globalAlias" => global_alias,
+		"alias" => alias,
+		"accessKeyId" => access_key_id,
+		"showSecretKey" => show_secret_key
+	]
+}
@@ -1,4 +1,5 @@
-use std::net::SocketAddr;
+use std::fs::{self, Permissions};
+use std::os::unix::fs::PermissionsExt;
 use std::sync::Arc;

 use async_trait::async_trait;
@@ -11,6 +12,10 @@ use hyper::service::{make_service_fn, service_fn};
 use hyper::{Body, Request, Response, Server};
 use hyper::{HeaderMap, StatusCode};

+use hyperlocal::UnixServerExt;
+
+use tokio::net::UnixStream;
+
 use opentelemetry::{
 	global,
 	metrics::{Counter, ValueRecorder},
@@ -21,6 +26,7 @@ use opentelemetry::{
 use garage_util::error::Error as GarageError;
 use garage_util::forwarded_headers;
 use garage_util::metrics::{gen_trace_id, RecordDuration};
+use garage_util::socket_address::UnixOrTCPSocketAddress;

 pub(crate) trait ApiEndpoint: Send + Sync + 'static {
 	fn name(&self) -> &'static str;
@@ -91,10 +97,11 @@ impl<A: ApiHandler> ApiServer<A> {

 	pub async fn run_server(
 		self: Arc<Self>,
-		bind_addr: SocketAddr,
+		bind_addr: UnixOrTCPSocketAddress,
+		unix_bind_addr_mode: Option<u32>,
 		shutdown_signal: impl Future<Output = ()>,
 	) -> Result<(), GarageError> {
-		let service = make_service_fn(|conn: &AddrStream| {
+		let tcp_service = make_service_fn(|conn: &AddrStream| {
 			let this = self.clone();

 			let client_addr = conn.remote_addr();
@@ -102,28 +109,63 @@ impl<A: ApiHandler> ApiServer<A> {
 				Ok::<_, GarageError>(service_fn(move |req: Request<Body>| {
 					let this = this.clone();

-					this.handler(req, client_addr)
+					this.handler(req, client_addr.to_string())
 				}))
 			}
 		});

-		let server = Server::bind(&bind_addr).serve(service);
+		let unix_service = make_service_fn(|_: &UnixStream| {
+			let this = self.clone();
+
+			let path = bind_addr.to_string();
+			async move {
+				Ok::<_, GarageError>(service_fn(move |req: Request<Body>| {
+					let this = this.clone();
+
+					this.handler(req, path.clone())
+				}))
+			}
+		});

-		let graceful = server.with_graceful_shutdown(shutdown_signal);
 		info!(
-			"{} API server listening on http://{}",
+			"{} API server listening on {}",
 			A::API_NAME_DISPLAY,
 			bind_addr
 		);

-		graceful.await?;
+		match bind_addr {
+			UnixOrTCPSocketAddress::TCPSocket(addr) => {
+				Server::bind(&addr)
+					.serve(tcp_service)
+					.with_graceful_shutdown(shutdown_signal)
+					.await?
+			}
+			UnixOrTCPSocketAddress::UnixSocket(ref path) => {
+				if path.exists() {
+					fs::remove_file(path)?
+				}
+
+				let bound = Server::bind_unix(path)?;
+
+				fs::set_permissions(
+					path,
+					Permissions::from_mode(unix_bind_addr_mode.unwrap_or(0o222)),
+				)?;
+
+				bound
+					.serve(unix_service)
+					.with_graceful_shutdown(shutdown_signal)
+					.await?;
+			}
+		};
+
 		Ok(())
 	}

 	async fn handler(
 		self: Arc<Self>,
 		req: Request<Body>,
-		addr: SocketAddr,
+		addr: String,
 	) -> Result<Response<Body>, GarageError> {
 		let uri = req.uri().clone();

@@ -1,4 +1,3 @@
-use std::net::SocketAddr;
 use std::sync::Arc;

 use async_trait::async_trait;
@@ -9,6 +8,7 @@ use hyper::{Body, Method, Request, Response};
 use opentelemetry::{trace::SpanRef, KeyValue};

 use garage_util::error::Error as GarageError;
+use garage_util::socket_address::UnixOrTCPSocketAddress;

 use garage_model::garage::Garage;

@@ -37,12 +37,12 @@ pub(crate) struct K2VApiEndpoint {
 impl K2VApiServer {
 	pub async fn run(
 		garage: Arc<Garage>,
-		bind_addr: SocketAddr,
+		bind_addr: UnixOrTCPSocketAddress,
 		s3_region: String,
 		shutdown_signal: impl Future<Output = ()>,
 	) -> Result<(), GarageError> {
 		ApiServer::new(s3_region, K2VApiServer { garage })
-			.run_server(bind_addr, shutdown_signal)
+			.run_server(bind_addr, None, shutdown_signal)
 			.await
 	}
 }
@@ -1,4 +1,3 @@
-use std::net::SocketAddr;
 use std::sync::Arc;

 use async_trait::async_trait;
@@ -10,6 +9,7 @@ use hyper::{Body, Request, Response};
 use opentelemetry::{trace::SpanRef, KeyValue};

 use garage_util::error::Error as GarageError;
+use garage_util::socket_address::UnixOrTCPSocketAddress;

 use garage_model::garage::Garage;
 use garage_model::key_table::Key;
@@ -46,12 +46,12 @@ pub(crate) struct S3ApiEndpoint {
 impl S3ApiServer {
 	pub async fn run(
 		garage: Arc<Garage>,
-		addr: SocketAddr,
+		addr: UnixOrTCPSocketAddress,
 		s3_region: String,
 		shutdown_signal: impl Future<Output = ()>,
 	) -> Result<(), GarageError> {
 		ApiServer::new(s3_region, S3ApiServer { garage })
-			.run_server(addr, shutdown_signal)
+			.run_server(addr, None, shutdown_signal)
 			.await
 	}

@@ -453,7 +453,7 @@ pub enum RepairWhat {
 	/// Do a full sync of metadata tables
 	#[structopt(name = "tables", version = garage_version())]
 	Tables,
-	/// Repair (resync/rebalance) the set of stored blocks
+	/// Repair (resync/rebalance) the set of stored blocks in the cluster
 	#[structopt(name = "blocks", version = garage_version())]
 	Blocks,
 	/// Repropagate object deletions to the version table
@@ -471,7 +471,7 @@ pub enum RepairWhat {
 		#[structopt(subcommand)]
 		cmd: ScrubCmd,
 	},
-	/// Rebalance data blocks among storage locations
+	/// Rebalance data blocks among HDDs on individual nodes
 	#[structopt(name = "rebalance", version = garage_version())]
 	Rebalance,
 }
@@ -79,7 +79,7 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er
 			"S3 API",
 			tokio::spawn(S3ApiServer::run(
 				garage.clone(),
-				*s3_bind_addr,
+				s3_bind_addr.clone(),
 				config.s3_api.s3_region.clone(),
 				wait_from(watch_cancel.clone()),
 			)),
@@ -94,7 +94,7 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er
 				"K2V API",
 				tokio::spawn(K2VApiServer::run(
 					garage.clone(),
-					config.k2v_api.as_ref().unwrap().api_bind_addr,
+					config.k2v_api.as_ref().unwrap().api_bind_addr.clone(),
 					config.s3_api.s3_region.clone(),
 					wait_from(watch_cancel.clone()),
 				)),
@@ -110,7 +110,7 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er
 			"Web",
 			tokio::spawn(WebServer::run(
 				garage.clone(),
-				web_config.bind_addr,
+				web_config.bind_addr.clone(),
 				web_config.root_domain.clone(),
 				wait_from(watch_cancel.clone()),
 			)),
@@ -121,7 +121,9 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er
 		info!("Launching Admin API server...");
 		servers.push((
 			"Admin",
-			tokio::spawn(admin_server.run(*admin_bind_addr, wait_from(watch_cancel.clone()))),
+			tokio::spawn(
+				admin_server.run(admin_bind_addr.clone(), wait_from(watch_cancel.clone())),
+			),
 		));
 	}

@@ -7,6 +7,7 @@ use std::path::PathBuf;
 use serde::{de, Deserialize};

 use crate::error::Error;
+use crate::socket_address::UnixOrTCPSocketAddress;

 /// Represent the whole configuration
 #[derive(Deserialize, Debug, Clone)]
@@ -129,7 +130,7 @@ pub struct DataDir {
 #[derive(Deserialize, Debug, Clone)]
 pub struct S3ApiConfig {
 	/// Address and port to bind for api serving
-	pub api_bind_addr: Option<SocketAddr>,
+	pub api_bind_addr: Option<UnixOrTCPSocketAddress>,
 	/// S3 region to use
 	pub s3_region: String,
 	/// Suffix to remove from domain name to find bucket. If None,
@@ -141,14 +142,14 @@ pub struct S3ApiConfig {
 #[derive(Deserialize, Debug, Clone)]
 pub struct K2VApiConfig {
 	/// Address and port to bind for api serving
-	pub api_bind_addr: SocketAddr,
+	pub api_bind_addr: UnixOrTCPSocketAddress,
 }

 /// Configuration for serving files as normal web server
 #[derive(Deserialize, Debug, Clone)]
 pub struct WebConfig {
 	/// Address and port to bind for web serving
-	pub bind_addr: SocketAddr,
+	pub bind_addr: UnixOrTCPSocketAddress,
 	/// Suffix to remove from domain name to find bucket
 	pub root_domain: String,
 }
@@ -157,7 +158,7 @@ pub struct WebConfig {
 #[derive(Deserialize, Debug, Clone, Default)]
 pub struct AdminConfig {
 	/// Address and port to bind for admin API serving
-	pub api_bind_addr: Option<SocketAddr>,
+	pub api_bind_addr: Option<UnixOrTCPSocketAddress>,

 	/// Bearer token to use to scrape metrics
 	pub metrics_token: Option<String>,
@@ -14,6 +14,7 @@ pub mod forwarded_headers;
 pub mod metrics;
 pub mod migrate;
 pub mod persister;
+pub mod socket_address;
 pub mod time;
 pub mod tranquilizer;
 pub mod version;
@@ -0,0 +1,44 @@
+use std::fmt::{Debug, Display, Formatter};
+use std::net::SocketAddr;
+use std::path::PathBuf;
+use std::str::FromStr;
+
+use serde::de::Error;
+use serde::{Deserialize, Deserializer};
+
+#[derive(Debug, Clone)]
+pub enum UnixOrTCPSocketAddress {
+	TCPSocket(SocketAddr),
+	UnixSocket(PathBuf),
+}
+
+impl Display for UnixOrTCPSocketAddress {
+	fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result {
+		match self {
+			UnixOrTCPSocketAddress::TCPSocket(address) => write!(formatter, "http://{}", address),
+			UnixOrTCPSocketAddress::UnixSocket(path) => {
+				write!(formatter, "http+unix://{}", path.to_string_lossy())
+			}
+		}
+	}
+}
+
+impl<'de> Deserialize<'de> for UnixOrTCPSocketAddress {
+	fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+	where
+		D: Deserializer<'de>,
+	{
+		let string = String::deserialize(deserializer)?;
+		let string = string.as_str();
+
+		if string.starts_with("/") {
+			Ok(UnixOrTCPSocketAddress::UnixSocket(
+				PathBuf::from_str(string).map_err(Error::custom)?,
+			))
+		} else {
+			Ok(UnixOrTCPSocketAddress::TCPSocket(
+				SocketAddr::from_str(string).map_err(Error::custom)?,
+			))
+		}
+	}
+}
@@ -27,5 +27,8 @@ futures = "0.3"

 http = "0.2"
 hyper = { version = "0.14", features = ["server", "http1", "runtime", "tcp", "stream"] }
+hyperlocal = { version = "0.8.0", default-features = false, features = ["server"] }
+
+tokio = { version = "1.0", default-features = false, features = ["net"] }

 opentelemetry = "0.17"
@@ -1,4 +1,6 @@
-use std::{convert::Infallible, net::SocketAddr, sync::Arc};
+use std::fs::{self, Permissions};
+use std::os::unix::prelude::PermissionsExt;
+use std::{convert::Infallible, sync::Arc};

 use futures::future::Future;

@@ -9,6 +11,10 @@ use hyper::{
 	Body, Method, Request, Response, Server, StatusCode,
 };

+use hyperlocal::UnixServerExt;
+
+use tokio::net::UnixStream;
+
 use opentelemetry::{
 	global,
 	metrics::{Counter, ValueRecorder},
@@ -32,6 +38,7 @@ use garage_util::data::Uuid;
 use garage_util::error::Error as GarageError;
 use garage_util::forwarded_headers;
 use garage_util::metrics::{gen_trace_id, RecordDuration};
+use garage_util::socket_address::UnixOrTCPSocketAddress;

 struct WebMetrics {
 	request_counter: Counter<u64>,
@@ -69,7 +76,7 @@ impl WebServer {
 	/// Run a web server
 	pub async fn run(
 		garage: Arc<Garage>,
-		addr: SocketAddr,
+		addr: UnixOrTCPSocketAddress,
 		root_domain: String,
 		shutdown_signal: impl Future<Output = ()>,
 	) -> Result<(), GarageError> {
@@ -80,7 +87,7 @@ impl WebServer {
 			root_domain,
 		});

-		let service = make_service_fn(|conn: &AddrStream| {
+		let tcp_service = make_service_fn(|conn: &AddrStream| {
 			let web_server = web_server.clone();

 			let client_addr = conn.remote_addr();
@@ -88,23 +95,56 @@ impl WebServer {
 				Ok::<_, Error>(service_fn(move |req: Request<Body>| {
 					let web_server = web_server.clone();

-					web_server.handle_request(req, client_addr)
+					web_server.handle_request(req, client_addr.to_string())
 				}))
 			}
 		});

-		let server = Server::bind(&addr).serve(service);
-		let graceful = server.with_graceful_shutdown(shutdown_signal);
-		info!("Web server listening on http://{}", addr);
+		let unix_service = make_service_fn(|_: &UnixStream| {
+			let web_server = web_server.clone();
+
+			let path = addr.to_string();
+			async move {
+				Ok::<_, Error>(service_fn(move |req: Request<Body>| {
+					let web_server = web_server.clone();
+
+					web_server.handle_request(req, path.clone())
+				}))
+			}
+		});
+
+		info!("Web server listening on {}", addr);
+
+		match addr {
+			UnixOrTCPSocketAddress::TCPSocket(addr) => {
+				Server::bind(&addr)
+					.serve(tcp_service)
+					.with_graceful_shutdown(shutdown_signal)
+					.await?
+			}
+			UnixOrTCPSocketAddress::UnixSocket(ref path) => {
+				if path.exists() {
+					fs::remove_file(path)?
+				}
+
+				let bound = Server::bind_unix(path)?;
+
+				fs::set_permissions(path, Permissions::from_mode(0o222))?;
+
+				bound
+					.serve(unix_service)
+					.with_graceful_shutdown(shutdown_signal)
+					.await?;
+			}
+		};

-		graceful.await?;
 		Ok(())
 	}

 	async fn handle_request(
 		self: Arc<Self>,
 		req: Request<Body>,
-		addr: SocketAddr,
+		addr: String,
 	) -> Result<Response<Body>, Infallible> {
 		if let Ok(forwarded_for_ip_addr) =
 			forwarded_headers::handle_forwarded_for_headers(req.headers())
Author	SHA1	Message	Date
Alex Auvolat	0c431b0c03	admin api: increased compatibility for v0/ endpoints	2023-10-05 16:56:13 +02:00
Alex Auvolat	1c13135f25	admin api: remove broken GET /v0/key router rule	2023-10-05 16:27:29 +02:00
Alex Auvolat	2448eb7713	upgrade doc: fixes and precisions	2023-10-05 15:29:55 +02:00
Alex Auvolat	6790e24f5a	Add migration to v0.9 guide	2023-10-05 15:20:48 +02:00
Alex Auvolat	9ccc1d6f4a	move upgrade test to release build	2023-10-05 10:42:10 +02:00
Alex Auvolat	920dec393a	cli: more precise doc comment	2023-10-04 10:44:42 +02:00
Alex Auvolat	2e656b541b	Merge branch 'main' into next	2023-10-03 18:40:37 +02:00
Alex	1243db87f2	Merge pull request 'Add support for binding to unix domain sockets' (#640 ) from networkException/garage:unix-sockets into main Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/640	2023-10-03 16:23:02 +00:00
networkException	6f8a87814b	doc: add documentation for specifying unix socket paths	2023-10-03 17:56:34 +02:00
networkException	7907a09acc	api: allow custom unix bind mode and use 0o220 for admin server	2023-10-03 17:31:40 +02:00
Alex	16aa418e47	Merge pull request 'doc: update endpoint_url documentation' (#641 ) from flokli/garage:aws-endpoint-url into main Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/641	2023-10-02 14:30:53 +00:00
Florian Klink	cb359b4434	doc: update endpoint_url documentation Since `awscli` `>=1.29.0` or `>=2.13.0` it is now possible to use the `AWS_ENDPOINT_URL` environment variable, or the `endpoint_url` config key to override the endpoint URL. This means, the aws bash function to wrap with --endpoint-url is not necessary anymore. Update invocations to reflect that. https://docs.aws.amazon.com/sdkref/latest/guide/feature-ss-endpoints.html https://github.com/aws/aws-cli/issues/4454#issuecomment-1626116607	2023-10-02 17:16:11 +03:00
networkException	8ec6a53b35	everywhere: support unix sockets when binding in various places this patch implements binding to paths as a unix socket for generic server and web server.	2023-09-29 18:57:44 +02:00
networkException	7353038a64	config: allow using paths for unix domain sockets in various places this patch updates the config format to also allow paths in bind addresses for unix domain sockets. this has been added to all apis except rpc.	2023-09-29 18:38:30 +02:00
networkException	10195f1567	util: add helper sum type for unix and tcp socket addresses this patch introduces a new sum type that can represent either a tcp socket address or a unix domain socket path.	2023-09-29 18:37:36 +02:00
networkException	6086a3fa07	cargo: add hyperlocal as a dependency	2023-09-29 18:37:12 +02:00
Alex Auvolat	9ac1d5be0e	add upgrade test for garage 0.8 -> 0.9	2023-09-27 14:57:37 +02:00