wip formal algorithm

.woodpecker/debug.yaml

@@ -12,32 +12,32 @@ when:
 
 steps:
   - name: check formatting
-    image: nixpkgs/nix:nixos-24.05
+    image: nixpkgs/nix:nixos-22.05
     commands:
-      - nix-build -j4 --attr flakePackages.fmt
+      - nix-shell --attr devShell --run "cargo fmt -- --check"
 
   - name: build
-    image: nixpkgs/nix:nixos-24.05
+    image: nixpkgs/nix:nixos-22.05
     commands:
       - nix-build -j4 --attr flakePackages.dev
 
   - name: unit + func tests (lmdb)
-    image: nixpkgs/nix:nixos-24.05
+    image: nixpkgs/nix:nixos-22.05
     commands:
       - nix-build -j4 --attr flakePackages.tests-lmdb
 
   - name: unit + func tests (sqlite)
-    image: nixpkgs/nix:nixos-24.05
+    image: nixpkgs/nix:nixos-22.05
     commands:
       - nix-build -j4 --attr flakePackages.tests-sqlite
 
   - name: unit + func tests (fjall)
-    image: nixpkgs/nix:nixos-24.05
+    image: nixpkgs/nix:nixos-22.05
     commands:
       - nix-build -j4 --attr flakePackages.tests-fjall
 
   - name: integration tests
-    image: nixpkgs/nix:nixos-24.05
+    image: nixpkgs/nix:nixos-22.05
     commands:
       - nix-build -j4 --attr flakePackages.dev
       - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)

.woodpecker/publish.yaml

@@ -11,7 +11,7 @@ depends_on:
 
 steps:
   - name: refresh-index
-    image: nixpkgs/nix:nixos-24.05
+    image: nixpkgs/nix:nixos-22.05
     environment:
       AWS_ACCESS_KEY_ID:
         from_secret: garagehq_aws_access_key_id
@@ -22,7 +22,7 @@ steps:
       - nix-shell --attr ci --run "refresh_index"
 
   - name: multiarch-docker
-    image: nixpkgs/nix:nixos-24.05
+    image: nixpkgs/nix:nixos-22.05
     environment:
       DOCKER_AUTH:
         from_secret: docker_auth

.woodpecker/release.yaml

@@ -19,17 +19,17 @@ matrix:
 
 steps:
   - name: build
-    image: nixpkgs/nix:nixos-24.05
+    image: nixpkgs/nix:nixos-22.05
     commands:
       - nix-build --attr releasePackages.${ARCH} --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
 
   - name: check is static binary
-    image: nixpkgs/nix:nixos-24.05
+    image: nixpkgs/nix:nixos-22.05
     commands:
       - nix-shell --attr ci --run "./script/not-dynamic.sh result/bin/garage"
 
   - name: integration tests
-    image: nixpkgs/nix:nixos-24.05
+    image: nixpkgs/nix:nixos-22.05
     commands:
       - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
     when:
@@ -39,7 +39,7 @@ steps:
           ARCH: i386
 
   - name: upgrade tests from v1.0.0
-    image: nixpkgs/nix:nixos-24.05
+    image: nixpkgs/nix:nixos-22.05
     commands:
       - nix-shell --attr ci --run "./script/test-upgrade.sh v1.0.0 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
     when:
@@ -47,7 +47,7 @@ steps:
           ARCH: amd64
 
   - name: upgrade tests from v0.8.4
-    image: nixpkgs/nix:nixos-24.05
+    image: nixpkgs/nix:nixos-22.05
     commands:
       - nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
     when:
@@ -55,7 +55,7 @@ steps:
           ARCH: amd64
 
   - name: push static binary
-    image: nixpkgs/nix:nixos-24.05
+    image: nixpkgs/nix:nixos-22.05
     environment:
       TARGET: "${TARGET}"
       AWS_ACCESS_KEY_ID:
@@ -66,7 +66,7 @@ steps:
       - nix-shell --attr ci --run "to_s3"
 
   - name: docker build and publish
-    image: nixpkgs/nix:nixos-24.05
+    image: nixpkgs/nix:nixos-22.05
     environment:
       DOCKER_PLATFORM: "linux/${ARCH}"
       CONTAINER_NAME: "dxflrs/${ARCH}_garage"

Cargo.toml

@@ -24,18 +24,18 @@ default-members = ["src/garage"]
 
 # Internal Garage crates
 format_table = { version = "0.1.1", path = "src/format-table" }
-garage_api_common = { version = "2.2.0", path = "src/api/common" }
+garage_api_common = { version = "2.1.0", path = "src/api/common" }
-garage_api_admin = { version = "2.2.0", path = "src/api/admin" }
+garage_api_admin = { version = "2.1.0", path = "src/api/admin" }
-garage_api_s3 = { version = "2.2.0", path = "src/api/s3" }
+garage_api_s3 = { version = "2.1.0", path = "src/api/s3" }
-garage_api_k2v = { version = "2.2.0", path = "src/api/k2v" }
+garage_api_k2v = { version = "2.1.0", path = "src/api/k2v" }
-garage_block = { version = "2.2.0", path = "src/block" }
+garage_block = { version = "2.1.0", path = "src/block" }
-garage_db = { version = "2.2.0", path = "src/db", default-features = false }
+garage_db = { version = "2.1.0", path = "src/db", default-features = false }
-garage_model = { version = "2.2.0", path = "src/model", default-features = false }
+garage_model = { version = "2.1.0", path = "src/model", default-features = false }
-garage_net = { version = "2.2.0", path = "src/net" }
+garage_net = { version = "2.1.0", path = "src/net" }
-garage_rpc = { version = "2.2.0", path = "src/rpc" }
+garage_rpc = { version = "2.1.0", path = "src/rpc" }
-garage_table = { version = "2.2.0", path = "src/table" }
+garage_table = { version = "2.1.0", path = "src/table" }
-garage_util = { version = "2.2.0", path = "src/util" }
+garage_util = { version = "2.1.0", path = "src/util" }
-garage_web = { version = "2.2.0", path = "src/web" }
+garage_web = { version = "2.1.0", path = "src/web" }
 k2v-client = { version = "0.0.4", path = "src/k2v-client" }
 
 # External crates from crates.io

doc/api/garage-admin-v2.json

@@ -12,7 +12,7 @@
       "name": "AGPL-3.0",
       "identifier": "AGPL-3.0"
     },
-    "version": "v2.2.0"
+    "version": "v2.1.0"
   },
   "servers": [
     {
@@ -103,7 +103,7 @@
           "content": {
             "application/json": {
               "schema": {
-                "$ref": "#/components/schemas/BucketAliasEnum"
+                "$ref": "#/components/schemas/AddBucketAliasRequest"
               }
             }
           },
@@ -1409,7 +1409,7 @@
           "content": {
             "application/json": {
               "schema": {
-                "$ref": "#/components/schemas/BucketAliasEnum"
+                "$ref": "#/components/schemas/RemoveBucketAliasRequest"
               }
             }
           },
@@ -1722,6 +1722,24 @@
   },
   "components": {
     "schemas": {
+      "AddBucketAliasRequest": {
+        "allOf": [
+          {
+            "$ref": "#/components/schemas/BucketAliasEnum"
+          },
+          {
+            "type": "object",
+            "required": [
+              "bucketId"
+            ],
+            "properties": {
+              "bucketId": {
+                "type": "string"
+              }
+            }
+          }
+        ]
+      },
       "AddBucketAliasResponse": {
         "$ref": "#/components/schemas/GetBucketInfoResponse"
       },
@@ -1939,13 +1957,9 @@
           {
             "type": "object",
             "required": [
-              "bucketId",
               "globalAlias"
             ],
             "properties": {
-              "bucketId": {
-                "type": "string"
-              },
               "globalAlias": {
                 "type": "string"
               }
@@ -1954,7 +1968,6 @@
           {
             "type": "object",
             "required": [
-              "bucketId",
               "localAlias",
               "accessKeyId"
             ],
@@ -1962,9 +1975,6 @@
               "accessKeyId": {
                 "type": "string"
               },
-              "bucketId": {
-                "type": "string"
-              },
               "localAlias": {
                 "type": "string"
               }
@@ -3902,49 +3912,6 @@
           }
         ]
       },
-      "NodeRoleChangeRequest": {
-        "oneOf": [
-          {
-            "type": "object",
-            "required": [
-              "id",
-              "remove"
-            ],
-            "properties": {
-              "id": {
-                "type": "string",
-                "description": "ID of the node for which this change applies"
-              },
-              "remove": {
-                "type": "boolean",
-                "description": "Set `remove` to `true` to remove the node from the layout"
-              }
-            }
-          },
-          {
-            "$ref": "#/components/schemas/NodeRoleUpdate"
-          }
-        ]
-      },
-      "NodeRoleUpdate": {
-        "allOf": [
-          {
-            "$ref": "#/components/schemas/NodeAssignedRole"
-          },
-          {
-            "type": "object",
-            "required": [
-              "id"
-            ],
-            "properties": {
-              "id": {
-                "type": "string",
-                "description": "ID of the node for which this change applies"
-              }
-            }
-          }
-        ]
-      },
       "NodeUpdateTrackers": {
         "type": "object",
         "required": [
@@ -4006,6 +3973,24 @@
           }
         ]
       },
+      "RemoveBucketAliasRequest": {
+        "allOf": [
+          {
+            "$ref": "#/components/schemas/BucketAliasEnum"
+          },
+          {
+            "type": "object",
+            "required": [
+              "bucketId"
+            ],
+            "properties": {
+              "bucketId": {
+                "type": "string"
+              }
+            }
+          }
+        ]
+      },
       "RemoveBucketAliasResponse": {
         "$ref": "#/components/schemas/GetBucketInfoResponse"
       },
@@ -4195,7 +4180,7 @@
           "roles": {
             "type": "array",
             "items": {
-              "$ref": "#/components/schemas/NodeRoleChangeRequest"
+              "$ref": "#/components/schemas/NodeRoleChange"
             },
             "description": "New node roles to assign or remove in the cluster layout"
           }
@@ -4353,39 +4338,43 @@
       "WorkerStateResp": {
         "oneOf": [
           {
-            "$ref": "#/components/schemas/Yolo"
+            "type": "string",
-	  },
+            "enum": [
+              "busy"
+            ]
+          },
           {
-            "$ref": "#/components/schemas/WorkerStateRespStrs"
-	  }
-        ]
-      },
-      "Yolo": {
-        "type": "object",
-        "required": [
-          "throttled"
-        ],
-        "properties": {
-          "throttled": {
             "type": "object",
             "required": [
-              "durationSecs"
+              "throttled"
             ],
             "properties": {
-              "durationSecs": {
+              "throttled": {
-                "type": "number",
+                "type": "object",
-                "format": "float"
+                "required": [
+                  "durationSecs"
+                ],
+                "properties": {
+                  "durationSecs": {
+                    "type": "number",
+                    "format": "float"
+                  }
+                }
               }
             }
+          },
+          {
+            "type": "string",
+            "enum": [
+              "idle"
+            ]
+          },
+          {
+            "type": "string",
+            "enum": [
+              "done"
+            ]
           }
-        }
-      },
-      "WorkerStateRespStrs": {
-        "type": "string",
-        "enum": [
-          "busy",
-          "idle",
-          "done"
         ]
       },
       "ZoneRedundancy": {

doc/book/connect/apps/index.md

@@ -12,9 +12,8 @@ In this section, we cover the following web applications:
 | [Mastodon](#mastodon)     | ✅       | Natively supported    |
 | [Matrix](#matrix)     | ✅       |  Tested with `synapse-s3-storage-provider`    |
 | [ejabberd](#ejabberd)     | ✅       |  `mod_s3_upload`    |
-| [Ente](#ente)     | ✅       |  Natively supported |
+| [Pixelfed](#pixelfed)     | ✅       |  Natively supported    |
-| [Pixelfed](#pixelfed)     | ❓       |  Natively supported    |
+| [Pleroma](#pleroma)     | ❓       |  Not yet tested    |
-| [Pleroma](#pleroma)     | ✅        |  Natively supported    |
 | [Lemmy](#lemmy)     | ✅        |  Supported with pict-rs    |
 | [Funkwhale](#funkwhale)     | ❓       | Not yet tested     |
 | [Misskey](#misskey)     | ❓       | Not yet tested     |
@@ -568,186 +567,13 @@ The module can then be configured with:
 Other configuration options can be found in the
 [configuration YAML file](https://github.com/processone/ejabberd-contrib/blob/master/mod_s3_upload/conf/mod_s3_upload.yml).
 
-
-## Ente
-
-Ente is an alternative for Google Photos and Apple Photos. It [can be selfhosted](https://help.ente.io/self-hosting/) and is working fine with Garage as of May 2024.
-As a first step we need to create a bucket and a key for Ente:
-
-```bash
-garage bucket create ente
-garage key create ente-key
-# For the CORS setup to work, the key needs to be --owner as well, at least temporarily.
-garage bucket allow ente --read --write --owner --key ente-key
-```
-
-We also need to setup some CORS rules to allow the Ente frontend to access the bucket:
-
-```bash
-export CORS='{"CORSRules":[{"AllowedHeaders":["*"],"AllowedMethods":["GET", "PUT", "POST", "DELETE"],"AllowedOrigins":["*"], "ExposeHeaders":["ETag"]}]}'
-aws s3api put-bucket-cors --bucket ente --cors-configuration $CORS
-```
-
-Now we need to configure ente-server to use our bucket. This is explained [in the Ente S3 documentation](https://help.ente.io/self-hosting/guides/external-s3).
-Prepare a configuration file for ente's backend as `museum.yaml`:
-
-```yaml
-credentials-file: /credentials.yaml
-apps:
-  public-albums: https://albums.example.tld # If you want to use the share album feature
-    internal:
-hardcoded-ott:
-  local-domain-suffix: "@example.com" # Your domain
-  local-domain-value: 123456 # Custom One-Time Password since we are not sending mail by default
-key:
-  # WARNING -- You MUST CHANGE the values below
-  # Someone has made an image that can do it for you : https://github.com/EdyTheCow/ente-selfhost/blob/main/images/ente-server-tools/Dockerfile
-  # Simply build it yourself or run docker run --rm ghcr.io/edythecow/ente-server-tools go run tools/gen-random-keys/main.go
-  encryption: yvmG/RnzKrbCb9L3mgsmoxXr9H7i2Z4qlbT0mL3ln4w= # CHANGE THIS VALUE
-  hash: KXYiG07wC7GIgvCSdg+WmyWdXDAn6XKYJtp/wkEU7x573+byBRAYtpTP0wwvi8i/4l37uicX1dVTUzwH3sLZyw== # CHANGE THIS VALUE
-jwt:
-  secret: i2DecQmfGreG6q1vBj5tCokhlN41gcfS2cjOs9Po-u8= # CHANGE THIS VALUE
-```
-
-The full configuration file can be found [here](https://github.com/ente-io/ente/blob/main/server/configurations/local.yaml)
-Then prepare a credentials file as `credentials.yaml`
-
-```yaml
-db:
-    host: postgres
-    port: 5432
-    name: <ente_db_name>
-    user: <pguser>
-    password: <pgpass>
-
-s3:
-    # Override the primary and secondary hot storage. The commented out values
-    # are the defaults.
-    #
-    hot_storage:
-        primary: b2-eu-cen
-    #    secondary: wasabi-eu-central-2-v3
-
-    # If true, enable some workarounds to allow us to use a local minio instance
-    # for object storage.
-    #
-    # 1. Disable SSL.
-    # 2. Use "path" style S3 URLs (see `use_path_style_urls` below).
-    # 3. Directly download the file during replication instead of going via the
-    #    Cloudflare worker.
-    # 4. Do not specify storage classes when uploading objects (since minio does
-    #    not support them, specifically it doesn't support GLACIER).
-    are_local_buckets: true
-
-    # To use "path" style S3 URLs instead of DNS-based bucket access
-    # default to true if you set "are_local_buckets: true"
-    # use_path_style_urls: true
-
-    b2-eu-cen: # Don't change this key, it is hardcoded
-        key: <keyID> 
-        secret: <keySecret>
-        endpoint: garage:3900 # publically accessible endpoint of your garage instance
-        region: garage
-        bucket: <yourbucketName>
-        use_path_style: true
-    # you can specify secondary locations, names are hardcoded as well
-    # wasabi-eu-central-2-v3:
-    # scw-eu-fr-v3:
-    
-    # and you can also specify a bucket to be used for embeddings, preview etc..
-    # default to the first bucket 
-    # derived-storage: wasabi-eu-central-2-derived
-```
-
-Finally you can run it with Docker : 
-
-```bash
-docker run -d --name ente-server --restart unless-stopped -v /path/to/museum.yaml:/museum.yaml -v /path/to/credentials.yaml:/credentials.yaml -p 8080:8080 ghcr.io/ente-io/ente-server
-```
-
-For more information on deployment you can check the [ente documentation](https://help.ente.io/self-hosting/)
-
 ## Pixelfed
 
 [Pixelfed Technical Documentation > Configuration](https://docs.pixelfed.org/technical-documentation/env.html#filesystem)
 
 ## Pleroma
 
-### Creating your bucket
+[Pleroma Documentation > Pleroma.Uploaders.S3](https://docs-develop.pleroma.social/backend/configuration/cheatsheet/#pleromauploaderss3)
-
-This is the usual Garage setup:
-
-```bash
-garage key new --name pleroma-key
-garage bucket create pleroma
-garage bucket allow pleroma --read --write --owner --key pleroma-key
-```
-
-We also need to expose these buckets publicly to serve their content to users:
-
-```bash
-garage bucket website --allow pleroma
-```
-
-Note the Key ID and Secret Key.
-
-### Configure Pleroma
-
-Update your Pleroma configuration like that in `/etc/pleroma/config.exs`.
-
-```
-config :pleroma, Pleroma.Upload,
-    uploader: Pleroma.Uploaders.S3,
-    base_url: "https://pleroma.garage.example.tld"
-
-config :ex_aws, :s3,
-    access_key_id: "GW...",
-    secret_access_key: "XXX",
-    region: "garage",
-    host: "api.garage.example.tld"
-```
-
-And restart Pleroma.
-
-You can found more information in [Pleroma Documentation > Pleroma.Uploaders.S3](https://docs-develop.pleroma.social/backend/configuration/cheatsheet/#pleromauploaderss3)
-
-### Migrating your data
-
-Pleroma have an internal migration tool that can encounter some fatal error
-
-```
-** (EXIT from #PID<0.98.0>) an exception was raised:
-    ** (File.Error) could not stream "/var/lib/pleroma/uploads/09/f8": illegal operation on a directory
-        (elixir 1.17.3) lib/file/stream.ex:100: anonymous fn/3 in Enumerable.File.Stream.reduce/3
-        (elixir 1.17.3) lib/stream.ex:1675: anonymous fn/5 in Stream.resource/3
-        (elixir 1.17.3) lib/stream.ex:1891: Enumerable.Stream.do_each/4
-        (elixir 1.17.3) lib/task/supervised.ex:370: Task.Supervised.stream_reduce/7
-        (elixir 1.17.3) lib/enum.ex:4423: Enum.map/2
-        (ex_aws_s3 2.5.8) lib/ex_aws/s3/upload.ex:141: ExAws.Operation.ExAws.S3.Upload.perform/2
-        (pleroma 2.10.0) lib/pleroma/uploaders/s3.ex:60: Pleroma.Uploaders.S3.put_file/1
-        (pleroma 2.10.0) lib/pleroma/uploaders/uploader.ex:49: Pleroma.Uploaders.Uploader.put_file/2
-```
-
-So, use [your best tool](https://garagehq.deuxfleurs.fr/documentation/connect/cli/) to sync `/var/lib/pleroma/uploads/` in your S3.
-
-Then, to avoid some non existant problem (just in case of), run this command
-
-```bash
-while true
-do
-    rm -vr $(./bin/pleroma_ctl uploads migrate_local S3 2>&1 | grep "could not stream" | awk -F '"' '{print $2}')
-    sleep 5
-done
-```
-
-If you have many files, stop this command sometime and the command bellow (interactive) to delete local
-file after upload. Then restart the loop.
-
-```bash
-./bin/pleroma_ctl uploads migrate_local S3 --delete
-```
-
-And *voilà*
 
 ## Lemmy
 

doc/book/connect/backup.md

@@ -207,13 +207,3 @@ $ plakar at @garageS3 ls
 ```
 
 More information in Plakar documentation: https://www.plakar.io/docs/main/quickstart/
-
-## Synology HyperBackup
-
-HyperBackup can be configured to upload backups to garage using a custom S3 destination. However, the HyperBackup client hardcodes the `us-east-1` region that is a critical input to the v4 signature process. If garage is not set to `us-east-1`, HyperBackup will recognize available buckets, but fail during the final setup stage.
-
-In garage.toml:
-```toml
-[s3_api]
-s3_region = "us-east-1"
-```

doc/book/connect/cli.md

@@ -149,15 +149,6 @@ rclone help
 This will tremendously accelerate operations such as `rclone sync` or `rclone ncdu` by reducing the number
 of ListObjects calls that are made.
 
-**Garage behind Cloudflare proxy:** when running Garage behind Cloudflare proxy, you might see `Response: error 403 Forbidden, Forbidden: Invalid signature` error in your garage logs or `AccessDenied: Forbidden: Invalid signature` error in rclone logs. Try adding `--s3-sign-accept-encoding=false` flag to your rclone command and see if the issue is resolved.
-
-```bash
-# this throws an error
-rclone lsd garage:
-
-# this should work
-rclone lsd --s3-sign-accept-encoding=false garage:
-```
 
 ## `s3cmd`
 
@@ -323,3 +314,4 @@ ls
 ```
 
 And through the web interface at http://[::1]:8080/web/client
+

doc/book/cookbook/binary-packages.md

@@ -15,10 +15,9 @@ Alpine Linux repositories (available since v3.17):
 apk add garage
 ```
 
-The default configuration file is installed to `/etc/garage/garage.toml`. You can run
+The default configuration file is installed to `/etc/garage.toml`. You can run
-Garage using: `rc-service garage start`.
+Garage using: `rc-service garage start`. If you don't specify `rpc_secret`, it
-
+will be automatically replaced with a random string on the first start.
-If you don't specify `rpc_secret`, it will be automatically replaced with a random string on the first start.
 
 Please note that this package is built without Consul discovery, Kubernetes
 discovery, OpenTelemetry exporter, and K2V features (K2V will be enabled once
@@ -27,11 +26,7 @@ it's stable).
 
 ## Arch Linux
 
-Garage is available in the official repositories under [extra](https://archlinux.org/packages/extra/x86_64/garage).
+Garage is available in the [AUR](https://aur.archlinux.org/packages/garage).
-
-```bash
-pacman -S garage
-```
 
 ## FreeBSD
 
@@ -44,9 +39,3 @@ pkg install garage
 ```bash
 nix-shell -p garage
 ```
-
-## conda-forge
-
-```bash
-pixi global install garage
-```

doc/book/cookbook/from-source.md

@@ -20,10 +20,10 @@ sudo apt-get update
 sudo apt-get install build-essential
 ```
 
-## Building from source from the Forgejo repository
+## Building from source from the Gitea repository
 
 The primary location for Garage's source code is the
-[Forgejo repository](https://git.deuxfleurs.fr/Deuxfleurs/garage),
+[Gitea repository](https://git.deuxfleurs.fr/Deuxfleurs/garage),
 which contains all of the released versions as well as the code
 for the developpement of the next version.
 
@@ -85,14 +85,11 @@ The following feature flags are available in v0.8.0:
 | Feature flag | Enabled | Description |
 | ------------ | ------- | ----------- |
 | `bundled-libs` | *by default* | Use bundled version of sqlite3, zstd, lmdb and libsodium |
-| `consul-discovery` | optional | Enable automatic registration and discovery<br>of cluster nodes through the Consul API |
+| `system-libs` | optional | Use system version of sqlite3, zstd, lmdb and libsodium<br>if available (exclusive with `bundled-libs`, build using<br>`cargo build --no-default-features --features system-libs`) |
-| `fjall` | experimental | Enable using Fjall to store Garage's metadata |
-| `journald` | optional | Enable logging to systemd-journald with<br>`GARAGE_LOG_TO_JOURNALD=true` environment variable set |
 | `k2v` | optional | Enable the experimental K2V API (if used, all nodes on your<br>Garage cluster must have it enabled as well) |
 | `kubernetes-discovery` | optional | Enable automatic registration and discovery<br>of cluster nodes through the Kubernetes API |
-| `lmdb` | *by default* | Enable using LMDB to store Garage's metadata |
 | `metrics` | *by default* | Enable collection of metrics in Prometheus format on the admin API |
-| `sqlite` | *by default* | Enable using Sqlite3 to store Garage's metadata |
-| `syslog` | optional | Enable logging to Syslog with<br>`GARAGE_LOG_TO_SYSLOG=true` environment variable set |
-| `system-libs` | optional | Use system version of sqlite3, zstd, lmdb and libsodium<br>if available (exclusive with `bundled-libs`, build using<br>`cargo build --no-default-features --features system-libs`) |
 | `telemetry-otlp` | optional | Enable collection of execution traces using OpenTelemetry |
+| `syslog` | optional | Enable logging to Syslog |
+| `lmdb` | *by default* | Enable using LMDB to store Garage's metadata |
+| `sqlite` | *by default* | Enable using Sqlite3 to store Garage's metadata |

doc/book/cookbook/kubernetes.md

@@ -11,7 +11,7 @@ Firstly clone the repository:
 
 ```bash
 git clone https://git.deuxfleurs.fr/Deuxfleurs/garage
-cd garage/script/helm
+cd garage/scripts/helm
 ```
 
 Deploy with default options:

doc/book/cookbook/real-world.md

@@ -96,14 +96,14 @@ to store 2 TB of data in total.
 ## Get a Docker image
 
 Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
-We encourage you to use a fixed tag (eg. `v2.2.0`) and not the `latest` tag.
+We encourage you to use a fixed tag (eg. `v2.1.0`) and not the `latest` tag.
-For this example, we will use the latest published version at the time of the writing which is `v2.2.0` but it's up to you
+For this example, we will use the latest published version at the time of the writing which is `v2.1.0` but it's up to you
 to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
 
 For example:
 
 ```
-sudo docker pull dxflrs/garage:v2.2.0
+sudo docker pull dxflrs/garage:v2.1.0
 ```
 
 ## Deploying and configuring Garage
@@ -171,7 +171,7 @@ docker run \
   -v /etc/garage.toml:/etc/garage.toml \
   -v /var/lib/garage/meta:/var/lib/garage/meta \
   -v /var/lib/garage/data:/var/lib/garage/data \
-  dxflrs/garage:v2.2.0
+  dxflrs/garage:v2.1.0
 ```
 
 With this command line, Garage should be started automatically at each boot.
@@ -185,7 +185,7 @@ If you want to use `docker-compose`, you may use the following `docker-compose.y
 version: "3"
 services:
   garage:
-    image: dxflrs/garage:v2.2.0
+    image: dxflrs/garage:v2.1.0
     network_mode: "host"
     restart: unless-stopped
     volumes:

doc/book/cookbook/reverse-proxy.md

@@ -7,7 +7,7 @@ The main reason to add a reverse proxy in front of Garage is to provide TLS to y
 
 In production you will likely need your certificates signed by a certificate authority.
 The most automated way is to use a provider supporting the [ACME protocol](https://datatracker.ietf.org/doc/html/rfc8555) 
-such as [Let's Encrypt](https://letsencrypt.org/) or [ZeroSSL](https://zerossl.com/).
+such as [Let's Encrypt](https://letsencrypt.org/), [ZeroSSL](https://zerossl.com/) or [Buypass Go SSL](https://www.buypass.com/ssl/products/acme).
 
 If you are only testing Garage, you can generate a self-signed certificate to follow the documentation:
 

doc/book/quick-start/_index.md

@@ -132,7 +132,7 @@ docker run \
   -v /path/to/garage.toml:/etc/garage.toml \
   -v /path/to/garage/meta:/var/lib/garage/meta \
   -v /path/to/garage/data:/var/lib/garage/data \
-  dxflrs/garage:v2.2.0
+  dxflrs/garage:v2.1.0
 ```
 
 Under Linux, you can substitute `--network host` for `-p 3900:3900 -p 3901:3901 -p 3902:3902 -p 3903:3903`

doc/book/reference-manual/configuration.md

@@ -25,7 +25,7 @@ db_engine = "lmdb"
 block_size = "1M"
 block_ram_buffer_max = "256MiB"
 block_max_concurrent_reads = 16
-block_max_concurrent_writes_per_request =10
+
 lmdb_map_size = "1T"
 
 compression_level = 1
@@ -51,20 +51,17 @@ allow_punycode = false
 
 [consul_discovery]
 api = "catalog"
-consul_http_addr = "https://127.0.0.1:8500"
+consul_http_addr = "http://127.0.0.1:8500"
-tls_skip_verify = false
 service_name = "garage-daemon"
-
 ca_cert = "/etc/consul/consul-ca.crt"
 client_cert = "/etc/consul/consul-client.crt"
 client_key = "/etc/consul/consul-key.crt"
-
 # for `agent` API mode, unset client_cert and client_key, and optionally enable `token`
 # token = "abcdef-01234-56789"
-
+tls_skip_verify = false
 tags = [ "dns-enabled" ]
 meta = { dns-acl = "allow trusted" }
-datacenters = ["dc1", "dc2", "dc3"]
+
 
 [kubernetes_discovery]
 namespace = "garage"
@@ -102,7 +99,6 @@ Top-level configuration options, in alphabetical order:
 [`allow_punycode`](#allow_punycode),
 [`allow_world_readable_secrets`](#allow_world_readable_secrets),
 [`block_max_concurrent_reads`](#block_max_concurrent_reads),
-[`block_max_concurrent_writes_per_request`](#block_max_concurrent_writes_per_request),
 [`block_ram_buffer_max`](#block_ram_buffer_max),
 [`block_size`](#block_size),
 [`bootstrap_peers`](#bootstrap_peers),
@@ -131,14 +127,12 @@ The `[consul_discovery]` section:
 [`client_cert`](#consul_client_cert_and_key),
 [`client_key`](#consul_client_cert_and_key),
 [`consul_http_addr`](#consul_http_addr),
-[`datacenters`](#consul_datacenters)
 [`meta`](#consul_tags_and_meta),
 [`service_name`](#consul_service_name),
 [`tags`](#consul_tags_and_meta),
 [`tls_skip_verify`](#consul_tls_skip_verify),
 [`token`](#consul_token).
 
-
 The `[kubernetes_discovery]` section:
 [`namespace`](#kube_namespace),
 [`service_name`](#kube_service_name),
@@ -560,14 +554,6 @@ metric in Prometheus: a non-zero number of such events indicates an I/O
 bottleneck on HDD read speed.
 
 
-#### `block_max_concurrent_writes_per_request` (since `v1.3.1` / `v2.2.0`) {#block_max_concurrent_writes_per_request}
-
-This parameter is designed to adapt to the concurrent write performance of 
-different storage media. Maximum number of parallel block writes per put request.
-Higher values may improve throughput but increase memory usage.
-
-Default value: 3. Recommended values: 10-30 for NVMe, 3-10 for spinning HDD.
-
 #### `lmdb_map_size` {#lmdb_map_size}
 
 This parameters can be used to set the map size used by LMDB,
@@ -740,18 +726,6 @@ node_prefix "" {
 }
 ```
 
-
-#### `datacenters` {#consul_datacenters}
-
-Optional list of datacenters that allow garage to do service discovery when Consul is configured in WAN federation.
-
-Example: `datacenters = ["dc1", "dc2", "dc3"]`
-
-In a WAN configuration, by default the Consul services API only responds with
-local LAN services.  When a list of datacenters is specified using this option,
-Garage will query the consul server API by datacenter directly, allowing for
-Garage to discover nodes across the Consul WAN.
-
 #### `tags` and `meta` {#consul_tags_and_meta}
 
 Additional list of tags and map of service meta to add during service registration.

doc/book/reference-manual/s3-compatibility.md

@@ -27,7 +27,7 @@ Feel free to open a PR to suggest fixes this table. Minio is missing because the
 
 | Feature                      | Garage                           | [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) | [Ceph Object Gateway](https://docs.ceph.com/en/latest/radosgw/s3/) | [Riak CS](https://docs.riak.com/riak/cs/2.1.1/references/apis/storage/s3/index.html) | [OpenIO](https://docs.openio.io/latest/source/arch-design/s3_compliancy.html) |
 |------------------------------|----------------------------------|-----------------|---------------|---------|-----|
-| [signature v2](https://docs.aws.amazon.com/AmazonS3/latest/API/Appendix-Sigv2.html) (deprecated) | ❌ Missing | ✅ |  ✅ | ✅ | ✅ |
+| [signature v2](https://docs.aws.amazon.com/general/latest/gr/signature-version-2.html) (deprecated) | ❌ Missing | ✅ |  ✅ | ✅ | ✅ |
 | [signature v4](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html) |  ✅ Implemented |  ✅ |  ✅ | ❌ | ✅ |
 | [URL path-style](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#path-style-access) (eg. `host.tld/bucket/key`) |  ✅ Implemented | ✅ |  ✅ | ❓| ✅ |
 | [URL vhost-style](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#virtual-hosted-style-access) URL (eg. `bucket.host.tld/key`) |  ✅ Implemented | ❌| ✅| ✅ | ✅ |

doc/drafts/garbage-collector.md

@@ -0,0 +1,142 @@
+# [RFC] Garbage Collector Elimination
+
+## Statement of problem and prior art
+
+Currently, Garage's garbage collector has a few identified issues.
+Namely, it can only be run if all nodes in a partition are currently online,
+it may not be correct in front of a rebalancing (this is partially mitigated by a 24h delay added to tombstone deletion),
+and it isn't resilient to a subset of nodes being restored from snapshots.
+It's not clear if it is possible to implement a garbage collection process that can eliminate tombstones, but also support
+a node rollback to a point in time where a key existed.
+
+This problematic, perhaps unsurprisingly, maps very well to the general abstraction of CRDTs for sets, where a whole partition
+would be a single CRDT. The semantic required in Garage demands reinsertion of deleted keys, which excludes the most simple
+forms of sets such as G-Sets and 2P-Sets. As the goal is to not handle garbage collection of tombstones, a standard ORSet
+is also unfitting. In fact it could be argued Garage already uses something akin to an ORSet with a garbage collector today.
+
+There exists CRDTs supporting this feature-set, one of which is an OptORSet[^1].
+It only needs a set-wide metadata proportional in the number of writers, and a per alive-key metadata proportional in the number
+of writers to that key (but no metadata for dead keys).
+These metadata are akin to DVVs[^2]. An element is considered new if its DVV comes causaly after the current DVV of the set.
+Correctness of this algorithm depends however on having causal delivery, which isn't given in Garage, neither in general nor in
+presence of snapshot restoration.
+
+## Proposal
+
+We devise a new kind of CRDT based on the core ideas of an OptORSet, but replacing each version inside its DVVs with a list of
+range of observed updates, which we name a Seen Vector (SV). Such metadata can in the worst case grow linearly with the number of
+insertion. In practice, assuming all elements are eventually known to every replica, the storage requirement of a SV is equivalent
+to that of a standard DVV. Under causal delivery, a SV degenerates into a standard DVV.
+
+To add an element to the set, a node increments its own version counter, and sends an update containing (element, replica, version).
+On receiving such an update, a node checks if it has already seen this particular (replica, version), and if so ignore it.
+If it hasn't seen that update, it saves the new element, and update its SV to include that new (replica, version).
+
+TODO: describe formaly the algorithm
+
+
+Algorithm 1, Seen Vector:
+```
+payload set S    -- S: set of triple (replica i, timestamp s, timestamp e)
+
+initial ∅
+query seen (replica i, timestamp c): boolean b
+  let b = (∃s <= c,e > c: (i,s,e) ∈ S)
+
+update increment ()
+  prepare ()
+    let r = myID()
+    let t = e|∀s,s', ∄e' > e: (r,s,e) ∈ S, (r,s',e') ∈ S    -- t is the maximum end bound for this replica
+  effect(r, t)
+    if ¬seen (r, t) then
+      if seen (r, t - 1) ∧ seen (r, t + 1) then
+        let R = {∃s: (r, s, t) ∈ S}
+        let R' = {∃e: (r, t + 1, e) ∈ S}
+        let M = S \ R
+        let M' = M \ R'
+        S := M' ∪ {(r, s, e)}
+      else if seen (r, t - 1)then
+        let R = {∃s: (r, s, t) ∈ S}
+        let M = S \ R
+        S := M ∪ {(r, s, t+1)}
+      else if seen (r, t + 1)then
+        let R = {∃e: (r, t + 1, e) ∈ S}
+        let M = S \ R
+        E := E ∪ {(r, t, e)}
+      else
+        E := E ∪ {(r, t, t+1)}
+
+merge (B)
+  # TODO this is correct, but largelly suboptimal. We should perform the increment.effect subroutine for all elements of B instead
+  S := S ∪ B.S
+```
+
+Algorithm 2, OptORSet with SV. This algorithm is largely copied and adapted from Figure 3 of [^1]
+```
+payload set E, SV sv  -- E: elements, set of triples (element e, timestamp c, replica i)
+                      -- sv: SeenVector of received triples
+
+initial ∅, ∅
+query contains (element e) : boolean b
+  let b = (∃c, i : (e, c, i) ∈ E)
+
+query elements () : set S
+  let S = {e|∃c, i : (e, c, i) ∈ E}
+
+update add (element e)
+  prepare (e)
+    let r = myID() -- r = source replica
+    let c = sv.increment.prepare().t
+  effect (e, c, r)
+    if ¬sv.seen(r, c) then
+    let O = {(e, c′, r) ∈ E|c′ < c}
+    sv.increment.effect(r, c)
+    E := E ∪ {(e, c, r)} \ O
+
+update remove (element e)
+  prepare (e) -- Collect all unique triples containing e
+    let R = {(e, c, i) ∈ E}
+  effect (R) -- Remove triples observed at source
+    pre causal delivery
+    E := E \ R
+
+merge (B)
+  let M = (E ∩ B.E)
+  let M ′ = {(e, c, i) ∈ E \ B.E| ¬B.sv.seen(i, c)}
+  let M ′′ = {(e, c, i) ∈ B.E \ E| ¬sv.seen(i, c)}
+  let U = M ∪ M ′ ∪ M ′′
+  let O = {(e, c, i) ∈ U |∃(e, c′, i) ∈ U : c < c′}
+  E := U \ O
+  sv := sv.merge(B.sv)
+```
+
+## Storage evaluation
+
+As stated, if all updates are received, the SV is similar in size to that of a standard DVV. It may be however that a node create and
+immediately delete an element, creating holes in its sequence from the point of view of other replicas. These holes could be filled
+through an interactive process where the replica observing holes asks the node to scan over the whole set, and for each version in these
+hole, reply if no element has that exact version number. Holes caused by existing elements should be eventually fixed by an anti-entropy
+process, so replying with these elements appears unnecessary.
+
+The per-element storage requirement is proportional to the number of replicas having modified that element, even under non steady-state.
+This happens because as we always exchange whole elements, we have causal delivery for individual keys.
+
+## Replica version rollback
+
+This scheme assumes the same node won't issue the same version twice, which isn't a given when a node might be rollback to a previous state.
+The author proposes that on initialization, a replica asks all other replicas for the highest version number they know for it.
+If all replicas reply with a number less than or equal to the current version, it is safe to reuse the currently known number.
+If some replicas reply with a number higher, the node increase its version to that number.
+If at least one replica doesn't reply, it can't make any assumption about its actual version number.
+The node then increment a generation number, which is made part of its replica id, starts a new sequence from zero.
+
+## Appendix: providing SV to the underlying elements
+
+Some more complexe elements may want to have access to a version id and the Seen Vector to perform their own internal merge operations.
+The author reckon this may help implementing S3 versioning, by giving a simple way for Objects to know if an ObjectVersion was yet
+unknown or is known and already deleted.
+
+## References
+ 	
+[^1]: An optimized conflict-free replicated set, https://doi.org/10.48550/arXiv.1210.3368
+[^2]: Dotted Version Vectors: Logical Clocks for Optimistic Replication, https://doi.org/10.48550/arXiv.1011.5808

flake.lock

@@ -12,17 +12,16 @@
       "original": {
         "owner": "ipetkov",
         "repo": "crane",
-        "rev": "6fe74265bbb6d016d663b1091f015e2976c4a527",
         "type": "github"
       }
     },
     "flake-compat": {
       "locked": {
-        "lastModified": 1761640442,
+        "lastModified": 1717312683,
-        "narHash": "sha256-AtrEP6Jmdvrqiv4x2xa5mrtaIp3OEe8uBYCDZDS+hu8=",
+        "narHash": "sha256-FrlieJH50AuvagamEvWMIE6D2OAnERuDboFDYAED/dE=",
         "owner": "nix-community",
         "repo": "flake-compat",
-        "rev": "4a56054d8ffc173222d09dad23adf4ba946c8884",
+        "rev": "38fd3954cf65ce6faf3d0d45cd26059e059f07ea",
         "type": "github"
       },
       "original": {
@@ -51,17 +50,17 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1763977559,
+        "lastModified": 1747825515,
-        "narHash": "sha256-g4MKqsIRy5yJwEsI+fYODqLUnAqIY4kZai0nldAP6EM=",
+        "narHash": "sha256-BWpMQymVI73QoKZdcVCxUCCK3GNvr/xa2Dc4DM1o2BE=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "cfe2c7d5b5d3032862254e68c37a6576b633d632",
+        "rev": "cd2812de55cf87df88a9e09bf3be1ce63d50c1a6",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "cfe2c7d5b5d3032862254e68c37a6576b633d632",
+        "rev": "cd2812de55cf87df88a9e09bf3be1ce63d50c1a6",
         "type": "github"
       }
     },
@@ -81,17 +80,17 @@
         ]
       },
       "locked": {
-        "lastModified": 1763952169,
+        "lastModified": 1738549608,
-        "narHash": "sha256-+PeDBD8P+NKauH+w7eO/QWCIp8Cx4mCfWnh9sJmy9CM=",
+        "narHash": "sha256-GdyT9QEUSx5k/n8kILuNy83vxxdyUfJ8jL5mMpQZWfw=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "ab726555a9a72e6dc80649809147823a813fa95b",
+        "rev": "35c6f8c4352f995ecd53896200769f80a3e8f22d",
         "type": "github"
       },
       "original": {
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "ab726555a9a72e6dc80649809147823a813fa95b",
+        "rev": "35c6f8c4352f995ecd53896200769f80a3e8f22d",
         "type": "github"
       }
     },

flake.nix

@@ -2,17 +2,16 @@
   description =
     "Garage, an S3-compatible distributed object store for self-hosted deployments";
 
-  # Nixpkgs 25.05 as of 2025-11-24
+  # Nixpkgs 25.05 as of 2025-05-22
   inputs.nixpkgs.url =
-    "github:NixOS/nixpkgs/cfe2c7d5b5d3032862254e68c37a6576b633d632";
+    "github:NixOS/nixpkgs/cd2812de55cf87df88a9e09bf3be1ce63d50c1a6";
 
-  # Rust overlay as of 2025-11-24
+  # Rust overlay as of 2025-02-03
   inputs.rust-overlay.url =
-  "github:oxalica/rust-overlay/ab726555a9a72e6dc80649809147823a813fa95b";
+  "github:oxalica/rust-overlay/35c6f8c4352f995ecd53896200769f80a3e8f22d";
   inputs.rust-overlay.inputs.nixpkgs.follows = "nixpkgs";
 
-  # Crane as of 2025-01-24
+  inputs.crane.url = "github:ipetkov/crane";
-  inputs.crane.url = "github:ipetkov/crane/6fe74265bbb6d016d663b1091f015e2976c4a527";
 
   inputs.flake-compat.url = "github:nix-community/flake-compat";
   inputs.flake-utils.url = "github:numtide/flake-utils";
@@ -31,10 +30,6 @@
           inherit system nixpkgs crane rust-overlay extraTestEnv;
           release = false;
         }).garage-test;
-        lints = (compile {
-          inherit system nixpkgs crane rust-overlay;
-          release = false;
-        });
       in
       {
         packages = {
@@ -61,10 +56,6 @@
           tests-fjall = testWith {
             GARAGE_TEST_INTEGRATION_DB_ENGINE = "fjall";
           };
-
-          # lints (fmt, clippy)
-          fmt = lints.garage-cargo-fmt;
-          clippy = lints.garage-cargo-clippy;
         };
 
         # ---- developpment shell, for making native builds only ----

nix/build_index.nix

@@ -167,7 +167,7 @@ let
                  </ul></p>
                  <p> Sources:
                  <ul>
-                  <li><a href="https://git.deuxfleurs.fr/Deuxfleurs/garage/src/${r.type}/${x.version}">Forgejo</a></li>
+                  <li><a href="https://git.deuxfleurs.fr/Deuxfleurs/garage/src/${r.type}/${x.version}">gitea</a></li>
                   <li><a href="https://git.deuxfleurs.fr/Deuxfleurs/garage/archive/${x.version}.zip">.zip</a></li>
                   <li><a href="https://git.deuxfleurs.fr/Deuxfleurs/garage/archive/${x.version}.tar.gz">.tar.gz</a></li>
                  </ul></p>

nix/compile.nix

@@ -48,7 +48,7 @@ let
 
   inherit (pkgs) lib stdenv;
 
-  toolchainFn = (p: p.rust-bin.stable."1.91.0".default.override {
+  toolchainFn = (p: p.rust-bin.stable."1.82.0".default.override {
     targets = lib.optionals (target != null) [ rustTarget ];
     extensions = [
       "rust-src"
@@ -190,15 +190,4 @@ in rec {
       pkgs.cacert
     ];
   } // extraTestEnv);
-
-  # ---- source code linting ----
-
-  garage-cargo-fmt = craneLib.cargoFmt (commonArgs // {
-    cargoExtraArgs = "";
-  });
-
-  garage-cargo-clippy = craneLib.cargoClippy (commonArgs // {
-    cargoArtifacts = garage-deps;
-    cargoClippyExtraArgs = "--all-targets -- -D warnings";
-  });
 }

script/helm/garage/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: garage
 description: S3-compatible object store for small self-hosted geo-distributed deployments
 type: application
-version: 0.9.2
+version: 0.9.1
-appVersion: "v2.2.0"
+appVersion: "v2.1.0"
 home: https://garagehq.deuxfleurs.fr/
 icon: https://garagehq.deuxfleurs.fr/images/garage-logo.svg
 

script/helm/garage/README.md

@@ -1,6 +1,6 @@
 # garage
 
-![Version: 0.9.2](https://img.shields.io/badge/Version-0.9.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v2.2.0](https://img.shields.io/badge/AppVersion-v2.2.0-informational?style=flat-square)
+![Version: 0.9.1](https://img.shields.io/badge/Version-0.9.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v2.1.0](https://img.shields.io/badge/AppVersion-v2.1.0-informational?style=flat-square)
 
 S3-compatible object store for small self-hosted geo-distributed deployments
 

script/helm/garage/templates/_helpers.tpl

@@ -47,8 +47,8 @@ helm.sh/chart: {{ include "garage.chart" . }}
 app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
 {{- end }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
-{{- with .Values.commonLabels }}
+{{ with .Values.commonLabels }}
-{{- toYaml . | nindent 0 }}
+{{- toYaml . }}
 {{- end }}
 {{- end }}
 

script/helm/garage/templates/configmap.yaml

@@ -28,11 +28,7 @@ data:
     # rpc_secret will be populated by the init container from a k8s secret object
     rpc_secret = "__RPC_SECRET_REPLACE__"
 
-    bootstrap_peers = [
+    bootstrap_peers = {{ .Values.garage.bootstrapPeers }}
-    {{- range $index, $peer := .Values.garage.bootstrapPeers }}
-    {{- if $index}}, {{ end }}{{ $peer | quote }}
-    {{ end }} 
-    ]
 
     {{- if .Values.garage.additionalTopLevelConfig }}
     {{ .Values.garage.additionalTopLevelConfig | nindent 4 }}

script/helm/garage/templates/service.yaml

@@ -4,10 +4,6 @@ metadata:
   name: {{ include "garage.fullname" . }}
   labels:
     {{- include "garage.labels" . | nindent 4 }}
-  {{- with .Values.service.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
 spec:
   type: {{ .Values.service.type }}
   ports:
@@ -41,4 +37,4 @@ spec:
       name: metrics
   selector:
     {{- include "garage.selectorLabels" . | nindent 4 }}
-{{- end }}
+{{- end }}

script/helm/garage/values.yaml

@@ -144,8 +144,6 @@ service:
   # - NodePort (+ Ingress)
   # - LoadBalancer
   type: ClusterIP
-  # -- Annotations to add to the service
-  annotations: {}
   s3:
     api:
       port: 3900

shell.nix

@@ -26,7 +26,7 @@ in
       s3cmd
       minio-client
       rclone
-      (python313.withPackages (ps: [ ps.boto3 ]))
+      (python312.withPackages (ps: [ ps.boto3 ]))
 
       socat
       psmisc
@@ -36,8 +36,6 @@ in
       jq
     ];
     shellHook = ''
-      export AWS_REQUEST_CHECKSUM_CALCULATION='when_required'
-
       function to_s3 {
         AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED AWS_RESPONSE_CHECKSUM_VALIDATION=WHEN_REQUIRED \
         aws \

src/api/admin/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_api_admin"
-version = "2.2.0"
+version = "2.1.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/api/admin/error.rs

@@ -26,7 +26,7 @@ pub enum Error {
 	NoSuchAdminToken(String),
 
 	/// The API access key does not exist
-	#[error("Access key not found: {0}")]
+	#[error("Access key not found: {00}")]
 	NoSuchAccessKey(String),
 
 	/// The requested block does not exist

src/api/admin/openapi.rs

@@ -1,8 +1,7 @@
 #![allow(dead_code)]
 #![allow(non_snake_case)]
 
-use serde::{Deserialize, Serialize};
+use utoipa::{Modify, OpenApi};
-use utoipa::{Modify, OpenApi, ToSchema};
 
 use crate::api::*;
 
@@ -247,7 +246,7 @@ For example to declare 100GB, you must set `capacity: 100000000000`.
 Garage uses internally the International System of Units (SI), it assumes that 1kB = 1000 bytes, and displays storage as kB, MB, GB (and not KiB, MiB, GiB that assume 1KiB = 1024 bytes).
     ",
     request_body(
-        content=UpdateClusterLayoutRequestOpenapi,
+        content=UpdateClusterLayoutRequest,
         description="
 To add a new node to the layout or to change the configuration of an existing node, simply set the values you want (`zone`, `capacity`, and `tags`).
 To remove a node, simply pass the `remove: true` field.
@@ -263,48 +262,6 @@ Contrary to the CLI that may update only a subset of the fields capacity, zone a
 )]
 fn UpdateClusterLayout() -> () {}
 
-// Hack: we cannot use the UpdateClusterLayoutRequest from api.rs,
-// as it contains (via NodeRoleChange) an untagged enum flattenned into
-// a struct, which breaks the openapi generator.
-// See issue #1249.
-// Instead, we use a rewritten version of the NodeRoleChange struct where
-// the struct fields are distributed into the enum variants (this is an equivalent
-// representation, but this way we avoid having to rewrite all uses of the original
-// struct in the Garage codebase).
-#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
-#[schema(as = UpdateClusterLayoutRequest)]
-pub struct UpdateClusterLayoutRequestOpenapi {
-	/// New node roles to assign or remove in the cluster layout
-	#[serde(default)]
-	pub roles: Vec<NodeRoleChangeOpenapi>,
-	/// New layout computation parameters to use
-	#[serde(default)]
-	pub parameters: Option<LayoutParameters>,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
-#[schema(as = NodeRoleChangeRequest)]
-#[serde(untagged)]
-pub enum NodeRoleChangeOpenapi {
-	#[serde(rename_all = "camelCase")]
-	Remove {
-		/// ID of the node for which this change applies
-		id: String,
-		/// Set `remove` to `true` to remove the node from the layout
-		remove: bool,
-	},
-	Update(NodeRoleUpdate),
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
-#[serde(rename_all = "camelCase")]
-pub struct NodeRoleUpdate {
-	/// ID of the node for which this change applies
-	id: String,
-	#[serde(flatten)]
-	role: NodeAssignedRole,
-}
-
 #[utoipa::path(post,
     path = "/v2/PreviewClusterLayoutChanges",
     tag = "Cluster layout",
@@ -629,7 +586,7 @@ fn DenyBucketKey() -> () {}
     path = "/v2/AddBucketAlias",
     tag = "Bucket alias",
     description = "Add an alias for the target bucket.  This can be either a global or a local alias, depending on which fields are specified.",
-    request_body = BucketAliasEnumOpenapi,
+    request_body = AddBucketAliasRequest,
 	responses(
             (status = 200, description = "Returns exhaustive information about the bucket", body = AddBucketAliasResponse),
             (status = 500, description = "Internal server error")
@@ -641,7 +598,7 @@ fn AddBucketAlias() -> () {}
     path = "/v2/RemoveBucketAlias",
     tag = "Bucket alias",
     description = "Remove an alias for the target bucket.  This can be either a global or a local alias, depending on which fields are specified.",
-    request_body = BucketAliasEnumOpenapi,
+    request_body = RemoveBucketAliasRequest,
 	responses(
             (status = 200, description = "Returns exhaustive information about the bucket", body = RemoveBucketAliasResponse),
             (status = 500, description = "Internal server error")
@@ -649,24 +606,6 @@ fn AddBucketAlias() -> () {}
 )]
 fn RemoveBucketAlias() -> () {}
 
-// Hack for issue #1249 (see UpdateClusterLayout)
-#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
-#[serde(untagged)]
-#[schema(as = BucketAliasEnum)]
-pub enum BucketAliasEnumOpenapi {
-	#[serde(rename_all = "camelCase")]
-	Global {
-		bucket_id: String,
-		global_alias: String,
-	},
-	#[serde(rename_all = "camelCase")]
-	Local {
-		bucket_id: String,
-		local_alias: String,
-		access_key_id: String,
-	},
-}
-
 // **********************************************
 //      Node operations
 // **********************************************
@@ -879,7 +818,7 @@ impl Modify for SecurityAddon {
 #[derive(OpenApi)]
 #[openapi(
     info(
-        version = "v2.2.0",
+        version = "v2.1.0",
         title = "Garage administration API",
         description = "Administrate your Garage cluster programatically, including status, layout, keys, buckets, and maintainance tasks.
 

src/api/common/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_api_common"
-version = "2.2.0"
+version = "2.1.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/api/common/signature/payload.rs

@@ -105,7 +105,7 @@ fn check_standard_signature(
 	// Verify that all necessary request headers are included in signed_headers
 	// The following must be included for all signatures:
 	// - the Host header (mandatory)
-	// - all x-amz-* headers used in the request (except x-amz-content-sha256)
+	// - all x-amz-* headers used in the request
 	// AWS also indicates that the Content-Type header should be signed if
 	// it is used, but Minio client doesn't sign it so we don't check it for compatibility.
 	let signed_headers = split_signed_headers(&authorization)?;
@@ -152,7 +152,7 @@ fn check_presigned_signature(
 	// Verify that all necessary request headers are included in signed_headers
 	// For AWSv4 pre-signed URLs, the following must be included:
 	// - the Host header (mandatory)
-	// - all x-amz-* headers used in the request (except x-amz-content-sha256)
+	// - all x-amz-* headers used in the request
 	let signed_headers = split_signed_headers(&authorization)?;
 	verify_signed_headers(request.headers(), &signed_headers)?;
 
@@ -269,9 +269,7 @@ fn verify_signed_headers(headers: &HeaderMap, signed_headers: &[HeaderName]) ->
 		return Err(Error::bad_request("Header `Host` should be signed"));
 	}
 	for (name, _) in headers.iter() {
-		// Enforce signature of all x-amz-* headers, except x-amz-content-sh256
+		if name.as_str().starts_with("x-amz-") {
-		// because it is included in the canonical request in all cases
-		if name.as_str().starts_with("x-amz-") && name != X_AMZ_CONTENT_SHA256 {
 			if !signed_headers.contains(name) {
 				return Err(Error::bad_request(format!(
 					"Header `{}` should be signed",
@@ -477,7 +475,8 @@ impl Authorization {
 
 		let date = headers
 			.get(X_AMZ_DATE)
-			.ok_or_bad_request("Missing X-Amz-Date field")?
+			.ok_or_bad_request("Missing X-Amz-Date field")
+			.map_err(Error::from)?
 			.to_str()?;
 		let date = parse_date(date)?;
 

src/api/k2v/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_api_k2v"
-version = "2.2.0"
+version = "2.1.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/api/s3/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_api_s3"
-version = "2.2.0"
+version = "2.1.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/api/s3/get.rs

@@ -853,9 +853,7 @@ impl PreconditionHeaders {
 	}
 
 	fn check(&self, v: &ObjectVersion, etag: &str) -> Result<Option<StatusCode>, Error> {
-		// we store date with ms precision, but headers are precise to the second: truncate
+		let v_date = UNIX_EPOCH + Duration::from_millis(v.timestamp);
-		// the timestamp to handle the same-second edge case
-		let v_date = UNIX_EPOCH + Duration::from_secs(v.timestamp / 1000);
 
 		// Implemented from https://datatracker.ietf.org/doc/html/rfc7232#section-6
 

src/api/s3/post_object.rs

@@ -138,26 +138,10 @@ pub async fn handle_post_object(
 
 	let mut conditions = decoded_policy.into_conditions()?;
 
-	// If there are conditions on the bucket name, check these against the actual bucket_name rather
-	// than the one in params, which is allowed to be absent.
-	if let Some(conds) = conditions.params.remove("bucket") {
-		for cond in conds {
-			let ok = match cond {
-				Operation::Equal(s) => s.as_str() == bucket_name,
-				Operation::StartsWith(s) => bucket_name.starts_with(&s),
-			};
-			if !ok {
-				return Err(Error::bad_request(
-					"Key 'bucket' has value not allowed in policy",
-				));
-			}
-		}
-	}
-
 	for (param_key, value) in params.iter() {
 		let param_key = param_key.as_str();
 		match param_key {
-			"policy" | "x-amz-signature" | "bucket" => (), // this is always accepted, as it's required to validate other fields
+			"policy" | "x-amz-signature" => (), // this is always accepted, as it's required to validate other fields
 			"content-type" => {
 				let conds = conditions.params.remove("content-type").ok_or_else(|| {
 					Error::bad_request(format!("Key '{}' is not allowed in policy", param_key))

src/api/s3/put.rs

@@ -39,6 +39,8 @@ use crate::encryption::{EncryptionParams, OekDerivationInfo};
 use crate::error::*;
 use crate::website::X_AMZ_WEBSITE_REDIRECT_LOCATION;
 
+const PUT_BLOCKS_MAX_PARALLEL: usize = 3;
+
 pub(crate) struct SaveStreamResult {
 	pub(crate) version_uuid: Uuid,
 	pub(crate) version_timestamp: u64,
@@ -505,7 +507,7 @@ pub(crate) async fn read_and_put_blocks<S: Stream<Item = Result<Bytes, Error>> +
 			};
 			let recv_next = async {
 				// If more than a maximum number of writes are in progress, don't add more for now
-				if currently_running >= ctx.garage.config.block_max_concurrent_writes_per_request {
+				if currently_running >= PUT_BLOCKS_MAX_PARALLEL {
 					futures::future::pending().await
 				} else {
 					block_rx3.recv().await

src/block/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_block"
-version = "2.2.0"
+version = "2.1.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/db/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_db"
-version = "2.2.0"
+version = "2.1.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/garage/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage"
-version = "2.2.0"
+version = "2.1.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/garage/main.rs

@@ -68,30 +68,24 @@ struct Opt {
 async fn main() {
 	// Initialize version and features info
 	let features = &[
-		#[cfg(feature = "bundled-libs")]
-		"bundled-libs",
-		#[cfg(feature = "consul-discovery")]
-		"consul-discovery",
-		#[cfg(feature = "fjall")]
-		"fjall",
-		#[cfg(feature = "journald")]
-		"journald",
 		#[cfg(feature = "k2v")]
 		"k2v",
-		#[cfg(feature = "kubernetes-discovery")]
-		"kubernetes-discovery",
 		#[cfg(feature = "lmdb")]
 		"lmdb",
-		#[cfg(feature = "metrics")]
-		"metrics",
 		#[cfg(feature = "sqlite")]
 		"sqlite",
-		#[cfg(feature = "syslog")]
+		#[cfg(feature = "consul-discovery")]
-		"syslog",
+		"consul-discovery",
-		#[cfg(feature = "system-libs")]
+		#[cfg(feature = "kubernetes-discovery")]
-		"system-libs",
+		"kubernetes-discovery",
+		#[cfg(feature = "metrics")]
+		"metrics",
 		#[cfg(feature = "telemetry-otlp")]
 		"telemetry-otlp",
+		#[cfg(feature = "bundled-libs")]
+		"bundled-libs",
+		#[cfg(feature = "system-libs")]
+		"system-libs",
 	][..];
 	if let Some(git_version) = option_env!("GIT_VERSION") {
 		garage_util::version::init_version(git_version);

src/garage/tests/s3/objects.rs

@@ -198,7 +198,6 @@ async fn test_precondition() {
 		);
 	}
 	let older_date = DateTime::from_secs_f64(last_modified.as_secs_f64() - 10.0);
-	let same_date = DateTime::from_secs_f64(last_modified.as_secs_f64());
 	let newer_date = DateTime::from_secs_f64(last_modified.as_secs_f64() + 10.0);
 	{
 		let err = ctx
@@ -213,18 +212,6 @@ async fn test_precondition() {
 			matches!(err, Err(SdkError::ServiceError(se)) if se.raw().status().as_u16() == 304)
 		);
 
-		let err = ctx
-			.client
-			.get_object()
-			.bucket(&bucket)
-			.key(STD_KEY)
-			.if_modified_since(same_date)
-			.send()
-			.await;
-		assert!(
-			matches!(err, Err(SdkError::ServiceError(se)) if se.raw().status().as_u16() == 304)
-		);
-
 		let o = ctx
 			.client
 			.get_object()
@@ -249,17 +236,6 @@ async fn test_precondition() {
 			matches!(err, Err(SdkError::ServiceError(se)) if se.raw().status().as_u16() == 412)
 		);
 
-		let o = ctx
-			.client
-			.get_object()
-			.bucket(&bucket)
-			.key(STD_KEY)
-			.if_unmodified_since(same_date)
-			.send()
-			.await
-			.unwrap();
-		assert_eq!(o.e_tag.as_ref().unwrap().as_str(), etag);
-
 		let o = ctx
 			.client
 			.get_object()

src/model/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_model"
-version = "2.2.0"
+version = "2.1.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/net/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_net"
-version = "2.2.0"
+version = "2.1.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/rpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_rpc"
-version = "2.2.0"
+version = "2.1.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/rpc/consul.rs

@@ -126,61 +126,38 @@ impl ConsulDiscovery {
 	}
 
 	// ---- READING FROM CONSUL CATALOG ----
-	/// Query Consul for Garage nodes registered under the configured service name.
+
-	///
-	/// This method supports querying multiple Consul datacenters for WAN or
-	/// multi-datacenter deployments. If `config.datacenters` is set and non-empty,
-	/// each listed datacenter is queried and the results are aggregated. Otherwise,
-	/// only the local datacenter is queried. `config.datacenters` does not need to be set
-	/// when all the datacenters are on the same LAN, in this case service discovery works normally
-	///
-	/// # Returns
-	/// A list of `(NodeID, SocketAddr)` pairs corresponding to all valid discovered
-	/// nodes across the queried datacenters.
 	pub async fn get_consul_nodes(&self) -> Result<Vec<(NodeID, SocketAddr)>, ConsulError> {
+		let url = format!(
+			"{}/v1/catalog/service/{}",
+			self.config.consul_http_addr, self.config.service_name
+		);
+
+		let http = self.client.get(&url).send().await?;
+		let entries: Vec<ConsulQueryEntry> = http.json().await?;
+
 		let mut ret = vec![];
-
+		for ent in entries {
-		let dcs_to_query: Vec<Option<&str>> = match &self.config.datacenters {
+			let ip = ent.address.parse::<IpAddr>().ok();
-			dcs if !dcs.is_empty() => dcs.iter().map(|dc| Some(dc.as_str())).collect(),
+			let pubkey = ent
-			_ => vec![None],
+				.meta
-		};
+				.get(&format!("{}-pubkey", META_PREFIX))
-
+				.and_then(|k| hex::decode(k).ok())
-		for dc in dcs_to_query {
+				.and_then(|k| NodeID::from_slice(&k[..]));
-			let url = match dc {
+			if let (Some(ip), Some(pubkey)) = (ip, pubkey) {
-				Some(datacenter) => format!(
+				ret.push((pubkey, SocketAddr::new(ip, ent.service_port)));
-					"{}/v1/catalog/service/{}?dc={}",
+			} else {
-					self.config.consul_http_addr, self.config.service_name, datacenter
+				warn!(
-				),
+					"Could not process node spec from Consul: {:?} (invalid IP address or node ID/pubkey)",
-				None => format!(
+					ent
-					"{}/v1/catalog/service/{}",
+				);
-					self.config.consul_http_addr, self.config.service_name
-				),
-			};
-
-			let http = self.client.get(&url).send().await?;
-			let entries: Vec<ConsulQueryEntry> = http.json().await?;
-
-			for ent in entries {
-				let ip = ent.address.parse::<IpAddr>().ok();
-				let pubkey = ent
-					.meta
-					.get(&format!("{}-pubkey", META_PREFIX))
-					.and_then(|k| hex::decode(k).ok())
-					.and_then(|k| NodeID::from_slice(&k[..]));
-				if let (Some(ip), Some(pubkey)) = (ip, pubkey) {
-					ret.push((pubkey, SocketAddr::new(ip, ent.service_port)));
-				} else {
-					warn!(
-						"Could not process node spec from Consul: {:?} (invalid IP address or node ID/pubkey)",
-						ent
-					);
-				}
 			}
 		}
+		debug!("Got nodes from Consul: {:?}", ret);
 
-		debug!("Got {} nodes from Consul", ret.len());
 		Ok(ret)
 	}
+
 	// ---- PUBLISHING TO CONSUL CATALOG ----
 
 	pub async fn publish_consul_service(

src/rpc/layout/version.rs

@@ -517,7 +517,7 @@ impl LayoutVersion {
 		g.compute_maximal_flow()?;
 		if g.get_flow_value()? < (NB_PARTITIONS * self.replication_factor) as i64 {
 			return Err(Error::Message(
-				"The storage capacity of the cluster is too small. It is \
+				"The storage capacity of he cluster is to small. It is \
                        impossible to store partitions of size 1."
 					.into(),
 			));

src/table/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_table"
-version = "2.2.0"
+version = "2.1.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/util/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_util"
-version = "2.2.0"
+version = "2.1.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/util/config.rs

@@ -45,11 +45,6 @@ pub struct Config {
 	)]
 	pub block_size: usize,
 
-	/// Maximum number of parallel block writes per PUT request
-	/// Higher values improve throughput but increase memory usage
-	/// Default: 3, Recommended: 10-30 for NVMe, 3-10 for HDD
-	#[serde(default = "default_block_max_concurrent_writes_per_request")]
-	pub block_max_concurrent_writes_per_request: usize,
 	/// Number of replicas. Can be any positive integer, but uneven numbers are more favorable.
 	/// - 1 for single-node clusters, or to disable replication
 	/// - 3 is the recommended and supported setting.
@@ -262,8 +257,6 @@ pub struct ConsulDiscoveryConfig {
 	/// Additional service metadata to add
 	#[serde(default)]
 	pub meta: Option<std::collections::HashMap<String, String>>,
-	#[serde(default)]
-	pub datacenters: Vec<String>,
 }
 
 #[derive(Deserialize, Debug, Clone)]
@@ -277,9 +270,6 @@ pub struct KubernetesDiscoveryConfig {
 	pub skip_crd: bool,
 }
 
-pub fn default_block_max_concurrent_writes_per_request() -> usize {
-	3
-}
 /// Read and parse configuration
 pub fn read_config(config_file: PathBuf) -> Result<Config, Error> {
 	let config = std::fs::read_to_string(config_file)?;

src/web/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_web"
-version = "2.2.0"
+version = "2.1.0"
 authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
 edition = "2018"
 license = "AGPL-3.0"