Compare commits

...

6 Commits

27 changed files with 1397 additions and 11 deletions
+12
View File
@@ -53,3 +53,15 @@ steps:
- nix-build -j4 --attr flakePackages.dev
- nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
depends_on: [ build ]
- name: helm chart tests
image: helmunittest/helm-unittest:4.2.3-1.1.2
commands:
- helm lint --strict script/helm/garage
- helm lint --strict script/helm/garage -f script/helm/garage/tests/values/daemonset.yaml
- helm lint --strict script/helm/garage -f script/helm/garage/tests/values/ingress.yaml
- helm lint --strict script/helm/garage -f script/helm/garage/tests/values/existing-secret.yaml
- helm lint --strict script/helm/garage -f script/helm/garage/tests/values/monitoring.yaml
- helm lint --strict script/helm/garage -f script/helm/garage/tests/values/minimal.yaml
- helm lint --strict script/helm/garage -f script/helm/garage/complex-values.yaml
- helm unittest --strict script/helm/garage
Generated
+179
View File
@@ -272,6 +272,29 @@ dependencies = [
"zeroize",
]
[[package]]
name = "aws-lc-rs"
version = "1.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e"
dependencies = [
"aws-lc-sys",
"zeroize",
]
[[package]]
name = "aws-lc-sys"
version = "0.44.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f09fae7be8bb3174e05c6afdb34199e6dc0c7c04ba9fa237b1967adfbde27483"
dependencies = [
"cc",
"cmake",
"dunce",
"fs_extra",
"pkg-config",
]
[[package]]
name = "aws-runtime"
version = "1.7.3"
@@ -762,6 +785,17 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
[[package]]
name = "chacha20"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06"
dependencies = [
"cfg-if",
"cpufeatures 0.3.0",
"rand_core 0.10.1",
]
[[package]]
name = "chrono"
version = "0.4.44"
@@ -837,6 +871,15 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
[[package]]
name = "cmake"
version = "0.1.58"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678"
dependencies = [
"cc",
]
[[package]]
name = "cmov"
version = "0.5.3"
@@ -1206,6 +1249,12 @@ dependencies = [
"phf",
]
[[package]]
name = "dunce"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
[[package]]
name = "dyn-clone"
version = "1.0.20"
@@ -1399,6 +1448,12 @@ dependencies = [
name = "format_table"
version = "0.1.1"
[[package]]
name = "fs_extra"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fundu"
version = "2.0.1"
@@ -1944,8 +1999,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
dependencies = [
"cfg-if",
"js-sys",
"libc",
"wasi",
"wasm-bindgen",
]
[[package]]
@@ -1967,10 +2024,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555"
dependencies = [
"cfg-if",
"js-sys",
"libc",
"r-efi 6.0.0",
"rand_core 0.10.1",
"wasip2",
"wasip3",
"wasm-bindgen",
]
[[package]]
@@ -2833,6 +2893,7 @@ dependencies = [
"hyper-util",
"log",
"percent-encoding",
"reqwest",
"serde",
"serde_json",
"sha2 0.10.9",
@@ -3089,6 +3150,12 @@ dependencies = [
"hashbrown 0.16.1",
]
[[package]]
name = "lru-slab"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
[[package]]
name = "lsm-tree"
version = "2.10.4"
@@ -3845,6 +3912,63 @@ dependencies = [
"hashbrown 0.16.1",
]
[[package]]
name = "quinn"
version = "0.11.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8"
dependencies = [
"bytes",
"cfg_aliases 0.2.1",
"pin-project-lite",
"quinn-proto",
"quinn-udp",
"rustc-hash",
"rustls 0.23.39",
"socket2 0.5.10",
"thiserror 2.0.18",
"tokio",
"tracing",
"web-time",
]
[[package]]
name = "quinn-proto"
version = "0.11.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83"
dependencies = [
"aws-lc-rs",
"bytes",
"getrandom 0.4.2",
"lru-slab",
"rand 0.10.2",
"rand_pcg",
"ring",
"rustc-hash",
"rustls 0.23.39",
"rustls-pki-types",
"slab",
"thiserror 2.0.18",
"tinyvec",
"tracing",
"web-time",
]
[[package]]
name = "quinn-udp"
version = "0.5.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694"
dependencies = [
"cfg_aliases 0.2.1",
"libc",
"once_cell",
"socket2 0.5.10",
"tracing",
"windows-sys 0.52.0",
]
[[package]]
name = "quote"
version = "1.0.45"
@@ -3909,6 +4033,17 @@ dependencies = [
"rand_core 0.9.5",
]
[[package]]
name = "rand"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80"
dependencies = [
"chacha20",
"getrandom 0.4.2",
"rand_core 0.10.1",
]
[[package]]
name = "rand_chacha"
version = "0.3.1"
@@ -3947,6 +4082,21 @@ dependencies = [
"getrandom 0.3.4",
]
[[package]]
name = "rand_core"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
[[package]]
name = "rand_pcg"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a"
dependencies = [
"rand_core 0.10.1",
]
[[package]]
name = "redox_syscall"
version = "0.5.18"
@@ -4030,6 +4180,7 @@ dependencies = [
"log",
"percent-encoding",
"pin-project-lite",
"quinn",
"rustls 0.23.39",
"rustls-pki-types",
"rustls-platform-verifier",
@@ -4179,6 +4330,7 @@ version = "0.23.39"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c2c118cb077cca2822033836dfb1b975355dfb784b5e8da48f7b6c5db74e60e"
dependencies = [
"aws-lc-rs",
"log",
"once_cell",
"ring",
@@ -4206,6 +4358,7 @@ version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd"
dependencies = [
"web-time",
"zeroize",
]
@@ -4252,6 +4405,7 @@ version = "0.103.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
dependencies = [
"aws-lc-rs",
"ring",
"rustls-pki-types",
"untrusted",
@@ -4893,6 +5047,21 @@ dependencies = [
"zerovec",
]
[[package]]
name = "tinyvec"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa"
dependencies = [
"tinyvec_macros",
]
[[package]]
name = "tinyvec_macros"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
version = "1.52.1"
@@ -5534,6 +5703,16 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "web-time"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "webpki-root-certs"
version = "1.0.7"
+1 -5
View File
@@ -117,11 +117,7 @@ kube = { version = "3.0", default-features = false, features = [
"rustls-tls",
] }
schemars = "1.2"
reqwest = { version = "0.13", default-features = false, features = [
"rustls-no-provider",
"json",
] }
reqwest = { version = "0.13", default-features = false }
form_urlencoded = "1.2"
http = "1.4"
httpdate = "1.0"
+2
View File
@@ -21,3 +21,5 @@
.idea/
*.tmproj
.vscode/
# helm-unittest test suites
tests/
+4 -2
View File
@@ -2,7 +2,7 @@ apiVersion: v2
name: garage
description: S3-compatible object store for small self-hosted geo-distributed deployments
type: application
version: 0.9.4
version: 0.10.0
appVersion: "v2.4.0"
home: https://garagehq.deuxfleurs.fr/
icon: https://garagehq.deuxfleurs.fr/images/garage-logo.svg
@@ -15,4 +15,6 @@ keywords:
sources:
- https://git.deuxfleurs.fr/Deuxfleurs/garage.git
maintainers: []
maintainers:
- name: Garage maintainer team
email: garagehq@deuxfleurs.fr
+331
View File
@@ -0,0 +1,331 @@
# An "everything and the kitchen sink" values file for the helm chart: combines many non-default
# settings at once, including examples for the fields that default to empty in
# values.yaml and are therefore hard to guess the expected shape of.
#
# Aside the documentation value, it doubles as an integration-test fixture:
# CI renders and lints the chart with this file (see .woodpecker/debug.yaml)
# to catch feature interactions that per-feature fixtures wouldn't exercise together
# (e.g. both ingresses enabled at once, monitoring + custom service account,
# a DaemonSet-incompatible field set alongside a StatefulSet, ...).
#
# Try it locally with:
# helm template script/helm/garage -f script/helm/garage/complex-values.yaml
# helm lint --strict script/helm/garage -f script/helm/garage/complex-values.yaml
# -- Additional labels to add to all resources created by this chart
commonLabels:
app.kubernetes.io/part-of: storage
team: platform-infrastructure
# Garage configuration. Values under this are written to garage.toml
garage:
# -- sqlite for durability, lmdb for performance
# https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#db_engine
dbEngine: "sqlite"
# -- Here set to 10MiB
# An increase can result in better performance in certain scenarios
# https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#block_size
blockSize: "10485760"
# https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#replication_factor
replicationFactor: "5"
# https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#consistency_mode
consistencyMode: "dangerous"
# -- zstd compression level of stored blocks
# https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#compression_level
compressionLevel: "5"
# -- If this value is set, Garage will automatically take a snapshot of the metadata DB file and save it in the metadata directory.
# https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#metadata_auto_snapshot_interval
metadataAutoSnapshotInterval: "30 days"
rpcBindAddr: "[::]:3901"
# -- If not given, a random secret will be generated and stored in a Secret object
rpcSecret: ""
# -- If you want to provide an rpcSecret within an existing k8s secret,
# specify the secret name here, and store the value under the secret key `rpcSecret`
# ! the default secret will not be created
existingRpcSecret: ""
# -- This is not required if you use the integrated kubernetes discovery. Each
# entry is "<garage_node_id>@<host>:<port>", where <garage_node_id> is the node's public key
# (shown by `garage node id` on that node).
bootstrapPeers:
- "563e1ac825ee3323aa441e72c26d1030d6d4222c43c986812dbf7cd47d18aef@garage-0.garage-headless:3901"
- "86f0f26ae4afbd59aaf9cfb302af3fe0464f2f7b5b21f80f7e6f4e9989b5c1f8@garage-1.garage-headless:3901"
# -- Set to true if you want to use k8s discovery but install the CRDs manually outside
# of the helm chart, for example if you operate at namespace level without cluster resources
kubernetesSkipCrd: true
s3:
api:
bindAddr: "[::]:3900"
region: "garage"
rootDomain: ".s3.garage.tld"
web:
bindAddr: "[::]:3902"
rootDomain: ".web.garage.tld"
index: "index.html"
admin:
apiBindAddr: "[::]:3903"
# -- Additional configuration to append to garage.toml. Use a multi-line string for custom config.
additionalTopLevelConfig: |-
data_fsync = true
# -- if not empty string, allow using an existing ConfigMap for the garage.toml,
# if set, ignores garage.toml
existingConfigMap: ""
# -- String Template for the garage configuration.
# if set, ignores every other garage.* value above and is rendered with `tpl`,
# so it can reference .Values/.Release/.Chart, e.g.:
# garageTomlString: |-
# metadata_dir = "/mnt/meta"
# data_dir = "/mnt/data"
# replication_factor = {{ .Values.garage.replicationFactor }}
# rpc_bind_addr = "{{ .Values.garage.rpcBindAddr }}"
# rpc_secret = "__RPC_SECRET_REPLACE__"
# [kubernetes_discovery]
# namespace = "{{ .Release.Namespace }}"
# service_name = "{{ include "garage.fullname" . }}"
# A rendering-verified version of this example lives in tests/configmap_test.yaml.
garageTomlString: ""
# Data persistence
persistence:
enabled: true
meta:
storageClass: "fast-ssd"
size: 100Mi
# used only for daemon sets
hostPath: /var/lib/garage/meta
data:
storageClass: "standard"
size: 100Mi
# used only for daemon sets
hostPath: /var/lib/garage/data
# Deployment configuration
deployment:
# -- Switchable to DaemonSet
kind: StatefulSet
# -- Number of StatefulSet replicas/garage nodes to start
replicaCount: 3
# -- If using statefulset, allow Parallel or OrderedReady (default)
podManagementPolicy: OrderedReady
image:
# -- default to amd64 docker image
repository: dxflrs/amd64_garage
# -- set the image tag, please prefer using the chart version and not this
# to avoid compatibility issues
tag: ""
pullPolicy: IfNotPresent
initImage:
repository: busybox
tag: stable
pullPolicy: IfNotPresent
# -- set if you need credentials to pull your custom image. Each entry needs a
# `name:` key, matching a Secret of type kubernetes.io/dockerconfigjson.
imagePullSecrets:
- name: my-pull-secret
nameOverride: ""
fullnameOverride: ""
serviceAccount:
# -- Specifies whether a service account should be created
create: true
# -- Annotations to add to the service account. Example below is for AWS IRSA.
annotations:
eks.amazonaws.com/role-arn: "arn:aws:iam::123456789012:role/garage-s3"
# -- The name of the service account to use.
# If not set and create is true, a name is generated using the fullname template
name: ""
# -- additional pod annotations
podAnnotations:
example.com/has-an-annotation: "true"
podSecurityContext:
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
fsGroupChangePolicy: "OnRootMismatch"
runAsNonRoot: true
securityContext:
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
service:
# -- You can rely on any service to expose your cluster
# - ClusterIP (+ Ingress)
# - NodePort (+ Ingress)
# - LoadBalancer
type: ClusterIP
# -- Annotations to add to the service. Example below is for an AWS NLB.
annotations:
service.beta.kubernetes.io/aws-load-balancer-type: "nlb"
s3:
api:
port: 3900
web:
port: 3902
# NOTE: the admin API is excluded for now as it is not consistent across nodes
ingress:
s3:
api:
enabled: true
className: "nginx"
annotations:
cert-manager.io/cluster-issuer: "letsencrypt-prod"
labels: {}
hosts:
# -- garage S3 API endpoint, to be used with awscli for example
- host: "s3.garage.tld"
paths:
- path: /
pathType: Prefix
# -- garage S3 API endpoint, DNS style bucket access
- host: "*.s3.garage.tld"
paths:
- path: /
pathType: Prefix
tls:
- secretName: garage-s3-api-tls
hosts:
- s3.garage.tld
- "*.s3.garage.tld"
web:
enabled: true
className: "nginx"
annotations:
cert-manager.io/cluster-issuer: "letsencrypt-prod"
labels: {}
hosts:
# -- wildcard website access with bucket name prefix
- host: "*.web.garage.tld"
paths:
- path: /
pathType: Prefix
# -- specific bucket access with FQDN bucket
- host: "mywebpage.example.com"
paths:
- path: /
pathType: Prefix
tls:
- secretName: garage-s3-web-tls
hosts:
- "*.web.garage.tld"
- mywebpage.example.com
# The following are indicative for a small-size deployment, for anything serious double them.
resources:
limits:
cpu: 200m
memory: 2048Mi
requests:
cpu: 100m
memory: 1024Mi
# -- Specifies a livenessProbe
livenessProbe:
httpGet:
path: /health
port: 3903 # or the port from garage.admin.apiBindAddr
initialDelaySeconds: 5
periodSeconds: 30
# -- Specifies a readinessProbe
readinessProbe:
httpGet:
path: /health
port: 3903 # or the port from garage.admin.apiBindAddr
initialDelaySeconds: 5
periodSeconds: 30
# -- Example: pin pods to a dedicated storage node pool, paired with the
# toleration below.
nodeSelector:
node-role.kubernetes.io/storage: "true"
tolerations:
- key: "dedicated"
operator: "Equal"
value: "storage"
effect: "NoSchedule"
# -- Example: spread garage replicas across different nodes, since it is a
# geo-distributed store that only helps availability if replicas don't share
# a failure domain or availability zone.
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchLabels:
app.kubernetes.io/name: garage
topologyKey: kubernetes.io/hostname
# -- Optional priority class name to assign to the pods.
# See https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/
# This is expected to reference a PriorityClass you define yourself.
priorityClassName: "high-priority-storage"
# -- Extra container env vars. Note this is a [] of {name, value} objects (ie. a pod env stanza)
# GARAGE_ADMIN_TOKEN_FILE below points garage at the token file mounted by
# extraVolumes/extraVolumeMounts, see below.
environment:
- name: RUST_LOG
value: "garage=debug"
- name: GARAGE_ADMIN_TOKEN_FILE
value: /mnt/secrets-store/admin-token
# -- Extra volumes/volumeMounts. Both are []. Example here mounts the admin API
# token from an external secrets manager via the Secrets Store CSI driver
# (https://secrets-store-csi-driver.sigs.k8s.io/) instead of a Secret volume.
# This allows, for example, providing the tokens without creating a Kubernetes
# secret. garage reads the mounted file through GARAGE_ADMIN_TOKEN_FILE above.
extraVolumes:
- name: secrets-store
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: garage-admin-token
extraVolumeMounts:
- name: secrets-store
mountPath: /mnt/secrets-store
readOnly: true
monitoring:
metrics:
# -- If true, a service for monitoring is created with a prometheus.io/scrape annotation
enabled: true
serviceMonitor:
# -- If true, a ServiceMonitor CRD is created for a prometheus operator
# https://github.com/coreos/prometheus-operator
enabled: true
path: /metrics
# -- Defaults to the namespace the chart is deployed to; this field is
# templated, so it can also reference .Release.Namespace itself.
namespace: "monitoring"
labels:
release: prometheus
interval: 30s
scheme: http
tlsConfig: {}
scrapeTimeout: 10s
relabelings:
- sourceLabels: ["__meta_kubernetes_pod_node_name"]
targetLabel: node
tracing:
# -- specify a sink endpoint for OpenTelemetry Traces, eg. `http://localhost:4317`
sink: "http://otel-collector.monitoring.svc:4317"
@@ -62,7 +62,9 @@ spec:
{{- end }}
{{- end }}
{{- end }}
{{- if and .Values.ingress.s3.api.enabled .Values.ingress.s3.web.enabled }}
---
{{ end }}
{{- if .Values.ingress.s3.web.enabled -}}
{{- $fullName := include "garage.fullname" . -}}
{{- $svcPort := .Values.service.s3.web.port -}}
@@ -28,11 +28,11 @@ spec:
scheme: {{ .Values.monitoring.metrics.serviceMonitor.scheme }}
{{- with .Values.monitoring.metrics.serviceMonitor.tlsConfig }}
tlsConfig:
{{- toYaml . | nindent 6 }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.monitoring.metrics.serviceMonitor.relabelings }}
relabelings:
{{- toYaml . | nindent 6 }}
{{- toYaml . | nindent 8 }}
{{- end }}
jobLabel: "{{ .Release.Name }}"
selector:
@@ -41,4 +41,4 @@ spec:
namespaceSelector:
matchNames:
- {{ .Release.Namespace }}
{{- end }}
{{- end }}
@@ -0,0 +1,31 @@
suite: rbac
templates:
- templates/clusterrole.yaml
tests:
- it: allows managing the garage CRD by default
asserts:
- hasDocuments:
count: 2
- documentIndex: 0
isKind:
of: ClusterRole
- documentIndex: 0
contains:
path: rules[0].resources
content: customresourcedefinitions
- documentIndex: 1
isKind:
of: ClusterRoleBinding
- documentIndex: 1
equal:
path: subjects[0].name
value: RELEASE-NAME-garage
- it: skips the CRD management rule when the CRD is installed manually
set:
garage.kubernetesSkipCrd: true
asserts:
- documentIndex: 0
notContains:
path: rules[0].resources
content: customresourcedefinitions
@@ -0,0 +1,82 @@
# Integration-style suite: renders the whole chart with complex-values.yaml
# (many non-default features combined at once) and checks that they don't
# clobber each other, rather than testing any single feature in isolation
# (that's what the other tests/*_test.yaml suites are for).
suite: complex-values integration
templates:
- templates/workload.yaml
- templates/service.yaml
- templates/service-headless.yaml
- templates/ingress.yaml
- templates/servicemonitor.yaml
- templates/serviceaccount.yaml
- templates/configmap.yaml
- templates/clusterrole.yaml
tests:
- it: renders a self-consistent deployment with every optional feature enabled
values:
- ../complex-values.yaml
asserts:
- template: templates/workload.yaml
isKind:
of: StatefulSet
- template: templates/workload.yaml
equal:
path: metadata.labels.team
value: platform-infrastructure
- template: templates/workload.yaml
equal:
path: spec.template.spec.containers[0].env[0].name
value: RUST_LOG
- template: templates/workload.yaml
contains:
path: spec.template.spec.volumes
content:
name: secrets-store
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: garage-admin-token
- template: templates/workload.yaml
equal:
path: spec.template.spec.containers[0].env[1].name
value: GARAGE_ADMIN_TOKEN_FILE
- template: templates/workload.yaml
equal:
path: spec.volumeClaimTemplates[0].spec.storageClassName
value: fast-ssd
- template: templates/workload.yaml
contains:
path: spec.template.spec.imagePullSecrets
content:
name: my-pull-secret
- template: templates/service.yaml
hasDocuments:
count: 2 # main service + metrics service, since monitoring.metrics.enabled is true here
- template: templates/service-headless.yaml
hasDocuments:
count: 1 # StatefulSet still gets a headless service
- template: templates/ingress.yaml
hasDocuments:
count: 2 # both s3 api and s3 web ingresses enabled together
- template: templates/servicemonitor.yaml
hasDocuments:
count: 1
- template: templates/servicemonitor.yaml
equal:
path: spec.endpoints[0].relabelings[0].targetLabel
value: node
- template: templates/serviceaccount.yaml
equal:
path: metadata.annotations["eks.amazonaws.com/role-arn"]
value: "arn:aws:iam::123456789012:role/garage-s3"
- template: templates/configmap.yaml
matchRegex:
path: data["garage.toml"]
pattern: 'data_fsync = true'
- template: templates/clusterrole.yaml
documentIndex: 0
notContains:
path: rules[0].resources
content: customresourcedefinitions # garage.kubernetesSkipCrd is true here
@@ -0,0 +1,136 @@
suite: configmap
templates:
- templates/configmap.yaml
tests:
- it: renders garage.toml with the default configuration
asserts:
- hasDocuments:
count: 1
- isKind:
of: ConfigMap
- equal:
path: metadata.name
value: RELEASE-NAME-garage-config
- matchRegex:
path: data["garage.toml"]
pattern: 'metadata_dir = "/mnt/meta"'
- matchRegex:
path: data["garage.toml"]
pattern: 'data_dir = "/mnt/data"'
- matchRegex:
path: data["garage.toml"]
pattern: 'db_engine = "lmdb"'
- matchRegex:
path: data["garage.toml"]
pattern: 'block_size = "1048576"'
- matchRegex:
path: data["garage.toml"]
pattern: 'replication_factor = 3'
- matchRegex:
path: data["garage.toml"]
pattern: 'consistency_mode = "consistent"'
- matchRegex:
path: data["garage.toml"]
pattern: 'compression_level = 1'
- matchRegex:
path: data["garage.toml"]
pattern: 'rpc_bind_addr = "\[::\]:3901"'
- matchRegex:
path: data["garage.toml"]
pattern: 'rpc_secret = "__RPC_SECRET_REPLACE__"'
- matchRegex:
path: data["garage.toml"]
pattern: '(?s)\[kubernetes_discovery\]\s*namespace = "NAMESPACE"\s*service_name = "RELEASE-NAME-garage"\s*skip_crd = false'
- matchRegex:
path: data["garage.toml"]
pattern: '(?s)\[s3_api\]\s*s3_region = "garage"\s*api_bind_addr = "\[::\]:3900"\s*root_domain = "\.s3\.garage\.tld"'
- matchRegex:
path: data["garage.toml"]
pattern: '(?s)\[s3_web\]\s*bind_addr = "\[::\]:3902"\s*root_domain = "\.web\.garage\.tld"\s*index = "index.html"'
- matchRegex:
path: data["garage.toml"]
pattern: '(?s)\[admin\]\s*api_bind_addr = "\[::\]:3903"'
- notMatchRegex:
path: data["garage.toml"]
pattern: 'metadata_auto_snapshot_interval'
- notMatchRegex:
path: data["garage.toml"]
pattern: 'trace_sink'
- it: reflects custom garage settings, bootstrap peers and additional config
set:
garage.dbEngine: sqlite
garage.blockSize: "2097152"
garage.replicationFactor: "5"
garage.consistencyMode: degraded
garage.compressionLevel: "3"
garage.metadataAutoSnapshotInterval: 6h
garage.bootstrapPeers:
- abc@peer1:3901
- def@peer2:3901
garage.additionalTopLevelConfig: "data_fsync = true"
monitoring.tracing.sink: http://otel:4317
asserts:
- matchRegex:
path: data["garage.toml"]
pattern: 'db_engine = "sqlite"'
- matchRegex:
path: data["garage.toml"]
pattern: 'block_size = "2097152"'
- matchRegex:
path: data["garage.toml"]
pattern: 'replication_factor = 5'
- matchRegex:
path: data["garage.toml"]
pattern: 'consistency_mode = "degraded"'
- matchRegex:
path: data["garage.toml"]
pattern: 'compression_level = 3'
- matchRegex:
path: data["garage.toml"]
pattern: 'metadata_auto_snapshot_interval = "6h"'
- matchRegex:
path: data["garage.toml"]
pattern: 'bootstrap_peers = \["abc@peer1:3901"\s*, "def@peer2:3901"'
- matchRegex:
path: data["garage.toml"]
pattern: 'data_fsync = true'
- matchRegex:
path: data["garage.toml"]
pattern: 'trace_sink = "http://otel:4317"'
- it: uses garageTomlString verbatim when set, ignoring the structured values
set:
garage.garageTomlString: |-
metadata_dir = "/custom/meta"
replication_factor = 1
garage.dbEngine: sqlite
asserts:
- equal:
path: data["garage.toml"]
value: |-
metadata_dir = "/custom/meta"
replication_factor = 1
- notMatchRegex:
path: data["garage.toml"]
pattern: 'db_engine'
- it: templates garageTomlString against the release and values context
set:
garage.garageTomlString: |-
# namespace: {{ .Release.Namespace }}
replication_factor = {{ .Values.garage.replicationFactor }}
garage.replicationFactor: "7"
asserts:
- equal:
path: data["garage.toml"]
value: |-
# namespace: NAMESPACE
replication_factor = 7
- it: does not render a ConfigMap when an existing one is referenced
set:
garage.existingConfigMap: my-external-cm
asserts:
- hasDocuments:
count: 0
@@ -0,0 +1,95 @@
suite: ingress
templates:
- templates/ingress.yaml
tests:
- it: renders no ingress by default
asserts:
- hasDocuments:
count: 0
- it: renders api and web ingresses with tls when enabled
values:
- ./values/ingress.yaml
asserts:
- hasDocuments:
count: 2
- isKind:
of: Ingress
- documentIndex: 0
equal:
path: metadata.name
value: RELEASE-NAME-garage-s3-api
- documentIndex: 0
equal:
path: spec.ingressClassName
value: nginx
- documentIndex: 0
equal:
path: spec.rules[0].host
value: s3.example.com
- documentIndex: 0
equal:
path: spec.tls[0].secretName
value: garage-s3-api-tls
- documentIndex: 1
equal:
path: metadata.name
value: RELEASE-NAME-garage-s3-web
- documentIndex: 1
equal:
path: spec.rules[0].host
value: "*.web.example.com"
- documentIndex: 1
equal:
path: spec.tls[0].secretName
value: garage-s3-web-tls
- it: can enable only the s3 api ingress
set:
ingress.s3.api.enabled: true
ingress.s3.api.hosts[0].host: s3.example.com
ingress.s3.api.hosts[0].paths[0].path: /
ingress.s3.api.hosts[0].paths[0].pathType: Prefix
asserts:
- hasDocuments:
count: 1
- equal:
path: metadata.name
value: RELEASE-NAME-garage-s3-api
- it: omits ingressClassName and tls when neither is configured
set:
ingress.s3.api.enabled: true
ingress.s3.api.hosts[0].host: s3.example.com
ingress.s3.api.hosts[0].paths[0].path: /
ingress.s3.api.hosts[0].paths[0].pathType: Prefix
asserts:
- isNull:
path: spec.ingressClassName
- isNull:
path: spec.tls
- it: renders multiple hosts on the same ingress
set:
ingress.s3.api.enabled: true
ingress.s3.api.hosts:
- host: s3.example.com
paths:
- path: /
pathType: Prefix
- host: s3-alt.example.com
paths:
- path: /
pathType: Prefix
asserts:
- hasDocuments:
count: 1
- lengthEqual:
path: spec.rules
count: 2
- equal:
path: spec.rules[0].host
value: s3.example.com
- equal:
path: spec.rules[1].host
value: s3-alt.example.com
+56
View File
@@ -0,0 +1,56 @@
suite: naming and common labels
templates:
- templates/workload.yaml
- templates/configmap.yaml
tests:
- it: applies commonLabels alongside the default chart labels
template: templates/workload.yaml
set:
commonLabels:
team: storage
asserts:
- equal:
path: metadata.labels.team
value: storage
- equal:
path: metadata.labels["app.kubernetes.io/managed-by"]
value: Helm
- it: uses fullnameOverride verbatim for resource names
template: templates/workload.yaml
set:
fullnameOverride: my-garage-cluster
asserts:
- equal:
path: metadata.name
value: my-garage-cluster
- equal:
path: spec.serviceName
value: my-garage-cluster-headless
- it: does not double-prefix when the release name already contains the chart name
template: templates/workload.yaml
release:
name: garage
asserts:
- equal:
path: metadata.name
value: garage
- it: prefixes the release name with the chart name otherwise
template: templates/workload.yaml
release:
name: prod
asserts:
- equal:
path: metadata.name
value: prod-garage
- it: truncates an overly long fullname to 63 characters and trims a trailing dash
template: templates/workload.yaml
set:
fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-suffix-that-will-be-cut-off
asserts:
- equal:
path: metadata.name
value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+33
View File
@@ -0,0 +1,33 @@
suite: rpc secret
templates:
- templates/secret.yaml
tests:
- it: generates a Secret holding the rpc secret by default
asserts:
- hasDocuments:
count: 1
- isKind:
of: Secret
- equal:
path: metadata.name
value: RELEASE-NAME-garage-rpc-secret
- equal:
path: type
value: Opaque
- isNotNull:
path: data.rpcSecret
- it: does not render a Secret when an existing one is referenced
values:
- ./values/existing-secret.yaml
asserts:
- hasDocuments:
count: 0
- it: base64-encodes an explicitly provided rpc secret
set:
garage.rpcSecret: my-plain-secret
asserts:
- equal:
path: data.rpcSecret
value: bXktcGxhaW4tc2VjcmV0
@@ -0,0 +1,26 @@
suite: headless service
templates:
- templates/service-headless.yaml
tests:
- it: creates a headless service for a StatefulSet by default
asserts:
- hasDocuments:
count: 1
- isKind:
of: Service
- equal:
path: metadata.name
value: RELEASE-NAME-garage-headless
- equal:
path: spec.clusterIP
value: None
- equal:
path: spec.type
value: ClusterIP
- it: does not create a headless service for a DaemonSet
values:
- ./values/daemonset.yaml
asserts:
- hasDocuments:
count: 0
@@ -0,0 +1,61 @@
suite: service
templates:
- templates/service.yaml
tests:
- it: creates a ClusterIP service with s3-api and s3-web ports by default
asserts:
- hasDocuments:
count: 1
- isKind:
of: Service
- equal:
path: spec.type
value: ClusterIP
- equal:
path: spec.ports[0].name
value: s3-api
- equal:
path: spec.ports[0].port
value: 3900
- equal:
path: spec.ports[1].name
value: s3-web
- equal:
path: spec.ports[1].port
value: 3902
- it: honors a custom service type and port
set:
service.type: LoadBalancer
service.s3.api.port: 9000
asserts:
- equal:
path: spec.type
value: LoadBalancer
- equal:
path: spec.ports[0].port
value: 9000
- it: does not create a metrics service by default
asserts:
- hasDocuments:
count: 1
- it: adds a headless metrics service when monitoring is enabled
values:
- ./values/monitoring.yaml
asserts:
- hasDocuments:
count: 2
- documentIndex: 1
equal:
path: metadata.name
value: RELEASE-NAME-garage-metrics
- documentIndex: 1
equal:
path: spec.clusterIP
value: None
- documentIndex: 1
equal:
path: metadata.annotations["prometheus.io/scrape"]
value: "true"
@@ -0,0 +1,28 @@
suite: service account
templates:
- templates/serviceaccount.yaml
tests:
- it: creates a ServiceAccount by default
asserts:
- hasDocuments:
count: 1
- isKind:
of: ServiceAccount
- equal:
path: metadata.name
value: RELEASE-NAME-garage
- it: does not create a ServiceAccount when disabled
values:
- ./values/minimal.yaml
asserts:
- hasDocuments:
count: 0
- it: honors a custom service account name
set:
serviceAccount.name: my-garage-sa
asserts:
- equal:
path: metadata.name
value: my-garage-sa
@@ -0,0 +1,65 @@
suite: service monitor
templates:
- templates/servicemonitor.yaml
tests:
- it: renders no ServiceMonitor by default
asserts:
- hasDocuments:
count: 0
- it: renders no ServiceMonitor when only metrics are enabled
set:
monitoring.metrics.enabled: true
asserts:
- hasDocuments:
count: 0
- it: renders a ServiceMonitor when explicitly enabled
values:
- ./values/monitoring.yaml
asserts:
- hasDocuments:
count: 1
- isKind:
of: ServiceMonitor
- equal:
path: metadata.name
value: RELEASE-NAME-garage
- equal:
path: metadata.namespace
value: NAMESPACE
- equal:
path: spec.endpoints[0].interval
value: 30s
- it: templates a custom namespace against the release context
values:
- ./values/monitoring.yaml
set:
monitoring.metrics.serviceMonitor.namespace: "{{ .Release.Namespace }}-monitoring"
asserts:
- equal:
path: metadata.namespace
value: NAMESPACE-monitoring
- it: applies custom labels, tlsConfig and relabelings
values:
- ./values/monitoring.yaml
set:
monitoring.metrics.serviceMonitor.labels:
team: storage
monitoring.metrics.serviceMonitor.tlsConfig:
insecureSkipVerify: true
monitoring.metrics.serviceMonitor.relabelings:
- sourceLabels: ["__meta_kubernetes_pod_name"]
targetLabel: pod
asserts:
- equal:
path: metadata.labels.team
value: storage
- equal:
path: spec.endpoints[0].tlsConfig.insecureSkipVerify
value: true
- equal:
path: spec.endpoints[0].relabelings[0].targetLabel
value: pod
@@ -0,0 +1,10 @@
# Run garage as a DaemonSet (one pod per node) instead of the default StatefulSet,
# using hostPath volumes for meta/data persistence.
deployment:
kind: DaemonSet
persistence:
enabled: true
meta:
hostPath: /var/lib/garage/meta
data:
hostPath: /var/lib/garage/data
@@ -0,0 +1,5 @@
# Use a pre-existing Kubernetes Secret for the RPC secret instead of letting
# the chart generate/manage one.
garage:
rpcSecret: ""
existingRpcSecret: "garage-rpc-secret-external"
@@ -0,0 +1,27 @@
# Expose both the S3 API and website endpoints through Ingress, with TLS.
ingress:
s3:
api:
enabled: true
className: "nginx"
hosts:
- host: "s3.example.com"
paths:
- path: /
pathType: Prefix
tls:
- secretName: garage-s3-api-tls
hosts:
- s3.example.com
web:
enabled: true
className: "nginx"
hosts:
- host: "*.web.example.com"
paths:
- path: /
pathType: Prefix
tls:
- secretName: garage-s3-web-tls
hosts:
- "*.web.example.com"
@@ -0,0 +1,8 @@
# Minimal single-node deployment without persistent storage or a dedicated
# service account, e.g. for local testing.
deployment:
replicaCount: 1
persistence:
enabled: false
serviceAccount:
create: false
@@ -0,0 +1,7 @@
# Enable Prometheus metrics scraping and a ServiceMonitor for the prometheus-operator.
monitoring:
metrics:
enabled: true
serviceMonitor:
enabled: true
interval: 30s
+187
View File
@@ -0,0 +1,187 @@
suite: workload (StatefulSet/DaemonSet)
templates:
- templates/workload.yaml
- templates/configmap.yaml
tests:
- it: defaults to a StatefulSet with 3 replicas and 2 volumes
template: templates/workload.yaml
asserts:
- isKind:
of: StatefulSet
- equal:
path: spec.replicas
value: 3
- equal:
path: spec.podManagementPolicy
value: OrderedReady
- equal:
path: spec.template.spec.volumes[1].name
value: etc
- lengthEqual:
path: spec.template.spec.volumes
count: 2
- isNotNull:
path: spec.volumeClaimTemplates
- it: uses a StatefulSet with PVC-backed volumeClaimTemplates by default
template: templates/workload.yaml
asserts:
- isKind:
of: StatefulSet
- isNotNull:
path: spec.volumeClaimTemplates
- equal:
path: spec.volumeClaimTemplates[0].spec.resources.requests.storage
value: 100Mi
- it: switches to a DaemonSet with hostPath volumes when requested
template: templates/workload.yaml
values:
- ./values/daemonset.yaml
asserts:
- isKind:
of: DaemonSet
- isNull:
path: spec.replicas
- isNull:
path: spec.volumeClaimTemplates
- contains:
path: spec.template.spec.volumes
content:
name: meta
hostPath:
path: /var/lib/garage/meta
type: DirectoryOrCreate
- contains:
path: spec.template.spec.volumes
content:
name: data
hostPath:
path: /var/lib/garage/data
type: DirectoryOrCreate
- it: renders emptyDir volumes when persistence is disabled
template: templates/workload.yaml
values:
- ./values/minimal.yaml
asserts:
- contains:
path: spec.template.spec.volumes
content:
name: meta
emptyDir: {}
- contains:
path: spec.template.spec.volumes
content:
name: data
emptyDir: {}
- isNull:
path: spec.volumeClaimTemplates
- it: honors a custom replicaCount
template: templates/workload.yaml
set:
deployment.replicaCount: 5
asserts:
- equal:
path: spec.replicas
value: 5
- it: points the init container at the configured rpc secret
template: templates/workload.yaml
asserts:
- equal:
path: spec.template.spec.initContainers[0].env[0].valueFrom.secretKeyRef.name
value: RELEASE-NAME-garage-rpc-secret
- it: points the init container at an existing rpc secret when configured
template: templates/workload.yaml
values:
- ./values/existing-secret.yaml
asserts:
- equal:
path: spec.template.spec.initContainers[0].env[0].valueFrom.secretKeyRef.name
value: garage-rpc-secret-external
- it: sets the container image from repository and tag
template: templates/workload.yaml
set:
image.repository: dxflrs/amd64_garage
image.tag: v1.2.3
asserts:
- equal:
path: spec.template.spec.containers[0].image
value: dxflrs/amd64_garage:v1.2.3
- it: falls back to the chart appVersion when no image tag is set
template: templates/workload.yaml
asserts:
- matchRegex:
path: spec.template.spec.containers[0].image
pattern: ^dxflrs/amd64_garage:v
- it: omits storageClassName from volumeClaimTemplates by default
template: templates/workload.yaml
asserts:
- isNull:
path: spec.volumeClaimTemplates[0].spec.storageClassName
- isNull:
path: spec.volumeClaimTemplates[1].spec.storageClassName
- it: sets storageClassName in volumeClaimTemplates when configured
template: templates/workload.yaml
set:
persistence.meta.storageClass: fast-storage
persistence.data.storageClass: slow-storage
asserts:
- equal:
path: spec.volumeClaimTemplates[0].spec.storageClassName
value: fast-storage
- equal:
path: spec.volumeClaimTemplates[1].spec.storageClassName
value: slow-storage
- it: renders emptyDir volumes for a DaemonSet when persistence is disabled
template: templates/workload.yaml
set:
deployment.kind: DaemonSet
persistence.enabled: false
asserts:
- contains:
path: spec.template.spec.volumes
content:
name: meta
emptyDir: {}
- contains:
path: spec.template.spec.volumes
content:
name: data
emptyDir: {}
- it: mounts the existing ConfigMap volume when configured
template: templates/workload.yaml
set:
garage.existingConfigMap: my-external-cm
asserts:
- equal:
path: spec.template.spec.volumes[0].configMap.name
value: my-external-cm
- it: uses a custom service account name without creating one when disabled
template: templates/workload.yaml
set:
serviceAccount.create: false
serviceAccount.name: my-external-sa
asserts:
- equal:
path: spec.template.spec.serviceAccountName
value: my-external-sa
- it: falls back to the default service account when disabled without a custom name
template: templates/workload.yaml
set:
serviceAccount.create: false
asserts:
- equal:
path: spec.template.spec.serviceAccountName
value: default
+4
View File
@@ -246,10 +246,14 @@ affinity: {}
# See https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/
priorityClassName: ""
# -- Extra container env vars, as a list of {name, value} objects (same shape
# as a Pod container's env)
environment: {}
# -- Extra volumes, as a list of volume objects (same shape as a PodSpec's volumes)
extraVolumes: {}
# -- Extra volume mounts, as a list of mount objects (same shape as a container's volumeMounts)
extraVolumeMounts: {}
monitoring:
+1
View File
@@ -24,6 +24,7 @@ percent-encoding.workspace = true
hyper = { workspace = true, default-features = false, features = ["http1", "http2"] }
hyper-util.workspace = true
hyper-rustls.workspace = true
reqwest = { workspace = true, default-features = false, features = ["json", "rustls-no-provider"] }
serde = { workspace = true, default-features = false, features = ["derive", "std"] }
serde_json.workspace = true
thiserror.workspace = true
+1 -1
View File
@@ -40,7 +40,7 @@ thiserror = { workspace = true, optional = true }
kube = { workspace = true, optional = true }
k8s-openapi = { workspace = true, optional = true }
schemars = { workspace = true, optional = true }
reqwest = { workspace = true, optional = true }
reqwest = { workspace = true, optional = true, features = ["json", "rustls"] }
pnet_datalink.workspace = true