diff --git a/.woodpecker/debug.yaml b/.woodpecker/debug.yaml index 52894b91..df4563bb 100644 --- a/.woodpecker/debug.yaml +++ b/.woodpecker/debug.yaml @@ -53,3 +53,15 @@ steps: - nix-build -j4 --attr flakePackages.dev - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) depends_on: [ build ] + + - name: helm chart tests + image: helmunittest/helm-unittest:4.2.3-1.1.2 + commands: + - helm lint --strict script/helm/garage + - helm lint --strict script/helm/garage -f script/helm/garage/tests/values/daemonset.yaml + - helm lint --strict script/helm/garage -f script/helm/garage/tests/values/ingress.yaml + - helm lint --strict script/helm/garage -f script/helm/garage/tests/values/existing-secret.yaml + - helm lint --strict script/helm/garage -f script/helm/garage/tests/values/monitoring.yaml + - helm lint --strict script/helm/garage -f script/helm/garage/tests/values/minimal.yaml + - helm lint --strict script/helm/garage -f script/helm/garage/complex-values.yaml + - helm unittest --strict script/helm/garage diff --git a/script/helm/garage/.helmignore b/script/helm/garage/.helmignore index 0e8a0eb3..8e6782e3 100644 --- a/script/helm/garage/.helmignore +++ b/script/helm/garage/.helmignore @@ -21,3 +21,5 @@ .idea/ *.tmproj .vscode/ +# helm-unittest test suites +tests/ diff --git a/script/helm/garage/Chart.yaml b/script/helm/garage/Chart.yaml index 110ba8c7..3636de46 100644 --- a/script/helm/garage/Chart.yaml +++ b/script/helm/garage/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: garage description: S3-compatible object store for small self-hosted geo-distributed deployments type: application -version: 0.9.3 +version: 0.9.4 appVersion: "v2.3.0" home: https://garagehq.deuxfleurs.fr/ icon: https://garagehq.deuxfleurs.fr/images/garage-logo.svg @@ -15,4 +15,6 @@ keywords: sources: - https://git.deuxfleurs.fr/Deuxfleurs/garage.git -maintainers: [] +maintainers: +- name: Garage maintainer team + email: garagehq@deuxfleurs.fr diff --git a/script/helm/garage/complex-values.yaml b/script/helm/garage/complex-values.yaml new file mode 100644 index 00000000..7a9074a3 --- /dev/null +++ b/script/helm/garage/complex-values.yaml @@ -0,0 +1,331 @@ +# An "everything and the kitchen sink" values file for the helm chart: combines many non-default +# settings at once, including examples for the fields that default to empty in +# values.yaml and are therefore hard to guess the expected shape of. +# +# Aside the documentation value, it doubles as an integration-test fixture: +# CI renders and lints the chart with this file (see .woodpecker/debug.yaml) +# to catch feature interactions that per-feature fixtures wouldn't exercise together +# (e.g. both ingresses enabled at once, monitoring + custom service account, +# a DaemonSet-incompatible field set alongside a StatefulSet, ...). +# +# Try it locally with: +# helm template script/helm/garage -f script/helm/garage/complex-values.yaml +# helm lint --strict script/helm/garage -f script/helm/garage/complex-values.yaml + +# -- Additional labels to add to all resources created by this chart +commonLabels: + app.kubernetes.io/part-of: storage + team: platform-infrastructure + +# Garage configuration. Values under this are written to garage.toml +garage: + # -- sqlite for durability, lmdb for performance + # https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#db_engine + dbEngine: "sqlite" + + # -- Here set to 10MiB + # An increase can result in better performance in certain scenarios + # https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#block_size + blockSize: "10485760" + + # https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#replication_factor + replicationFactor: "5" + + # https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#consistency_mode + consistencyMode: "dangerous" + + # -- zstd compression level of stored blocks + # https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#compression_level + compressionLevel: "5" + + # -- If this value is set, Garage will automatically take a snapshot of the metadata DB file and save it in the metadata directory. + # https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/#metadata_auto_snapshot_interval + metadataAutoSnapshotInterval: "30 days" + + rpcBindAddr: "[::]:3901" + # -- If not given, a random secret will be generated and stored in a Secret object + rpcSecret: "" + # -- If you want to provide an rpcSecret within an existing k8s secret, + # specify the secret name here, and store the value under the secret key `rpcSecret` + # ! the default secret will not be created + existingRpcSecret: "" + # -- This is not required if you use the integrated kubernetes discovery. Each + # entry is "@:", where is the node's public key + # (shown by `garage node id` on that node). + bootstrapPeers: + - "563e1ac825ee3323aa441e72c26d1030d6d4222c43c986812dbf7cd47d18aef@garage-0.garage-headless:3901" + - "86f0f26ae4afbd59aaf9cfb302af3fe0464f2f7b5b21f80f7e6f4e9989b5c1f8@garage-1.garage-headless:3901" + # -- Set to true if you want to use k8s discovery but install the CRDs manually outside + # of the helm chart, for example if you operate at namespace level without cluster resources + kubernetesSkipCrd: true + s3: + api: + bindAddr: "[::]:3900" + region: "garage" + rootDomain: ".s3.garage.tld" + web: + bindAddr: "[::]:3902" + rootDomain: ".web.garage.tld" + index: "index.html" + admin: + apiBindAddr: "[::]:3903" + + # -- Additional configuration to append to garage.toml. Use a multi-line string for custom config. + additionalTopLevelConfig: |- + data_fsync = true + + # -- if not empty string, allow using an existing ConfigMap for the garage.toml, + # if set, ignores garage.toml + existingConfigMap: "" + + # -- String Template for the garage configuration. + # if set, ignores every other garage.* value above and is rendered with `tpl`, + # so it can reference .Values/.Release/.Chart, e.g.: + # garageTomlString: |- + # metadata_dir = "/mnt/meta" + # data_dir = "/mnt/data" + # replication_factor = {{ .Values.garage.replicationFactor }} + # rpc_bind_addr = "{{ .Values.garage.rpcBindAddr }}" + # rpc_secret = "__RPC_SECRET_REPLACE__" + # [kubernetes_discovery] + # namespace = "{{ .Release.Namespace }}" + # service_name = "{{ include "garage.fullname" . }}" + # A rendering-verified version of this example lives in tests/configmap_test.yaml. + garageTomlString: "" + +# Data persistence +persistence: + enabled: true + meta: + storageClass: "fast-ssd" + size: 100Mi + # used only for daemon sets + hostPath: /var/lib/garage/meta + data: + storageClass: "standard" + size: 100Mi + # used only for daemon sets + hostPath: /var/lib/garage/data + +# Deployment configuration +deployment: + # -- Switchable to DaemonSet + kind: StatefulSet + # -- Number of StatefulSet replicas/garage nodes to start + replicaCount: 3 + # -- If using statefulset, allow Parallel or OrderedReady (default) + podManagementPolicy: OrderedReady + +image: + # -- default to amd64 docker image + repository: dxflrs/amd64_garage + # -- set the image tag, please prefer using the chart version and not this + # to avoid compatibility issues + tag: "" + pullPolicy: IfNotPresent + +initImage: + repository: busybox + tag: stable + pullPolicy: IfNotPresent + +# -- set if you need credentials to pull your custom image. Each entry needs a +# `name:` key, matching a Secret of type kubernetes.io/dockerconfigjson. +imagePullSecrets: + - name: my-pull-secret +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # -- Specifies whether a service account should be created + create: true + # -- Annotations to add to the service account. Example below is for AWS IRSA. + annotations: + eks.amazonaws.com/role-arn: "arn:aws:iam::123456789012:role/garage-s3" + # -- The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +# -- additional pod annotations +podAnnotations: + example.com/has-an-annotation: "true" + +podSecurityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + fsGroupChangePolicy: "OnRootMismatch" + runAsNonRoot: true + +securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + +service: + # -- You can rely on any service to expose your cluster + # - ClusterIP (+ Ingress) + # - NodePort (+ Ingress) + # - LoadBalancer + type: ClusterIP + # -- Annotations to add to the service. Example below is for an AWS NLB. + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: "nlb" + s3: + api: + port: 3900 + web: + port: 3902 + # NOTE: the admin API is excluded for now as it is not consistent across nodes + +ingress: + s3: + api: + enabled: true + className: "nginx" + annotations: + cert-manager.io/cluster-issuer: "letsencrypt-prod" + labels: {} + hosts: + # -- garage S3 API endpoint, to be used with awscli for example + - host: "s3.garage.tld" + paths: + - path: / + pathType: Prefix + # -- garage S3 API endpoint, DNS style bucket access + - host: "*.s3.garage.tld" + paths: + - path: / + pathType: Prefix + tls: + - secretName: garage-s3-api-tls + hosts: + - s3.garage.tld + - "*.s3.garage.tld" + web: + enabled: true + className: "nginx" + annotations: + cert-manager.io/cluster-issuer: "letsencrypt-prod" + labels: {} + hosts: + # -- wildcard website access with bucket name prefix + - host: "*.web.garage.tld" + paths: + - path: / + pathType: Prefix + # -- specific bucket access with FQDN bucket + - host: "mywebpage.example.com" + paths: + - path: / + pathType: Prefix + tls: + - secretName: garage-s3-web-tls + hosts: + - "*.web.garage.tld" + - mywebpage.example.com + +# The following are indicative for a small-size deployment, for anything serious double them. +resources: + limits: + cpu: 200m + memory: 2048Mi + requests: + cpu: 100m + memory: 1024Mi + +# -- Specifies a livenessProbe +livenessProbe: + httpGet: + path: /health + port: 3903 # or the port from garage.admin.apiBindAddr + initialDelaySeconds: 5 + periodSeconds: 30 +# -- Specifies a readinessProbe +readinessProbe: + httpGet: + path: /health + port: 3903 # or the port from garage.admin.apiBindAddr + initialDelaySeconds: 5 + periodSeconds: 30 + +# -- Example: pin pods to a dedicated storage node pool, paired with the +# toleration below. +nodeSelector: + node-role.kubernetes.io/storage: "true" + +tolerations: + - key: "dedicated" + operator: "Equal" + value: "storage" + effect: "NoSchedule" + +# -- Example: spread garage replicas across different nodes, since it is a +# geo-distributed store that only helps availability if replicas don't share +# a failure domain or availability zone. +affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/name: garage + topologyKey: kubernetes.io/hostname + +# -- Optional priority class name to assign to the pods. +# See https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/ +# This is expected to reference a PriorityClass you define yourself. +priorityClassName: "high-priority-storage" + +# -- Extra container env vars. Note this is a [] of {name, value} objects (ie. a pod env stanza) +# GARAGE_ADMIN_TOKEN_FILE below points garage at the token file mounted by +# extraVolumes/extraVolumeMounts, see below. +environment: + - name: RUST_LOG + value: "garage=debug" + - name: GARAGE_ADMIN_TOKEN_FILE + value: /mnt/secrets-store/admin-token + +# -- Extra volumes/volumeMounts. Both are []. Example here mounts the admin API +# token from an external secrets manager via the Secrets Store CSI driver +# (https://secrets-store-csi-driver.sigs.k8s.io/) instead of a Secret volume. +# This allows, for example, providing the tokens without creating a Kubernetes +# secret. garage reads the mounted file through GARAGE_ADMIN_TOKEN_FILE above. +extraVolumes: + - name: secrets-store + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: garage-admin-token +extraVolumeMounts: + - name: secrets-store + mountPath: /mnt/secrets-store + readOnly: true + +monitoring: + metrics: + # -- If true, a service for monitoring is created with a prometheus.io/scrape annotation + enabled: true + serviceMonitor: + # -- If true, a ServiceMonitor CRD is created for a prometheus operator + # https://github.com/coreos/prometheus-operator + enabled: true + path: /metrics + # -- Defaults to the namespace the chart is deployed to; this field is + # templated, so it can also reference .Release.Namespace itself. + namespace: "monitoring" + labels: + release: prometheus + interval: 30s + scheme: http + tlsConfig: {} + scrapeTimeout: 10s + relabelings: + - sourceLabels: ["__meta_kubernetes_pod_node_name"] + targetLabel: node + tracing: + # -- specify a sink endpoint for OpenTelemetry Traces, eg. `http://localhost:4317` + sink: "http://otel-collector.monitoring.svc:4317" diff --git a/script/helm/garage/tests/clusterrole_test.yaml b/script/helm/garage/tests/clusterrole_test.yaml new file mode 100644 index 00000000..094ac561 --- /dev/null +++ b/script/helm/garage/tests/clusterrole_test.yaml @@ -0,0 +1,31 @@ +suite: rbac +templates: + - templates/clusterrole.yaml +tests: + - it: allows managing the garage CRD by default + asserts: + - hasDocuments: + count: 2 + - documentIndex: 0 + isKind: + of: ClusterRole + - documentIndex: 0 + contains: + path: rules[0].resources + content: customresourcedefinitions + - documentIndex: 1 + isKind: + of: ClusterRoleBinding + - documentIndex: 1 + equal: + path: subjects[0].name + value: RELEASE-NAME-garage + + - it: skips the CRD management rule when the CRD is installed manually + set: + garage.kubernetesSkipCrd: true + asserts: + - documentIndex: 0 + notContains: + path: rules[0].resources + content: customresourcedefinitions diff --git a/script/helm/garage/tests/complex_test.yaml b/script/helm/garage/tests/complex_test.yaml new file mode 100644 index 00000000..03d70096 --- /dev/null +++ b/script/helm/garage/tests/complex_test.yaml @@ -0,0 +1,82 @@ +# Integration-style suite: renders the whole chart with complex-values.yaml +# (many non-default features combined at once) and checks that they don't +# clobber each other, rather than testing any single feature in isolation +# (that's what the other tests/*_test.yaml suites are for). +suite: complex-values integration +templates: + - templates/workload.yaml + - templates/service.yaml + - templates/service-headless.yaml + - templates/ingress.yaml + - templates/servicemonitor.yaml + - templates/serviceaccount.yaml + - templates/configmap.yaml + - templates/clusterrole.yaml +tests: + - it: renders a self-consistent deployment with every optional feature enabled + values: + - ../complex-values.yaml + asserts: + - template: templates/workload.yaml + isKind: + of: StatefulSet + - template: templates/workload.yaml + equal: + path: metadata.labels.team + value: platform + - template: templates/workload.yaml + equal: + path: spec.template.spec.containers[0].env[0].name + value: RUST_LOG + - template: templates/workload.yaml + contains: + path: spec.template.spec.volumes + content: + name: secrets-store + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: garage-admin-token + - template: templates/workload.yaml + equal: + path: spec.template.spec.containers[0].env[1].name + value: GARAGE_ADMIN_TOKEN_FILE + - template: templates/workload.yaml + equal: + path: spec.volumeClaimTemplates[0].spec.storageClassName + value: fast-ssd + - template: templates/workload.yaml + contains: + path: spec.template.spec.imagePullSecrets + content: + name: my-pull-secret + - template: templates/service.yaml + hasDocuments: + count: 2 # main service + metrics service, since monitoring.metrics.enabled is true here + - template: templates/service-headless.yaml + hasDocuments: + count: 1 # StatefulSet still gets a headless service + - template: templates/ingress.yaml + hasDocuments: + count: 2 # both s3 api and s3 web ingresses enabled together + - template: templates/servicemonitor.yaml + hasDocuments: + count: 1 + - template: templates/servicemonitor.yaml + equal: + path: spec.endpoints[0].relabelings[0].targetLabel + value: node + - template: templates/serviceaccount.yaml + equal: + path: metadata.annotations["eks.amazonaws.com/role-arn"] + value: "arn:aws:iam::123456789012:role/garage-s3" + - template: templates/configmap.yaml + matchRegex: + path: data["garage.toml"] + pattern: 'data_fsync = true' + - template: templates/clusterrole.yaml + documentIndex: 0 + notContains: + path: rules[0].resources + content: customresourcedefinitions # garage.kubernetesSkipCrd is true here diff --git a/script/helm/garage/tests/configmap_test.yaml b/script/helm/garage/tests/configmap_test.yaml new file mode 100644 index 00000000..82dae580 --- /dev/null +++ b/script/helm/garage/tests/configmap_test.yaml @@ -0,0 +1,136 @@ +suite: configmap +templates: + - templates/configmap.yaml +tests: + - it: renders garage.toml with the default configuration + asserts: + - hasDocuments: + count: 1 + - isKind: + of: ConfigMap + - equal: + path: metadata.name + value: RELEASE-NAME-garage-config + - matchRegex: + path: data["garage.toml"] + pattern: 'metadata_dir = "/mnt/meta"' + - matchRegex: + path: data["garage.toml"] + pattern: 'data_dir = "/mnt/data"' + - matchRegex: + path: data["garage.toml"] + pattern: 'db_engine = "lmdb"' + - matchRegex: + path: data["garage.toml"] + pattern: 'block_size = "1048576"' + - matchRegex: + path: data["garage.toml"] + pattern: 'replication_factor = 3' + - matchRegex: + path: data["garage.toml"] + pattern: 'consistency_mode = "consistent"' + - matchRegex: + path: data["garage.toml"] + pattern: 'compression_level = 1' + - matchRegex: + path: data["garage.toml"] + pattern: 'rpc_bind_addr = "\[::\]:3901"' + - matchRegex: + path: data["garage.toml"] + pattern: 'rpc_secret = "__RPC_SECRET_REPLACE__"' + - matchRegex: + path: data["garage.toml"] + pattern: '(?s)\[kubernetes_discovery\]\s*namespace = "NAMESPACE"\s*service_name = "RELEASE-NAME-garage"\s*skip_crd = false' + - matchRegex: + path: data["garage.toml"] + pattern: '(?s)\[s3_api\]\s*s3_region = "garage"\s*api_bind_addr = "\[::\]:3900"\s*root_domain = "\.s3\.garage\.tld"' + - matchRegex: + path: data["garage.toml"] + pattern: '(?s)\[s3_web\]\s*bind_addr = "\[::\]:3902"\s*root_domain = "\.web\.garage\.tld"\s*index = "index.html"' + - matchRegex: + path: data["garage.toml"] + pattern: '(?s)\[admin\]\s*api_bind_addr = "\[::\]:3903"' + - notMatchRegex: + path: data["garage.toml"] + pattern: 'metadata_auto_snapshot_interval' + - notMatchRegex: + path: data["garage.toml"] + pattern: 'trace_sink' + + - it: reflects custom garage settings, bootstrap peers and additional config + set: + garage.dbEngine: sqlite + garage.blockSize: "2097152" + garage.replicationFactor: "5" + garage.consistencyMode: degraded + garage.compressionLevel: "3" + garage.metadataAutoSnapshotInterval: 6h + garage.bootstrapPeers: + - abc@peer1:3901 + - def@peer2:3901 + garage.additionalTopLevelConfig: "data_fsync = true" + monitoring.tracing.sink: http://otel:4317 + asserts: + - matchRegex: + path: data["garage.toml"] + pattern: 'db_engine = "sqlite"' + - matchRegex: + path: data["garage.toml"] + pattern: 'block_size = "2097152"' + - matchRegex: + path: data["garage.toml"] + pattern: 'replication_factor = 5' + - matchRegex: + path: data["garage.toml"] + pattern: 'consistency_mode = "degraded"' + - matchRegex: + path: data["garage.toml"] + pattern: 'compression_level = 3' + - matchRegex: + path: data["garage.toml"] + pattern: 'metadata_auto_snapshot_interval = "6h"' + - matchRegex: + path: data["garage.toml"] + pattern: 'bootstrap_peers = \["abc@peer1:3901"\s*, "def@peer2:3901"' + - matchRegex: + path: data["garage.toml"] + pattern: 'data_fsync = true' + - matchRegex: + path: data["garage.toml"] + pattern: 'trace_sink = "http://otel:4317"' + + - it: uses garageTomlString verbatim when set, ignoring the structured values + set: + garage.garageTomlString: |- + metadata_dir = "/custom/meta" + replication_factor = 1 + garage.dbEngine: sqlite + asserts: + - equal: + path: data["garage.toml"] + value: |- + metadata_dir = "/custom/meta" + replication_factor = 1 + - notMatchRegex: + path: data["garage.toml"] + pattern: 'db_engine' + + - it: templates garageTomlString against the release and values context + set: + garage.garageTomlString: |- + # namespace: {{ .Release.Namespace }} + replication_factor = {{ .Values.garage.replicationFactor }} + garage.replicationFactor: "7" + asserts: + - equal: + path: data["garage.toml"] + value: |- + # namespace: NAMESPACE + replication_factor = 7 + + - it: does not render a ConfigMap when an existing one is referenced + set: + garage.existingConfigMap: my-external-cm + asserts: + - hasDocuments: + count: 0 diff --git a/script/helm/garage/tests/ingress_test.yaml b/script/helm/garage/tests/ingress_test.yaml new file mode 100644 index 00000000..0835d78e --- /dev/null +++ b/script/helm/garage/tests/ingress_test.yaml @@ -0,0 +1,95 @@ +suite: ingress +templates: + - templates/ingress.yaml +tests: + - it: renders no ingress by default + asserts: + - hasDocuments: + count: 0 + + - it: renders api and web ingresses with tls when enabled + values: + - ./values/ingress.yaml + asserts: + - hasDocuments: + count: 2 + - isKind: + of: Ingress + - documentIndex: 0 + equal: + path: metadata.name + value: RELEASE-NAME-garage-s3-api + - documentIndex: 0 + equal: + path: spec.ingressClassName + value: nginx + - documentIndex: 0 + equal: + path: spec.rules[0].host + value: s3.example.com + - documentIndex: 0 + equal: + path: spec.tls[0].secretName + value: garage-s3-api-tls + - documentIndex: 1 + equal: + path: metadata.name + value: RELEASE-NAME-garage-s3-web + - documentIndex: 1 + equal: + path: spec.rules[0].host + value: "*.web.example.com" + - documentIndex: 1 + equal: + path: spec.tls[0].secretName + value: garage-s3-web-tls + + - it: can enable only the s3 api ingress + set: + ingress.s3.api.enabled: true + ingress.s3.api.hosts[0].host: s3.example.com + ingress.s3.api.hosts[0].paths[0].path: / + ingress.s3.api.hosts[0].paths[0].pathType: Prefix + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.name + value: RELEASE-NAME-garage-s3-api + + - it: omits ingressClassName and tls when neither is configured + set: + ingress.s3.api.enabled: true + ingress.s3.api.hosts[0].host: s3.example.com + ingress.s3.api.hosts[0].paths[0].path: / + ingress.s3.api.hosts[0].paths[0].pathType: Prefix + asserts: + - isNull: + path: spec.ingressClassName + - isNull: + path: spec.tls + + - it: renders multiple hosts on the same ingress + set: + ingress.s3.api.enabled: true + ingress.s3.api.hosts: + - host: s3.example.com + paths: + - path: / + pathType: Prefix + - host: s3-alt.example.com + paths: + - path: / + pathType: Prefix + asserts: + - hasDocuments: + count: 1 + - lengthEqual: + path: spec.rules + count: 2 + - equal: + path: spec.rules[0].host + value: s3.example.com + - equal: + path: spec.rules[1].host + value: s3-alt.example.com diff --git a/script/helm/garage/tests/naming_test.yaml b/script/helm/garage/tests/naming_test.yaml new file mode 100644 index 00000000..128d0a5a --- /dev/null +++ b/script/helm/garage/tests/naming_test.yaml @@ -0,0 +1,56 @@ +suite: naming and common labels +templates: + - templates/workload.yaml + - templates/configmap.yaml +tests: + - it: applies commonLabels alongside the default chart labels + template: templates/workload.yaml + set: + commonLabels: + team: storage + asserts: + - equal: + path: metadata.labels.team + value: storage + - equal: + path: metadata.labels["app.kubernetes.io/managed-by"] + value: Helm + + - it: uses fullnameOverride verbatim for resource names + template: templates/workload.yaml + set: + fullnameOverride: my-garage-cluster + asserts: + - equal: + path: metadata.name + value: my-garage-cluster + - equal: + path: spec.serviceName + value: my-garage-cluster-headless + + - it: does not double-prefix when the release name already contains the chart name + template: templates/workload.yaml + release: + name: garage + asserts: + - equal: + path: metadata.name + value: garage + + - it: prefixes the release name with the chart name otherwise + template: templates/workload.yaml + release: + name: prod + asserts: + - equal: + path: metadata.name + value: prod-garage + + - it: truncates an overly long fullname to 63 characters and trims a trailing dash + template: templates/workload.yaml + set: + fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-suffix-that-will-be-cut-off + asserts: + - equal: + path: metadata.name + value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa diff --git a/script/helm/garage/tests/secret_test.yaml b/script/helm/garage/tests/secret_test.yaml new file mode 100644 index 00000000..bf938584 --- /dev/null +++ b/script/helm/garage/tests/secret_test.yaml @@ -0,0 +1,33 @@ +suite: rpc secret +templates: + - templates/secret.yaml +tests: + - it: generates a Secret holding the rpc secret by default + asserts: + - hasDocuments: + count: 1 + - isKind: + of: Secret + - equal: + path: metadata.name + value: RELEASE-NAME-garage-rpc-secret + - equal: + path: type + value: Opaque + - isNotNull: + path: data.rpcSecret + + - it: does not render a Secret when an existing one is referenced + values: + - ./values/existing-secret.yaml + asserts: + - hasDocuments: + count: 0 + + - it: base64-encodes an explicitly provided rpc secret + set: + garage.rpcSecret: my-plain-secret + asserts: + - equal: + path: data.rpcSecret + value: bXktcGxhaW4tc2VjcmV0 diff --git a/script/helm/garage/tests/service_headless_test.yaml b/script/helm/garage/tests/service_headless_test.yaml new file mode 100644 index 00000000..3448f229 --- /dev/null +++ b/script/helm/garage/tests/service_headless_test.yaml @@ -0,0 +1,26 @@ +suite: headless service +templates: + - templates/service-headless.yaml +tests: + - it: creates a headless service for a StatefulSet by default + asserts: + - hasDocuments: + count: 1 + - isKind: + of: Service + - equal: + path: metadata.name + value: RELEASE-NAME-garage-headless + - equal: + path: spec.clusterIP + value: None + - equal: + path: spec.type + value: ClusterIP + + - it: does not create a headless service for a DaemonSet + values: + - ./values/daemonset.yaml + asserts: + - hasDocuments: + count: 0 diff --git a/script/helm/garage/tests/service_test.yaml b/script/helm/garage/tests/service_test.yaml new file mode 100644 index 00000000..cf3cceb8 --- /dev/null +++ b/script/helm/garage/tests/service_test.yaml @@ -0,0 +1,61 @@ +suite: service +templates: + - templates/service.yaml +tests: + - it: creates a ClusterIP service with s3-api and s3-web ports by default + asserts: + - hasDocuments: + count: 1 + - isKind: + of: Service + - equal: + path: spec.type + value: ClusterIP + - equal: + path: spec.ports[0].name + value: s3-api + - equal: + path: spec.ports[0].port + value: 3900 + - equal: + path: spec.ports[1].name + value: s3-web + - equal: + path: spec.ports[1].port + value: 3902 + + - it: honors a custom service type and port + set: + service.type: LoadBalancer + service.s3.api.port: 9000 + asserts: + - equal: + path: spec.type + value: LoadBalancer + - equal: + path: spec.ports[0].port + value: 9000 + + - it: does not create a metrics service by default + asserts: + - hasDocuments: + count: 1 + + - it: adds a headless metrics service when monitoring is enabled + values: + - ./values/monitoring.yaml + asserts: + - hasDocuments: + count: 2 + - documentIndex: 1 + equal: + path: metadata.name + value: RELEASE-NAME-garage-metrics + - documentIndex: 1 + equal: + path: spec.clusterIP + value: None + - documentIndex: 1 + equal: + path: metadata.annotations["prometheus.io/scrape"] + value: "true" diff --git a/script/helm/garage/tests/serviceaccount_test.yaml b/script/helm/garage/tests/serviceaccount_test.yaml new file mode 100644 index 00000000..0c3278c3 --- /dev/null +++ b/script/helm/garage/tests/serviceaccount_test.yaml @@ -0,0 +1,28 @@ +suite: service account +templates: + - templates/serviceaccount.yaml +tests: + - it: creates a ServiceAccount by default + asserts: + - hasDocuments: + count: 1 + - isKind: + of: ServiceAccount + - equal: + path: metadata.name + value: RELEASE-NAME-garage + + - it: does not create a ServiceAccount when disabled + values: + - ./values/minimal.yaml + asserts: + - hasDocuments: + count: 0 + + - it: honors a custom service account name + set: + serviceAccount.name: my-garage-sa + asserts: + - equal: + path: metadata.name + value: my-garage-sa diff --git a/script/helm/garage/tests/servicemonitor_test.yaml b/script/helm/garage/tests/servicemonitor_test.yaml new file mode 100644 index 00000000..2bd3ef00 --- /dev/null +++ b/script/helm/garage/tests/servicemonitor_test.yaml @@ -0,0 +1,65 @@ +suite: service monitor +templates: + - templates/servicemonitor.yaml +tests: + - it: renders no ServiceMonitor by default + asserts: + - hasDocuments: + count: 0 + + - it: renders no ServiceMonitor when only metrics are enabled + set: + monitoring.metrics.enabled: true + asserts: + - hasDocuments: + count: 0 + + - it: renders a ServiceMonitor when explicitly enabled + values: + - ./values/monitoring.yaml + asserts: + - hasDocuments: + count: 1 + - isKind: + of: ServiceMonitor + - equal: + path: metadata.name + value: RELEASE-NAME-garage + - equal: + path: metadata.namespace + value: NAMESPACE + - equal: + path: spec.endpoints[0].interval + value: 30s + + - it: templates a custom namespace against the release context + values: + - ./values/monitoring.yaml + set: + monitoring.metrics.serviceMonitor.namespace: "{{ .Release.Namespace }}-monitoring" + asserts: + - equal: + path: metadata.namespace + value: NAMESPACE-monitoring + + - it: applies custom labels, tlsConfig and relabelings + values: + - ./values/monitoring.yaml + set: + monitoring.metrics.serviceMonitor.labels: + team: storage + monitoring.metrics.serviceMonitor.tlsConfig: + insecureSkipVerify: true + monitoring.metrics.serviceMonitor.relabelings: + - sourceLabels: ["__meta_kubernetes_pod_name"] + targetLabel: pod + asserts: + - equal: + path: metadata.labels.team + value: storage + - equal: + path: spec.endpoints[0].tlsConfig.insecureSkipVerify + value: true + - equal: + path: spec.endpoints[0].relabelings[0].targetLabel + value: pod diff --git a/script/helm/garage/tests/values/daemonset.yaml b/script/helm/garage/tests/values/daemonset.yaml new file mode 100644 index 00000000..568e4413 --- /dev/null +++ b/script/helm/garage/tests/values/daemonset.yaml @@ -0,0 +1,10 @@ +# Run garage as a DaemonSet (one pod per node) instead of the default StatefulSet, +# using hostPath volumes for meta/data persistence. +deployment: + kind: DaemonSet +persistence: + enabled: true + meta: + hostPath: /var/lib/garage/meta + data: + hostPath: /var/lib/garage/data diff --git a/script/helm/garage/tests/values/existing-secret.yaml b/script/helm/garage/tests/values/existing-secret.yaml new file mode 100644 index 00000000..795bd583 --- /dev/null +++ b/script/helm/garage/tests/values/existing-secret.yaml @@ -0,0 +1,5 @@ +# Use a pre-existing Kubernetes Secret for the RPC secret instead of letting +# the chart generate/manage one. +garage: + rpcSecret: "" + existingRpcSecret: "garage-rpc-secret-external" diff --git a/script/helm/garage/tests/values/ingress.yaml b/script/helm/garage/tests/values/ingress.yaml new file mode 100644 index 00000000..d18bdf82 --- /dev/null +++ b/script/helm/garage/tests/values/ingress.yaml @@ -0,0 +1,27 @@ +# Expose both the S3 API and website endpoints through Ingress, with TLS. +ingress: + s3: + api: + enabled: true + className: "nginx" + hosts: + - host: "s3.example.com" + paths: + - path: / + pathType: Prefix + tls: + - secretName: garage-s3-api-tls + hosts: + - s3.example.com + web: + enabled: true + className: "nginx" + hosts: + - host: "*.web.example.com" + paths: + - path: / + pathType: Prefix + tls: + - secretName: garage-s3-web-tls + hosts: + - "*.web.example.com" diff --git a/script/helm/garage/tests/values/minimal.yaml b/script/helm/garage/tests/values/minimal.yaml new file mode 100644 index 00000000..67e7d715 --- /dev/null +++ b/script/helm/garage/tests/values/minimal.yaml @@ -0,0 +1,8 @@ +# Minimal single-node deployment without persistent storage or a dedicated +# service account, e.g. for local testing. +deployment: + replicaCount: 1 +persistence: + enabled: false +serviceAccount: + create: false diff --git a/script/helm/garage/tests/values/monitoring.yaml b/script/helm/garage/tests/values/monitoring.yaml new file mode 100644 index 00000000..3f82916f --- /dev/null +++ b/script/helm/garage/tests/values/monitoring.yaml @@ -0,0 +1,7 @@ +# Enable Prometheus metrics scraping and a ServiceMonitor for the prometheus-operator. +monitoring: + metrics: + enabled: true + serviceMonitor: + enabled: true + interval: 30s diff --git a/script/helm/garage/tests/workload_test.yaml b/script/helm/garage/tests/workload_test.yaml new file mode 100644 index 00000000..19b6075f --- /dev/null +++ b/script/helm/garage/tests/workload_test.yaml @@ -0,0 +1,187 @@ +suite: workload (StatefulSet/DaemonSet) +templates: + - templates/workload.yaml + - templates/configmap.yaml +tests: + - it: defaults to a StatefulSet with 3 replicas and 2 volumes + template: templates/workload.yaml + asserts: + - isKind: + of: StatefulSet + - equal: + path: spec.replicas + value: 3 + - equal: + path: spec.podManagementPolicy + value: OrderedReady + - equal: + path: spec.template.spec.volumes[1].name + value: etc + - lengthEqual: + path: spec.template.spec.volumes + count: 2 + - isNotNull: + path: spec.volumeClaimTemplates + + - it: uses a StatefulSet with PVC-backed volumeClaimTemplates by default + template: templates/workload.yaml + asserts: + - isKind: + of: StatefulSet + - isNotNull: + path: spec.volumeClaimTemplates + - equal: + path: spec.volumeClaimTemplates[0].spec.resources.requests.storage + value: 100Mi + + - it: switches to a DaemonSet with hostPath volumes when requested + template: templates/workload.yaml + values: + - ./values/daemonset.yaml + asserts: + - isKind: + of: DaemonSet + - isNull: + path: spec.replicas + - isNull: + path: spec.volumeClaimTemplates + - contains: + path: spec.template.spec.volumes + content: + name: meta + hostPath: + path: /var/lib/garage/meta + type: DirectoryOrCreate + - contains: + path: spec.template.spec.volumes + content: + name: data + hostPath: + path: /var/lib/garage/data + type: DirectoryOrCreate + + - it: renders emptyDir volumes when persistence is disabled + template: templates/workload.yaml + values: + - ./values/minimal.yaml + asserts: + - contains: + path: spec.template.spec.volumes + content: + name: meta + emptyDir: {} + - contains: + path: spec.template.spec.volumes + content: + name: data + emptyDir: {} + - isNull: + path: spec.volumeClaimTemplates + + - it: honors a custom replicaCount + template: templates/workload.yaml + set: + deployment.replicaCount: 5 + asserts: + - equal: + path: spec.replicas + value: 5 + + - it: points the init container at the configured rpc secret + template: templates/workload.yaml + asserts: + - equal: + path: spec.template.spec.initContainers[0].env[0].valueFrom.secretKeyRef.name + value: RELEASE-NAME-garage-rpc-secret + + - it: points the init container at an existing rpc secret when configured + template: templates/workload.yaml + values: + - ./values/existing-secret.yaml + asserts: + - equal: + path: spec.template.spec.initContainers[0].env[0].valueFrom.secretKeyRef.name + value: garage-rpc-secret-external + + - it: sets the container image from repository and tag + template: templates/workload.yaml + set: + image.repository: dxflrs/amd64_garage + image.tag: v1.2.3 + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: dxflrs/amd64_garage:v1.2.3 + + - it: falls back to the chart appVersion when no image tag is set + template: templates/workload.yaml + asserts: + - matchRegex: + path: spec.template.spec.containers[0].image + pattern: ^dxflrs/amd64_garage:v + + - it: omits storageClassName from volumeClaimTemplates by default + template: templates/workload.yaml + asserts: + - isNull: + path: spec.volumeClaimTemplates[0].spec.storageClassName + - isNull: + path: spec.volumeClaimTemplates[1].spec.storageClassName + + - it: sets storageClassName in volumeClaimTemplates when configured + template: templates/workload.yaml + set: + persistence.meta.storageClass: fast-storage + persistence.data.storageClass: slow-storage + asserts: + - equal: + path: spec.volumeClaimTemplates[0].spec.storageClassName + value: fast-storage + - equal: + path: spec.volumeClaimTemplates[1].spec.storageClassName + value: slow-storage + + - it: renders emptyDir volumes for a DaemonSet when persistence is disabled + template: templates/workload.yaml + set: + deployment.kind: DaemonSet + persistence.enabled: false + asserts: + - contains: + path: spec.template.spec.volumes + content: + name: meta + emptyDir: {} + - contains: + path: spec.template.spec.volumes + content: + name: data + emptyDir: {} + + - it: mounts the existing ConfigMap volume when configured + template: templates/workload.yaml + set: + garage.existingConfigMap: my-external-cm + asserts: + - equal: + path: spec.template.spec.volumes[0].configMap.name + value: my-external-cm + + - it: uses a custom service account name without creating one when disabled + template: templates/workload.yaml + set: + serviceAccount.create: false + serviceAccount.name: my-external-sa + asserts: + - equal: + path: spec.template.spec.serviceAccountName + value: my-external-sa + + - it: falls back to the default service account when disabled without a custom name + template: templates/workload.yaml + set: + serviceAccount.create: false + asserts: + - equal: + path: spec.template.spec.serviceAccountName + value: default diff --git a/script/helm/garage/values.yaml b/script/helm/garage/values.yaml index a74faf3f..13e17c74 100644 --- a/script/helm/garage/values.yaml +++ b/script/helm/garage/values.yaml @@ -246,10 +246,14 @@ affinity: {} # See https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/ priorityClassName: "" +# -- Extra container env vars, as a list of {name, value} objects (same shape +# as a Pod container's env) environment: {} +# -- Extra volumes, as a list of volume objects (same shape as a PodSpec's volumes) extraVolumes: {} +# -- Extra volume mounts, as a list of mount objects (same shape as a container's volumeMounts) extraVolumeMounts: {} monitoring: