diff --git a/helm/rustfs/templates/statefulset.yaml b/helm/rustfs/templates/statefulset.yaml index b23929bb5..032395f60 100644 --- a/helm/rustfs/templates/statefulset.yaml +++ b/helm/rustfs/templates/statefulset.yaml @@ -4,9 +4,13 @@ {{- if and .Values.mode.distributed.enabled (le (int .Values.replicaCount) 1) -}} {{- fail "Distributed mode requires replicaCount >= 2" -}} {{- end -}} +{{- if and .Values.mode.distributed.enabled (le (int .Values.startupWaitTimeoutSeconds) 0) -}} +{{- fail "startupWaitTimeoutSeconds must be greater than 0 in distributed mode" -}} +{{- end -}} {{- if .Values.mode.distributed.enabled }} -{{- range $pool := include "rustfs.pools" . | fromJsonArray }} +{{- $pools := include "rustfs.pools" . | fromJsonArray }} +{{- range $pool := $pools }} {{- $drivesPerNode := int $pool.drives }} --- apiVersion: apps/v1 @@ -123,6 +127,10 @@ spec: env: - name: DRIVES_PER_NODE value: {{ $drivesPerNode | quote }} + - name: ENDPOINT_PORT + value: {{ $.Values.service.endpoint.port | quote }} + - name: STARTUP_WAIT_TIMEOUT_SECONDS + value: {{ $.Values.startupWaitTimeoutSeconds | quote }} command: - sh - -c @@ -138,6 +146,44 @@ spec: mkdir -p /mnt/rustfs/logs chmod 755 /mnt/rustfs/logs {{- end }} + deadline=$(($(date +%s) + STARTUP_WAIT_TIMEOUT_SECONDS)) + wait_for_peer() { + description=$1 + shift + until "$@" >/dev/null 2>&1; do + if [ "$(date +%s)" -ge "$deadline" ]; then + echo "Timed out after ${STARTUP_WAIT_TIMEOUT_SECONDS}s waiting for ${description}" >&2 + exit 1 + fi + sleep 2 + done + } + + echo "Waiting for distributed peer DNS records" + {{- range $peerPool := $pools }} + {{- range $i := until (int $peerPool.replicaCount) }} + wait_for_peer "DNS for {{ $peerPool.fullname }}-{{ $i }}" \ + nslookup "{{ $peerPool.fullname }}-{{ $i }}.{{ include "rustfs.fullname" $ }}-headless.{{ $.Release.Namespace }}.svc.{{ include "rustfs.clusterDomain" $ }}" + {{- end }} + {{- end }} + + ordinal=${HOSTNAME##*-} + echo "Waiting for earlier distributed peers to listen on port ${ENDPOINT_PORT}" + {{- range $peerPool := $pools }} + {{- if lt (int $peerPool.index) (int $pool.index) }} + {{- range $i := until (int $peerPool.replicaCount) }} + wait_for_peer "{{ $peerPool.fullname }}-{{ $i }}:${ENDPOINT_PORT}" \ + nc -z -w 2 "{{ $peerPool.fullname }}-{{ $i }}.{{ include "rustfs.fullname" $ }}-headless.{{ $.Release.Namespace }}.svc.{{ include "rustfs.clusterDomain" $ }}" "$ENDPOINT_PORT" + {{- end }} + {{- else if eq (int $peerPool.index) (int $pool.index) }} + i=0 + while [ "$i" -lt "$ordinal" ]; do + wait_for_peer "{{ $peerPool.fullname }}-${i}:${ENDPOINT_PORT}" \ + nc -z -w 2 "{{ $peerPool.fullname }}-${i}.{{ include "rustfs.fullname" $ }}-headless.{{ $.Release.Namespace }}.svc.{{ include "rustfs.clusterDomain" $ }}" "$ENDPOINT_PORT" + i=$((i + 1)) + done + {{- end }} + {{- end }} volumeMounts: {{- if gt $drivesPerNode 1 }} {{- range $i := until $drivesPerNode }} diff --git a/helm/rustfs/values.yaml b/helm/rustfs/values.yaml index 0ca1d0d90..f5048256c 100644 --- a/helm/rustfs/values.yaml +++ b/helm/rustfs/values.yaml @@ -30,6 +30,11 @@ drivesPerNode: null # Provide the DNS root only, without a `svc.` prefix or leading/trailing dots. clusterDomain: cluster.local +# Maximum time an init container waits for distributed peers to publish DNS +# and for earlier peers to start listening. The wait is shared across all +# peers, not applied once per peer. +startupWaitTimeoutSeconds: 300 + # This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ image: rustfs: # This sets the rustfs image repository and tag. diff --git a/scripts/test_helm_templates.sh b/scripts/test_helm_templates.sh index dffec8088..f9ac3e334 100755 --- a/scripts/test_helm_templates.sh +++ b/scripts/test_helm_templates.sh @@ -201,7 +201,7 @@ command -v yq >/dev/null 2>&1 || { echo "yq is required for extra-volumes struct assert_extra_volumes_wired() { local output="$1" label="$2" - if ! yq eval '.spec.template.spec.volumes[].name' - <<<"$output" | grep -q "^ca-bundle$"; then + if ! yq eval '.spec.template.spec.volumes[].name' - <<<"$output" | grep "^ca-bundle$" >/dev/null; then echo "ca-bundle not found in spec.template.spec.volumes of $label" >&2 exit 1 fi @@ -213,7 +213,7 @@ assert_extra_volumes_wired() { exit 1 fi - if yq eval '.spec.template.spec.initContainers[].volumeMounts[].name' - <<<"$output" | grep -q "^ca-bundle$"; then + if yq eval '.spec.template.spec.initContainers[].volumeMounts[].name' - <<<"$output" | grep "^ca-bundle$" >/dev/null; then echo "ca-bundle must not appear in initContainers volumeMounts of $label" >&2 exit 1 fi @@ -221,7 +221,7 @@ assert_extra_volumes_wired() { assert_extra_volumes_absent() { local output="$1" label="$2" - if yq eval '.spec.template.spec.volumes[].name' - <<<"$output" | grep -q "^ca-bundle$"; then + if yq eval '.spec.template.spec.volumes[].name' - <<<"$output" | grep "^ca-bundle$" >/dev/null; then echo "ca-bundle must not appear in spec.template.spec.volumes of $label with empty extraVolumes" >&2 exit 1 fi @@ -344,6 +344,54 @@ if grep -q "name: data$" <<<"$generic_eight_by_two"; then exit 1 fi +# Distributed startup coordination must use the configured endpoint port and +# stay transport-neutral so the same wait works with and without mTLS. +custom_port_startup=$(render_distributed_statefulset \ + --set service.endpoint.port=9443 \ + --set config.rustfs.address=:9443) +grep -q 'name: ENDPOINT_PORT' <<<"$custom_port_startup" +grep -A1 'name: ENDPOINT_PORT' <<<"$custom_port_startup" | grep -q 'value: "9443"' +grep -q 'nc -z -w 2' <<<"$custom_port_startup" +if grep -q 'http.*9000/health' <<<"$custom_port_startup"; then + echo "Distributed startup wait must not hardcode the HTTP scheme or port 9000" >&2 + exit 1 +fi + +mtls_startup=$(render_distributed_statefulset --set mtls.enabled=true) +grep -q 'nc -z -w 2' <<<"$mtls_startup" +if grep -q 'wget .*http.*health' <<<"$mtls_startup"; then + echo "mTLS startup wait must not use a plaintext HTTP health probe" >&2 + exit 1 +fi + +# The timeout is one global deadline and invalid non-positive values fail +# during rendering instead of creating an unbounded init wait. +grep -q 'deadline=$(($(date +%s) + STARTUP_WAIT_TIMEOUT_SECONDS))' <<<"$custom_port_startup" +invalid_startup_timeout_status=0 +render_distributed_statefulset --set startupWaitTimeoutSeconds=0 >/dev/null 2>&1 || invalid_startup_timeout_status=$? +if [[ $invalid_startup_timeout_status -eq 0 ]]; then + echo "Distributed mode with startupWaitTimeoutSeconds=0 must fail rendering" >&2 + exit 1 +fi + +# Every pool waits for DNS across the complete normalized pool list, while +# port availability follows the stable pool-index/ordinal startup order. +multi_pool_startup=$(helm template rustfs "$CHART_DIR" \ + --namespace rustfs \ + --set secret.rustfs.access_key=test-access-key \ + --set secret.rustfs.secret_key=test-secret-key \ + --set pools.enabled=true \ + --set 'pools.list[0].replicaCount=2' \ + --set 'pools.list[1].replicaCount=2') +grep -q 'nslookup "rustfs-0.rustfs-headless.rustfs.svc.cluster.local"' <<<"$multi_pool_startup" +grep -q 'nslookup "rustfs-pool1-1.rustfs-headless.rustfs.svc.cluster.local"' <<<"$multi_pool_startup" +if grep -q 'rustfs-pool1-headless' <<<"$multi_pool_startup"; then + echo "Multi-pool startup wait must use the shared root headless service" >&2 + exit 1 +fi +grep -q 'wait_for_peer "rustfs-1:${ENDPOINT_PORT}"' <<<"$multi_pool_startup" +grep -q 'wait_for_peer "rustfs-pool1-${i}:${ENDPOINT_PORT}"' <<<"$multi_pool_startup" + # volumeClaimTemplates must not contain empty annotations when pvcAnnotations are unset, # because Kubernetes treats annotations: {} as a mutation of the immutable field. no_ann_output=$(render_distributed_statefulset)