fix(helm): coordinate distributed pod startup (#5258)

helm/rustfs/templates/statefulset.yaml

@@ -4,9 +4,13 @@
 {{- if and .Values.mode.distributed.enabled (le (int .Values.replicaCount) 1) -}}
 {{- fail "Distributed mode requires replicaCount >= 2" -}}
 {{- end -}}
+{{- if and .Values.mode.distributed.enabled (le (int .Values.startupWaitTimeoutSeconds) 0) -}}
+{{- fail "startupWaitTimeoutSeconds must be greater than 0 in distributed mode" -}}
+{{- end -}}
 
 {{- if .Values.mode.distributed.enabled }}
-{{- range $pool := include "rustfs.pools" . | fromJsonArray }}
+{{- $pools := include "rustfs.pools" . | fromJsonArray }}
+{{- range $pool := $pools }}
 {{- $drivesPerNode := int $pool.drives }}
 ---
 apiVersion: apps/v1
@@ -123,6 +127,10 @@ spec:
           env:
             - name: DRIVES_PER_NODE
               value: {{ $drivesPerNode | quote }}
+            - name: ENDPOINT_PORT
+              value: {{ $.Values.service.endpoint.port | quote }}
+            - name: STARTUP_WAIT_TIMEOUT_SECONDS
+              value: {{ $.Values.startupWaitTimeoutSeconds | quote }}
           command:
             - sh
             - -c
@@ -138,6 +146,44 @@ spec:
               mkdir -p /mnt/rustfs/logs
               chmod 755 /mnt/rustfs/logs
               {{- end }}
+              deadline=$(($(date +%s) + STARTUP_WAIT_TIMEOUT_SECONDS))
+              wait_for_peer() {
+                description=$1
+                shift
+                until "$@" >/dev/null 2>&1; do
+                  if [ "$(date +%s)" -ge "$deadline" ]; then
+                    echo "Timed out after ${STARTUP_WAIT_TIMEOUT_SECONDS}s waiting for ${description}" >&2
+                    exit 1
+                  fi
+                  sleep 2
+                done
+              }
+
+              echo "Waiting for distributed peer DNS records"
+              {{- range $peerPool := $pools }}
+              {{- range $i := until (int $peerPool.replicaCount) }}
+              wait_for_peer "DNS for {{ $peerPool.fullname }}-{{ $i }}" \
+                nslookup "{{ $peerPool.fullname }}-{{ $i }}.{{ include "rustfs.fullname" $ }}-headless.{{ $.Release.Namespace }}.svc.{{ include "rustfs.clusterDomain" $ }}"
+              {{- end }}
+              {{- end }}
+
+              ordinal=${HOSTNAME##*-}
+              echo "Waiting for earlier distributed peers to listen on port ${ENDPOINT_PORT}"
+              {{- range $peerPool := $pools }}
+              {{- if lt (int $peerPool.index) (int $pool.index) }}
+              {{- range $i := until (int $peerPool.replicaCount) }}
+              wait_for_peer "{{ $peerPool.fullname }}-{{ $i }}:${ENDPOINT_PORT}" \
+                nc -z -w 2 "{{ $peerPool.fullname }}-{{ $i }}.{{ include "rustfs.fullname" $ }}-headless.{{ $.Release.Namespace }}.svc.{{ include "rustfs.clusterDomain" $ }}" "$ENDPOINT_PORT"
+              {{- end }}
+              {{- else if eq (int $peerPool.index) (int $pool.index) }}
+              i=0
+              while [ "$i" -lt "$ordinal" ]; do
+                wait_for_peer "{{ $peerPool.fullname }}-${i}:${ENDPOINT_PORT}" \
+                  nc -z -w 2 "{{ $peerPool.fullname }}-${i}.{{ include "rustfs.fullname" $ }}-headless.{{ $.Release.Namespace }}.svc.{{ include "rustfs.clusterDomain" $ }}" "$ENDPOINT_PORT"
+                i=$((i + 1))
+              done
+              {{- end }}
+              {{- end }}
           volumeMounts:
             {{- if gt $drivesPerNode 1 }}
             {{- range $i := until $drivesPerNode }}

helm/rustfs/values.yaml

@@ -30,6 +30,11 @@ drivesPerNode: null
 # Provide the DNS root only, without a `svc.` prefix or leading/trailing dots.
 clusterDomain: cluster.local
 
+# Maximum time an init container waits for distributed peers to publish DNS
+# and for earlier peers to start listening. The wait is shared across all
+# peers, not applied once per peer.
+startupWaitTimeoutSeconds: 300
+
 # This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/
 image:
   rustfs: # This sets the rustfs image repository and tag.

scripts/test_helm_templates.sh

@@ -201,7 +201,7 @@ command -v yq >/dev/null 2>&1 || { echo "yq is required for extra-volumes struct
 assert_extra_volumes_wired() {
   local output="$1" label="$2"
 
-  if ! yq eval '.spec.template.spec.volumes[].name' - <<<"$output" | grep -q "^ca-bundle$"; then
+  if ! yq eval '.spec.template.spec.volumes[].name' - <<<"$output" | grep "^ca-bundle$" >/dev/null; then
     echo "ca-bundle not found in spec.template.spec.volumes of $label" >&2
     exit 1
   fi
@@ -213,7 +213,7 @@ assert_extra_volumes_wired() {
     exit 1
   fi
 
-  if yq eval '.spec.template.spec.initContainers[].volumeMounts[].name' - <<<"$output" | grep -q "^ca-bundle$"; then
+  if yq eval '.spec.template.spec.initContainers[].volumeMounts[].name' - <<<"$output" | grep "^ca-bundle$" >/dev/null; then
     echo "ca-bundle must not appear in initContainers volumeMounts of $label" >&2
     exit 1
   fi
@@ -221,7 +221,7 @@ assert_extra_volumes_wired() {
 
 assert_extra_volumes_absent() {
   local output="$1" label="$2"
-  if yq eval '.spec.template.spec.volumes[].name' - <<<"$output" | grep -q "^ca-bundle$"; then
+  if yq eval '.spec.template.spec.volumes[].name' - <<<"$output" | grep "^ca-bundle$" >/dev/null; then
     echo "ca-bundle must not appear in spec.template.spec.volumes of $label with empty extraVolumes" >&2
     exit 1
   fi
@@ -344,6 +344,54 @@ if grep -q "name: data$" <<<"$generic_eight_by_two"; then
   exit 1
 fi
 
+# Distributed startup coordination must use the configured endpoint port and
+# stay transport-neutral so the same wait works with and without mTLS.
+custom_port_startup=$(render_distributed_statefulset \
+  --set service.endpoint.port=9443 \
+  --set config.rustfs.address=:9443)
+grep -q 'name: ENDPOINT_PORT' <<<"$custom_port_startup"
+grep -A1 'name: ENDPOINT_PORT' <<<"$custom_port_startup" | grep -q 'value: "9443"'
+grep -q 'nc -z -w 2' <<<"$custom_port_startup"
+if grep -q 'http.*9000/health' <<<"$custom_port_startup"; then
+  echo "Distributed startup wait must not hardcode the HTTP scheme or port 9000" >&2
+  exit 1
+fi
+
+mtls_startup=$(render_distributed_statefulset --set mtls.enabled=true)
+grep -q 'nc -z -w 2' <<<"$mtls_startup"
+if grep -q 'wget .*http.*health' <<<"$mtls_startup"; then
+  echo "mTLS startup wait must not use a plaintext HTTP health probe" >&2
+  exit 1
+fi
+
+# The timeout is one global deadline and invalid non-positive values fail
+# during rendering instead of creating an unbounded init wait.
+grep -q 'deadline=$(($(date +%s) + STARTUP_WAIT_TIMEOUT_SECONDS))' <<<"$custom_port_startup"
+invalid_startup_timeout_status=0
+render_distributed_statefulset --set startupWaitTimeoutSeconds=0 >/dev/null 2>&1 || invalid_startup_timeout_status=$?
+if [[ $invalid_startup_timeout_status -eq 0 ]]; then
+  echo "Distributed mode with startupWaitTimeoutSeconds=0 must fail rendering" >&2
+  exit 1
+fi
+
+# Every pool waits for DNS across the complete normalized pool list, while
+# port availability follows the stable pool-index/ordinal startup order.
+multi_pool_startup=$(helm template rustfs "$CHART_DIR" \
+  --namespace rustfs \
+  --set secret.rustfs.access_key=test-access-key \
+  --set secret.rustfs.secret_key=test-secret-key \
+  --set pools.enabled=true \
+  --set 'pools.list[0].replicaCount=2' \
+  --set 'pools.list[1].replicaCount=2')
+grep -q 'nslookup "rustfs-0.rustfs-headless.rustfs.svc.cluster.local"' <<<"$multi_pool_startup"
+grep -q 'nslookup "rustfs-pool1-1.rustfs-headless.rustfs.svc.cluster.local"' <<<"$multi_pool_startup"
+if grep -q 'rustfs-pool1-headless' <<<"$multi_pool_startup"; then
+  echo "Multi-pool startup wait must use the shared root headless service" >&2
+  exit 1
+fi
+grep -q 'wait_for_peer "rustfs-1:${ENDPOINT_PORT}"' <<<"$multi_pool_startup"
+grep -q 'wait_for_peer "rustfs-pool1-${i}:${ENDPOINT_PORT}"' <<<"$multi_pool_startup"
+
 # volumeClaimTemplates must not contain empty annotations when pvcAnnotations are unset,
 # because Kubernetes treats annotations: {} as a mutation of the immutable field.
 no_ann_output=$(render_distributed_statefulset)