mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-11 07:36:53 +00:00
fix(ecstore): start with unresolved Kubernetes peers (#5460)
* fix(ecstore): start with unresolved Kubernetes peers * fix(ecstore): infer Kubernetes endpoint identity safely * fix(ecstore): fail closed on unsafe format migration * fix(ecstore): reject poisoned format heal candidates * fix(ecstore): reject unsafe legacy migration outliers * fix(ecstore): resume interrupted format migrations * fix(ecstore): preserve Kubernetes startup compatibility
This commit is contained in:
cxymds
@@ -322,21 +322,23 @@ Render RUSTFS_SERVER_DOMAINS
|
||||
{{- join "," $domains -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Render probe command for liveness and readiness
|
||||
{{/* Render an mTLS probe command
|
||||
*/}}
|
||||
|
||||
{{- define "rustfs.probeCommand" -}}
|
||||
{{- $endpoint_port := .Values.service.endpoint.port | default 9000 -}}
|
||||
{{- $console_port := .Values.service.console.port | default 9001 -}}
|
||||
{{- $root := .root -}}
|
||||
{{- $endpointPath := .endpointPath -}}
|
||||
{{- $endpoint_port := $root.Values.service.endpoint.port | default 9000 -}}
|
||||
{{- $console_port := $root.Values.service.console.port | default 9001 -}}
|
||||
{{- $args := "-skf" -}}
|
||||
|
||||
{{- if and .Values.mtls.enabled -}}
|
||||
{{- $args = printf "%s --cert %s --key %s" $args .Values.mtls.clientCertPath .Values.mtls.clientKeyPath -}}
|
||||
{{- if and $root.Values.mtls.enabled -}}
|
||||
{{- $args = printf "%s --cert %s --key %s" $args $root.Values.mtls.clientCertPath $root.Values.mtls.clientKeyPath -}}
|
||||
{{- end -}}
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
curl {{ $args }} https://127.0.0.1:{{ $endpoint_port }}/health/ready && \
|
||||
curl {{ $args }} https://127.0.0.1:{{ $endpoint_port }}{{ $endpointPath }} && \
|
||||
curl {{ $args }} https://127.0.0.1:{{ $console_port }}/rustfs/console/health
|
||||
{{- end -}}
|
||||
|
||||
@@ -350,7 +352,7 @@ livenessProbe:
|
||||
{{- if .Values.mtls.enabled }}
|
||||
exec:
|
||||
command:
|
||||
{{ include "rustfs.probeCommand" . | nindent 6 }}
|
||||
{{ include "rustfs.probeCommand" (dict "root" . "endpointPath" "/health") | nindent 6 }}
|
||||
{{- else }}
|
||||
httpGet:
|
||||
path: /health
|
||||
@@ -369,7 +371,7 @@ readinessProbe:
|
||||
{{- if .Values.mtls.enabled }}
|
||||
exec:
|
||||
command:
|
||||
{{ include "rustfs.probeCommand" . | nindent 6 }}
|
||||
{{ include "rustfs.probeCommand" (dict "root" . "endpointPath" "/health/ready") | nindent 6 }}
|
||||
{{- else }}
|
||||
httpGet:
|
||||
path: /health/ready
|
||||
|
||||
@@ -1,12 +1,37 @@
|
||||
{{- $logDir := .Values.config.rustfs.obs_log_directory }}
|
||||
{{- $logDirEnabled := ne $logDir "" }}
|
||||
{{- $poolsEnabled := and .Values.pools .Values.pools.enabled }}
|
||||
{{- $generatedVolumes := empty .Values.config.rustfs.volumes }}
|
||||
{{- $localEndpointHostAutoInject := empty .Values.secret.existingSecret }}
|
||||
{{- with .Values.localEndpointHost }}
|
||||
{{- if and (hasKey . "autoInject") (ne .autoInject nil) }}
|
||||
{{- if not (kindIs "bool" .autoInject) }}
|
||||
{{- fail "localEndpointHost.autoInject must be a boolean or null" }}
|
||||
{{- end }}
|
||||
{{- $localEndpointHostAutoInject = .autoInject }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- $injectLocalEndpointHost := and $generatedVolumes $localEndpointHostAutoInject }}
|
||||
{{- range $env := .Values.extraEnv -}}
|
||||
{{- $name := default "" $env.name -}}
|
||||
{{- if eq $name "RUSTFS_VOLUMES" -}}
|
||||
{{- $injectLocalEndpointHost = false -}}
|
||||
{{- else if eq $name "RUSTFS_LOCAL_ENDPOINT_HOST" -}}
|
||||
{{- $injectLocalEndpointHost = false -}}
|
||||
{{- else if eq $name "RUSTFS_STARTUP_TOPOLOGY_WAIT_MODE" -}}
|
||||
{{- if hasKey $env "value" -}}
|
||||
{{- $mode := lower (trim (toString $env.value)) -}}
|
||||
{{- if and (ne $mode "") (ne $mode "auto") (ne $mode "orchestrated") -}}
|
||||
{{- $injectLocalEndpointHost = false -}}
|
||||
{{- end -}}
|
||||
{{- else -}}
|
||||
{{- $injectLocalEndpointHost = false -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- if and .Values.mode.distributed.enabled (le (int .Values.replicaCount) 1) -}}
|
||||
{{- fail "Distributed mode requires replicaCount >= 2" -}}
|
||||
{{- end -}}
|
||||
{{- if and .Values.mode.distributed.enabled (le (int .Values.startupWaitTimeoutSeconds) 0) -}}
|
||||
{{- fail "startupWaitTimeoutSeconds must be greater than 0 in distributed mode" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- if .Values.mode.distributed.enabled }}
|
||||
{{- $pools := include "rustfs.pools" . | fromJsonArray }}
|
||||
@@ -74,13 +99,10 @@ spec:
|
||||
{{- if $.Values.affinity.podAntiAffinity.enabled }}
|
||||
podAntiAffinity:
|
||||
{{- if $poolsEnabled }}
|
||||
{{- /* Pool-scoped and PREFERRED, not required: during in-place
|
||||
expansion the not-yet-rolled pods' required rules block new
|
||||
pods from their nodes, while the new pods' DNS must resolve
|
||||
before the roll can proceed - required rules deadlock
|
||||
expansion whenever the cluster has fewer nodes than total
|
||||
pods. Soft anti-affinity converges (the scheduler still
|
||||
spreads when capacity allows). */}}
|
||||
{{- /* Pool-scoped and PREFERRED, not required: additional pools
|
||||
must remain schedulable when the cluster has fewer nodes
|
||||
than total pods. The scheduler still spreads them when
|
||||
capacity allows. */}}
|
||||
preferredDuringSchedulingIgnoredDuringExecution:
|
||||
- weight: 100
|
||||
podAffinityTerm:
|
||||
@@ -127,10 +149,6 @@ spec:
|
||||
env:
|
||||
- name: DRIVES_PER_NODE
|
||||
value: {{ $drivesPerNode | quote }}
|
||||
- name: ENDPOINT_PORT
|
||||
value: {{ $.Values.service.endpoint.port | quote }}
|
||||
- name: STARTUP_WAIT_TIMEOUT_SECONDS
|
||||
value: {{ $.Values.startupWaitTimeoutSeconds | quote }}
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
@@ -146,44 +164,6 @@ spec:
|
||||
mkdir -p /mnt/rustfs/logs
|
||||
chmod 755 /mnt/rustfs/logs
|
||||
{{- end }}
|
||||
deadline=$(($(date +%s) + STARTUP_WAIT_TIMEOUT_SECONDS))
|
||||
wait_for_peer() {
|
||||
description=$1
|
||||
shift
|
||||
until "$@" >/dev/null 2>&1; do
|
||||
if [ "$(date +%s)" -ge "$deadline" ]; then
|
||||
echo "Timed out after ${STARTUP_WAIT_TIMEOUT_SECONDS}s waiting for ${description}" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
}
|
||||
|
||||
echo "Waiting for distributed peer DNS records"
|
||||
{{- range $peerPool := $pools }}
|
||||
{{- range $i := until (int $peerPool.replicaCount) }}
|
||||
wait_for_peer "DNS for {{ $peerPool.fullname }}-{{ $i }}" \
|
||||
nslookup "{{ $peerPool.fullname }}-{{ $i }}.{{ include "rustfs.fullname" $ }}-headless.{{ $.Release.Namespace }}.svc.{{ include "rustfs.clusterDomain" $ }}"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
ordinal=${HOSTNAME##*-}
|
||||
echo "Waiting for earlier distributed peers to listen on port ${ENDPOINT_PORT}"
|
||||
{{- range $peerPool := $pools }}
|
||||
{{- if lt (int $peerPool.index) (int $pool.index) }}
|
||||
{{- range $i := until (int $peerPool.replicaCount) }}
|
||||
wait_for_peer "{{ $peerPool.fullname }}-{{ $i }}:${ENDPOINT_PORT}" \
|
||||
nc -z -w 2 "{{ $peerPool.fullname }}-{{ $i }}.{{ include "rustfs.fullname" $ }}-headless.{{ $.Release.Namespace }}.svc.{{ include "rustfs.clusterDomain" $ }}" "$ENDPOINT_PORT"
|
||||
{{- end }}
|
||||
{{- else if eq (int $peerPool.index) (int $pool.index) }}
|
||||
i=0
|
||||
while [ "$i" -lt "$ordinal" ]; do
|
||||
wait_for_peer "{{ $peerPool.fullname }}-${i}:${ENDPOINT_PORT}" \
|
||||
nc -z -w 2 "{{ $peerPool.fullname }}-${i}.{{ include "rustfs.fullname" $ }}-headless.{{ $.Release.Namespace }}.svc.{{ include "rustfs.clusterDomain" $ }}" "$ENDPOINT_PORT"
|
||||
i=$((i + 1))
|
||||
done
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
{{- if gt $drivesPerNode 1 }}
|
||||
{{- range $i := until $drivesPerNode }}
|
||||
@@ -210,9 +190,19 @@ spec:
|
||||
containerPort: {{ $.Values.service.endpoint.port }}
|
||||
- name: console
|
||||
containerPort: {{ $.Values.service.console.port }}
|
||||
{{- with $.Values.extraEnv }}
|
||||
{{- if or $injectLocalEndpointHost (not (empty $.Values.extraEnv)) }}
|
||||
env:
|
||||
{{- if $injectLocalEndpointHost }}
|
||||
- name: RUSTFS_CHART_POD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
- name: RUSTFS_LOCAL_ENDPOINT_HOST
|
||||
value: {{ printf "$(RUSTFS_CHART_POD_NAME).%s-headless.%s.svc.%s" (include "rustfs.fullname" $) $.Release.Namespace (include "rustfs.clusterDomain" $) | quote }}
|
||||
{{- end }}
|
||||
{{- with $.Values.extraEnv }}
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
|
||||
+26
-4
@@ -30,11 +30,22 @@ drivesPerNode: null
|
||||
# Provide the DNS root only, without a `svc.` prefix or leading/trailing dots.
|
||||
clusterDomain: cluster.local
|
||||
|
||||
# Maximum time an init container waits for distributed peers to publish DNS
|
||||
# and for earlier peers to start listening. The wait is shared across all
|
||||
# peers, not applied once per peer.
|
||||
# Deprecated and ignored. Retained so existing values files continue to
|
||||
# render after peer startup coordination moved into RustFS.
|
||||
# RUSTFS_COMPAT_TODO(rustfs-5416): Keep the removed init-gate setting visible to existing values files. Remove after the minimum supported direct-upgrade chart includes localEndpointHost.autoInject and no longer renders peer gates.
|
||||
startupWaitTimeoutSeconds: 300
|
||||
|
||||
# Inject a pod-specific endpoint identity for chart-generated distributed
|
||||
# topologies. The default enables injection unless secret.existingSecret is
|
||||
# set, because the chart cannot inspect runtime overrides hidden in that
|
||||
# Secret. Without injection, the server infers a unique matching domain
|
||||
# endpoint from its kernel hostname in Kubernetes auto/orchestrated mode; custom
|
||||
# aliases retain bounded legacy DNS locality when no hostname matches. Configure
|
||||
# RUSTFS_LOCAL_ENDPOINT_HOST explicitly for DNS-free startup. Set true for a
|
||||
# credentials-only existing Secret or false to suppress chart injection.
|
||||
localEndpointHost:
|
||||
autoInject: null
|
||||
|
||||
# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/
|
||||
image:
|
||||
rustfs: # This sets the rustfs image repository and tag.
|
||||
@@ -223,7 +234,18 @@ config:
|
||||
default_key: ""
|
||||
|
||||
|
||||
extraEnv: [] # This is for setting extra environment variables in the rustfs container. It should be a list of key value pairs. For example:
|
||||
# Extra environment variables for the RustFS container.
|
||||
# In distributed mode with chart-generated volumes (config.rustfs.volumes is
|
||||
# empty), the chart injects a private Downward API pod name and
|
||||
# RUSTFS_LOCAL_ENDPOINT_HOST to identify the pod's local endpoints. An explicit
|
||||
# RUSTFS_LOCAL_ENDPOINT_HOST or RUSTFS_VOLUMES entry disables automatic anchor
|
||||
# injection. An explicit bounded or fail-fast
|
||||
# RUSTFS_STARTUP_TOPOLOGY_WAIT_MODE, or an unrecognized static value, also keeps
|
||||
# legacy DNS-based locality discovery. Otherwise Kubernetes auto-detection
|
||||
# selects orchestrated startup when RustFS consumes the generated anchor.
|
||||
# With custom volumes, the chart does not inject them and extraEnv may define
|
||||
# the local endpoint identity explicitly.
|
||||
extraEnv: []
|
||||
# extraEnv:
|
||||
# - name: RUSTFS_ERASURE_SET_DRIVE_COUNT
|
||||
# value: "16"
|
||||
|
||||
Reference in New Issue
Block a user