mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-05 12:57:42 +00:00
fix(lifecycle): prevent eager date-expiry deletion on config update (#2708)
This commit is contained in:
houseme
@@ -36,6 +36,24 @@ To start a 4-node cluster for distributed testing:
|
||||
docker compose -f .docker/compose/docker-compose.cluster.yaml up -d
|
||||
```
|
||||
|
||||
### Script-Based 4-Node Validation (Recommended)
|
||||
|
||||
Use the local validation script when you need local-source image build, failover checks,
|
||||
and benchmark workflow in one command:
|
||||
|
||||
```bash
|
||||
# Default mode: WAIT_PROBE_MODE=service
|
||||
# This avoids false negatives where /health/ready remains 503 locally
|
||||
# while the service path is already available.
|
||||
./scripts/run_four_node_cluster_failover_bench.sh
|
||||
```
|
||||
|
||||
Strict mode is available when you explicitly want `/health/ready == 200` as the gate:
|
||||
|
||||
```bash
|
||||
WAIT_PROBE_MODE=ready ./scripts/run_four_node_cluster_failover_bench.sh
|
||||
```
|
||||
|
||||
### (Deprecated) Minimal Observability
|
||||
|
||||
```bash
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
services:
|
||||
node1:
|
||||
image: ${RUSTFS_IMAGE:-rustfs/rustfs:local-4node}
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: Dockerfile.source
|
||||
hostname: node1
|
||||
environment:
|
||||
- RUSTFS_VOLUMES=http://node{1...4}:9000/data/rustfs{0...3}
|
||||
- RUSTFS_ADDRESS=:9000
|
||||
- RUSTFS_CONSOLE_ENABLE=true
|
||||
- RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin}
|
||||
- RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfsadmin}
|
||||
- RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
|
||||
- RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
volumes:
|
||||
- node1_data_0:/data/rustfs0
|
||||
- node1_data_1:/data/rustfs1
|
||||
- node1_data_2:/data/rustfs2
|
||||
- node1_data_3:/data/rustfs3
|
||||
ports:
|
||||
- "9000:9000"
|
||||
networks:
|
||||
- rustfs-cluster-net
|
||||
|
||||
node2:
|
||||
image: ${RUSTFS_IMAGE:-rustfs/rustfs:local-4node}
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: Dockerfile.source
|
||||
hostname: node2
|
||||
environment:
|
||||
- RUSTFS_VOLUMES=http://node{1...4}:9000/data/rustfs{0...3}
|
||||
- RUSTFS_ADDRESS=:9000
|
||||
- RUSTFS_CONSOLE_ENABLE=true
|
||||
- RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin}
|
||||
- RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfsadmin}
|
||||
- RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
|
||||
- RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
volumes:
|
||||
- node2_data_0:/data/rustfs0
|
||||
- node2_data_1:/data/rustfs1
|
||||
- node2_data_2:/data/rustfs2
|
||||
- node2_data_3:/data/rustfs3
|
||||
ports:
|
||||
- "9001:9000"
|
||||
networks:
|
||||
- rustfs-cluster-net
|
||||
|
||||
node3:
|
||||
image: ${RUSTFS_IMAGE:-rustfs/rustfs:local-4node}
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: Dockerfile.source
|
||||
hostname: node3
|
||||
environment:
|
||||
- RUSTFS_VOLUMES=http://node{1...4}:9000/data/rustfs{0...3}
|
||||
- RUSTFS_ADDRESS=:9000
|
||||
- RUSTFS_CONSOLE_ENABLE=true
|
||||
- RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin}
|
||||
- RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfsadmin}
|
||||
- RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
|
||||
- RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
volumes:
|
||||
- node3_data_0:/data/rustfs0
|
||||
- node3_data_1:/data/rustfs1
|
||||
- node3_data_2:/data/rustfs2
|
||||
- node3_data_3:/data/rustfs3
|
||||
ports:
|
||||
- "9002:9000"
|
||||
networks:
|
||||
- rustfs-cluster-net
|
||||
|
||||
node4:
|
||||
image: ${RUSTFS_IMAGE:-rustfs/rustfs:local-4node}
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: Dockerfile.source
|
||||
hostname: node4
|
||||
environment:
|
||||
- RUSTFS_VOLUMES=http://node{1...4}:9000/data/rustfs{0...3}
|
||||
- RUSTFS_ADDRESS=:9000
|
||||
- RUSTFS_CONSOLE_ENABLE=true
|
||||
- RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin}
|
||||
- RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfsadmin}
|
||||
- RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
|
||||
- RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
volumes:
|
||||
- node4_data_0:/data/rustfs0
|
||||
- node4_data_1:/data/rustfs1
|
||||
- node4_data_2:/data/rustfs2
|
||||
- node4_data_3:/data/rustfs3
|
||||
ports:
|
||||
- "9003:9000"
|
||||
networks:
|
||||
- rustfs-cluster-net
|
||||
|
||||
volumes:
|
||||
node1_data_0:
|
||||
node1_data_1:
|
||||
node1_data_2:
|
||||
node1_data_3:
|
||||
node2_data_0:
|
||||
node2_data_1:
|
||||
node2_data_2:
|
||||
node2_data_3:
|
||||
node3_data_0:
|
||||
node3_data_1:
|
||||
node3_data_2:
|
||||
node3_data_3:
|
||||
node4_data_0:
|
||||
node4_data_1:
|
||||
node4_data_2:
|
||||
node4_data_3:
|
||||
|
||||
networks:
|
||||
rustfs-cluster-net:
|
||||
driver: bridge
|
||||
@@ -26,7 +26,7 @@ services:
|
||||
restart: "no"
|
||||
|
||||
tempo:
|
||||
image: grafana/tempo:latest
|
||||
image: grafana/tempo:2.10.5
|
||||
user: "10001"
|
||||
command: [ "-config.file=/etc/tempo.yaml" ]
|
||||
volumes:
|
||||
@@ -36,9 +36,19 @@ services:
|
||||
- "3200:3200" # tempo
|
||||
- "4317" # otlp grpc
|
||||
- "4318" # otlp http
|
||||
- "7946" # memberlist
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- rustfs-network
|
||||
depends_on:
|
||||
tempo-init:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: [ "CMD", "/tempo", "-version" ]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 15s
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:latest
|
||||
@@ -61,6 +71,12 @@ services:
|
||||
- jaeger
|
||||
- prometheus
|
||||
- loki
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: [ "CMD", "/otelcol-contrib", "--version" ]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
jaeger:
|
||||
image: jaegertracing/jaeger:latest
|
||||
@@ -72,12 +88,23 @@ services:
|
||||
- BADGER_DIRECTORY_KEY=/badger/key
|
||||
- COLLECTOR_OTLP_ENABLED=true
|
||||
volumes:
|
||||
- ../../.docker/observability/jaeger.yaml:/etc/jaeger/config.yml:ro
|
||||
- jaeger-data:/badger
|
||||
ports:
|
||||
- "16686:16686" # Web UI
|
||||
- "14269:14269" # Admin/Metrics
|
||||
- "4317" # otlp grpc
|
||||
- "4318" # otlp http
|
||||
command: [ "--config", "/etc/jaeger/config.yml" ]
|
||||
networks:
|
||||
- rustfs-network
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: [ "CMD", "wget", "--spider", "-q", "http://localhost:14269" ]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 15s
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:latest
|
||||
@@ -85,6 +112,7 @@ services:
|
||||
- TZ=Asia/Shanghai
|
||||
volumes:
|
||||
- ../../.docker/observability/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ../../.docker/observability/prometheus-rules:/etc/prometheus/rules:ro
|
||||
- prometheus-data:/prometheus
|
||||
ports:
|
||||
- "9090:9090"
|
||||
@@ -94,23 +122,45 @@ services:
|
||||
- '--web.enable-remote-write-receiver'
|
||||
- '--enable-feature=promql-experimental-functions'
|
||||
- '--storage.tsdb.path=/prometheus'
|
||||
- '--web.console.libraries=/usr/share/prometheus/console_libraries'
|
||||
- '--web.console.templates=/usr/share/prometheus/consoles'
|
||||
- '--storage.tsdb.retention.time=30d'
|
||||
networks:
|
||||
- rustfs-network
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: [ "CMD", "wget", "--spider", "-q", "http://localhost:9090/-/healthy" ]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
loki:
|
||||
image: grafana/loki:latest
|
||||
environment:
|
||||
- TZ=Asia/Shanghai
|
||||
volumes:
|
||||
- ../../.docker/observability/loki.yaml:/etc/loki/local-config.yaml:ro
|
||||
- ../../.docker/observability/loki.yaml:/etc/loki/loki.yaml:ro
|
||||
- loki-data:/loki
|
||||
ports:
|
||||
- "3100:3100"
|
||||
command: -config.file=/etc/loki/local-config.yaml
|
||||
command: -config.file=/etc/loki/loki.yaml
|
||||
networks:
|
||||
- rustfs-network
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3100/ready" ]
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
start_period: 60s
|
||||
|
||||
pyroscope:
|
||||
image: grafana/pyroscope:latest
|
||||
ports:
|
||||
- "4040:4040"
|
||||
command:
|
||||
- -self-profiling.disable-push=true
|
||||
networks:
|
||||
- rustfs-network
|
||||
restart: unless-stopped
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:latest
|
||||
@@ -127,10 +177,17 @@ services:
|
||||
volumes:
|
||||
- ../../.docker/observability/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ../../.docker/observability/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafana-data:/var/lib/grafana
|
||||
depends_on:
|
||||
- prometheus
|
||||
- tempo
|
||||
- loki
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health" ]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
|
||||
# --- RustFS Cluster ---
|
||||
|
||||
@@ -215,6 +272,7 @@ volumes:
|
||||
tempo-data:
|
||||
loki-data:
|
||||
jaeger-data:
|
||||
grafana-data:
|
||||
|
||||
networks:
|
||||
rustfs-network:
|
||||
|
||||
Generated
+105
-86
@@ -1473,12 +1473,6 @@ dependencies = [
|
||||
"shlex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cesu8"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c"
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.4"
|
||||
@@ -3196,7 +3190,7 @@ dependencies = [
|
||||
"libc",
|
||||
"option-ext",
|
||||
"redox_users",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3475,7 +3469,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4607,9 +4601,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "idna_adapter"
|
||||
version = "1.2.1"
|
||||
version = "1.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344"
|
||||
checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714"
|
||||
dependencies = [
|
||||
"icu_normalizer",
|
||||
"icu_properties",
|
||||
@@ -4746,7 +4740,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4823,7 +4817,7 @@ dependencies = [
|
||||
"portable-atomic",
|
||||
"portable-atomic-util",
|
||||
"serde_core",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4854,27 +4848,32 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jni"
|
||||
version = "0.21.1"
|
||||
version = "0.22.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97"
|
||||
checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498"
|
||||
dependencies = [
|
||||
"cesu8",
|
||||
"cfg-if",
|
||||
"combine",
|
||||
"jni-sys 0.3.1",
|
||||
"jni-macros",
|
||||
"jni-sys",
|
||||
"log",
|
||||
"thiserror 1.0.69",
|
||||
"simd_cesu8",
|
||||
"thiserror 2.0.18",
|
||||
"walkdir",
|
||||
"windows-sys 0.45.0",
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jni-sys"
|
||||
version = "0.3.1"
|
||||
name = "jni-macros"
|
||||
version = "0.22.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258"
|
||||
checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3"
|
||||
dependencies = [
|
||||
"jni-sys 0.4.1",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"rustc_version",
|
||||
"simd_cesu8",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5793,7 +5792,7 @@ version = "0.50.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
|
||||
dependencies = [
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5965,7 +5964,7 @@ version = "5.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
|
||||
dependencies = [
|
||||
"base64 0.21.7",
|
||||
"base64 0.22.1",
|
||||
"chrono",
|
||||
"getrandom 0.2.17",
|
||||
"http 1.4.0",
|
||||
@@ -6600,9 +6599,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pkcs8"
|
||||
version = "0.11.0-rc.11"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "12922b6296c06eb741b02d7b5161e3aaa22864af38dfa025a1a3ba3f68c84577"
|
||||
checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7"
|
||||
dependencies = [
|
||||
"der 0.8.0",
|
||||
"spki 0.8.0",
|
||||
@@ -7181,7 +7180,7 @@ dependencies = [
|
||||
"once_cell",
|
||||
"socket2",
|
||||
"tracing",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -7690,16 +7689,16 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rsa"
|
||||
version = "0.10.0-rc.17"
|
||||
version = "0.10.0-rc.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "87ed3e93fc7e473e464b9726f4759659e72bc8665e4b8ea227547024f416d905"
|
||||
checksum = "30b2aa4ba0d89f73d1e332df05be0eeab8840351c36ca5654341dfdb57bb3caf"
|
||||
dependencies = [
|
||||
"const-oid 0.10.2",
|
||||
"crypto-bigint 0.7.3",
|
||||
"crypto-primes",
|
||||
"digest 0.11.2",
|
||||
"pkcs1 0.8.0-rc.4",
|
||||
"pkcs8 0.11.0-rc.11",
|
||||
"pkcs8 0.11.0",
|
||||
"rand_core 0.10.1",
|
||||
"signature 3.0.0-rc.10",
|
||||
"spki 0.8.0",
|
||||
@@ -7936,7 +7935,7 @@ version = "0.0.5"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"rand 0.10.1",
|
||||
"rsa 0.10.0-rc.17",
|
||||
"rsa 0.10.0-rc.18",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
@@ -8569,6 +8568,8 @@ dependencies = [
|
||||
"flatbuffers",
|
||||
"prost 0.14.3",
|
||||
"rustfs-common",
|
||||
"rustfs-config",
|
||||
"rustfs-utils",
|
||||
"tonic",
|
||||
"tonic-prost",
|
||||
"tonic-prost-build",
|
||||
@@ -8889,7 +8890,7 @@ dependencies = [
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys 0.12.1",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8932,9 +8933,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustls-platform-verifier"
|
||||
version = "0.6.2"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784"
|
||||
checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0"
|
||||
dependencies = [
|
||||
"core-foundation 0.10.1",
|
||||
"core-foundation-sys",
|
||||
@@ -8948,7 +8949,7 @@ dependencies = [
|
||||
"security-framework",
|
||||
"security-framework-sys",
|
||||
"webpki-root-certs",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9473,6 +9474,16 @@ version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214"
|
||||
|
||||
[[package]]
|
||||
name = "simd_cesu8"
|
||||
version = "1.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33"
|
||||
dependencies = [
|
||||
"rustc_version",
|
||||
"simdutf8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "simdutf8"
|
||||
version = "0.1.5"
|
||||
@@ -9955,7 +9966,7 @@ dependencies = [
|
||||
"getrandom 0.4.2",
|
||||
"once_cell",
|
||||
"rustix 1.1.4",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10945,7 +10956,7 @@ version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11066,15 +11077,6 @@ dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.45.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
|
||||
dependencies = [
|
||||
"windows-targets 0.42.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.52.0"
|
||||
@@ -11093,6 +11095,15 @@ dependencies = [
|
||||
"windows-targets 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.60.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
|
||||
dependencies = [
|
||||
"windows-targets 0.53.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.61.2"
|
||||
@@ -11102,21 +11113,6 @@ dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-targets"
|
||||
version = "0.42.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"
|
||||
dependencies = [
|
||||
"windows_aarch64_gnullvm 0.42.2",
|
||||
"windows_aarch64_msvc 0.42.2",
|
||||
"windows_i686_gnu 0.42.2",
|
||||
"windows_i686_msvc 0.42.2",
|
||||
"windows_x86_64_gnu 0.42.2",
|
||||
"windows_x86_64_gnullvm 0.42.2",
|
||||
"windows_x86_64_msvc 0.42.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-targets"
|
||||
version = "0.52.6"
|
||||
@@ -11126,13 +11122,30 @@ dependencies = [
|
||||
"windows_aarch64_gnullvm 0.52.6",
|
||||
"windows_aarch64_msvc 0.52.6",
|
||||
"windows_i686_gnu 0.52.6",
|
||||
"windows_i686_gnullvm",
|
||||
"windows_i686_gnullvm 0.52.6",
|
||||
"windows_i686_msvc 0.52.6",
|
||||
"windows_x86_64_gnu 0.52.6",
|
||||
"windows_x86_64_gnullvm 0.52.6",
|
||||
"windows_x86_64_msvc 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-targets"
|
||||
version = "0.53.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
"windows_aarch64_gnullvm 0.53.1",
|
||||
"windows_aarch64_msvc 0.53.1",
|
||||
"windows_i686_gnu 0.53.1",
|
||||
"windows_i686_gnullvm 0.53.1",
|
||||
"windows_i686_msvc 0.53.1",
|
||||
"windows_x86_64_gnu 0.53.1",
|
||||
"windows_x86_64_gnullvm 0.53.1",
|
||||
"windows_x86_64_msvc 0.53.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-threading"
|
||||
version = "0.2.1"
|
||||
@@ -11142,12 +11155,6 @@ dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.42.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.52.6"
|
||||
@@ -11155,10 +11162,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.42.2"
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
|
||||
checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
@@ -11167,10 +11174,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.42.2"
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
|
||||
checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
@@ -11178,6 +11185,12 @@ version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnullvm"
|
||||
version = "0.52.6"
|
||||
@@ -11185,10 +11198,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.42.2"
|
||||
name = "windows_i686_gnullvm"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
|
||||
checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
@@ -11197,10 +11210,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.42.2"
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
|
||||
checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
@@ -11209,10 +11222,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnullvm"
|
||||
version = "0.42.2"
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
|
||||
checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnullvm"
|
||||
@@ -11221,10 +11234,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.42.2"
|
||||
name = "windows_x86_64_gnullvm"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
|
||||
checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
@@ -11232,6 +11245,12 @@ version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
|
||||
|
||||
[[package]]
|
||||
name = "wit-bindgen"
|
||||
version = "0.51.0"
|
||||
|
||||
+1
-1
@@ -169,7 +169,7 @@ hmac = { version = "0.13.0" }
|
||||
jsonwebtoken = { version = "10.3.0", features = ["aws_lc_rs"] }
|
||||
openidconnect = { version = "4.0", default-features = false }
|
||||
pbkdf2 = "0.13.0"
|
||||
rsa = { version = "0.10.0-rc.17" }
|
||||
rsa = { version = "0.10.0-rc.18" }
|
||||
rustls = { version = "0.23.39", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
|
||||
rustls-pki-types = "1.14.1"
|
||||
sha1 = "0.11.0"
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
FROM rust:1.91-trixie
|
||||
FROM rust:1.93-trixie
|
||||
|
||||
RUN set -eux; \
|
||||
export DEBIAN_FRONTEND=noninteractive; \
|
||||
|
||||
+22
-5
@@ -30,7 +30,7 @@ ARG BUILDPLATFORM
|
||||
# -----------------------------
|
||||
# Build stage
|
||||
# -----------------------------
|
||||
FROM rust:1.91-trixie AS builder
|
||||
FROM rust:1.93-trixie AS builder
|
||||
|
||||
# Re-declare args after FROM
|
||||
ARG TARGETPLATFORM
|
||||
@@ -60,7 +60,15 @@ RUN set -eux; \
|
||||
|
||||
# Optional: cross toolchain for aarch64 (only when targeting linux/arm64)
|
||||
RUN set -eux; \
|
||||
if [ "${TARGETPLATFORM:-linux/amd64}" = "linux/arm64" ]; then \
|
||||
target_platform="${TARGETPLATFORM:-}"; \
|
||||
if [ -z "${target_platform}" ]; then \
|
||||
case "$(uname -m)" in \
|
||||
x86_64) target_platform="linux/amd64" ;; \
|
||||
aarch64|arm64) target_platform="linux/arm64" ;; \
|
||||
*) target_platform="linux/amd64" ;; \
|
||||
esac; \
|
||||
fi; \
|
||||
if [ "${target_platform}" = "linux/arm64" ]; then \
|
||||
export DEBIAN_FRONTEND=noninteractive; \
|
||||
apt-get update; \
|
||||
apt-get install -y --no-install-recommends gcc-aarch64-linux-gnu; \
|
||||
@@ -120,7 +128,16 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
--mount=type=cache,target=/usr/local/cargo/git \
|
||||
--mount=type=cache,target=/usr/src/rustfs/target \
|
||||
set -eux; \
|
||||
case "${TARGETPLATFORM:-linux/amd64}" in \
|
||||
rustup target add x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu; \
|
||||
target_platform="${TARGETPLATFORM:-}"; \
|
||||
if [ -z "${target_platform}" ]; then \
|
||||
case "$(uname -m)" in \
|
||||
x86_64) target_platform="linux/amd64" ;; \
|
||||
aarch64|arm64) target_platform="linux/arm64" ;; \
|
||||
*) target_platform="linux/amd64" ;; \
|
||||
esac; \
|
||||
fi; \
|
||||
case "${target_platform}" in \
|
||||
linux/amd64) \
|
||||
echo "Building for x86_64-unknown-linux-gnu"; \
|
||||
cargo build --release --locked --target x86_64-unknown-linux-gnu --bin rustfs -j "$(nproc)"; \
|
||||
@@ -132,7 +149,7 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
install -m 0755 target/aarch64-unknown-linux-gnu/release/rustfs /usr/local/bin/rustfs \
|
||||
;; \
|
||||
*) \
|
||||
echo "Unsupported TARGETPLATFORM=${TARGETPLATFORM}" >&2; exit 1 \
|
||||
echo "Unsupported target platform=${target_platform}" >&2; exit 1 \
|
||||
;; \
|
||||
esac
|
||||
|
||||
@@ -178,7 +195,7 @@ CMD ["cargo", "run", "--bin", "rustfs", "--"]
|
||||
# -----------------------------
|
||||
# Runtime stage (Ubuntu minimal)
|
||||
# -----------------------------
|
||||
FROM ubuntu:22.04
|
||||
FROM ubuntu:24.04
|
||||
|
||||
ARG BUILD_DATE
|
||||
ARG VCS_REF
|
||||
|
||||
@@ -39,6 +39,14 @@ pub const DEFAULT_DRIVE_WALKDIR_TIMEOUT_SECS: u64 = 5;
|
||||
pub const ENV_DRIVE_WALKDIR_STALL_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_WALKDIR_STALL_TIMEOUT_SECS";
|
||||
pub const DEFAULT_DRIVE_WALKDIR_STALL_TIMEOUT_SECS: u64 = 5;
|
||||
|
||||
/// Interval in seconds between active health probes for local and remote drives.
|
||||
pub const ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS: &str = "RUSTFS_DRIVE_ACTIVE_CHECK_INTERVAL_SECS";
|
||||
pub const DEFAULT_DRIVE_ACTIVE_CHECK_INTERVAL_SECS: u64 = 15;
|
||||
|
||||
/// Timeout in seconds for a single active health probe.
|
||||
pub const ENV_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS";
|
||||
pub const DEFAULT_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS: u64 = 5;
|
||||
|
||||
/// Number of consecutive failures before a suspect drive is classified as offline.
|
||||
pub const ENV_DRIVE_SUSPECT_FAILURE_THRESHOLD: &str = "RUSTFS_DRIVE_SUSPECT_FAILURE_THRESHOLD";
|
||||
pub const DEFAULT_DRIVE_SUSPECT_FAILURE_THRESHOLD: u64 = 2;
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
/// Timeout for establishing a new internode gRPC connection.
|
||||
pub const ENV_INTERNODE_CONNECT_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_CONNECT_TIMEOUT_SECS";
|
||||
pub const DEFAULT_INTERNODE_CONNECT_TIMEOUT_SECS: u64 = 3;
|
||||
|
||||
/// TCP keepalive interval for internode gRPC channels.
|
||||
pub const ENV_INTERNODE_TCP_KEEPALIVE_SECS: &str = "RUSTFS_INTERNODE_TCP_KEEPALIVE_SECS";
|
||||
pub const DEFAULT_INTERNODE_TCP_KEEPALIVE_SECS: u64 = 10;
|
||||
|
||||
/// HTTP/2 keepalive interval for internode gRPC channels.
|
||||
pub const ENV_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS: &str = "RUSTFS_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS";
|
||||
pub const DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS: u64 = 5;
|
||||
|
||||
/// HTTP/2 keepalive timeout for internode gRPC channels.
|
||||
pub const ENV_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS";
|
||||
pub const DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS: u64 = 3;
|
||||
|
||||
/// Overall timeout for a single internode gRPC request.
|
||||
pub const ENV_INTERNODE_RPC_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS";
|
||||
pub const DEFAULT_INTERNODE_RPC_TIMEOUT_SECS: u64 = 10;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn internode_timeout_defaults_stay_in_expected_bounds() {
|
||||
assert_eq!(DEFAULT_INTERNODE_CONNECT_TIMEOUT_SECS, 3);
|
||||
assert_eq!(DEFAULT_INTERNODE_TCP_KEEPALIVE_SECS, 10);
|
||||
assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS, 5);
|
||||
assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS, 3);
|
||||
assert_eq!(DEFAULT_INTERNODE_RPC_TIMEOUT_SECS, 10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn internode_timeout_env_names_are_stable() {
|
||||
assert_eq!(ENV_INTERNODE_CONNECT_TIMEOUT_SECS, "RUSTFS_INTERNODE_CONNECT_TIMEOUT_SECS");
|
||||
assert_eq!(ENV_INTERNODE_TCP_KEEPALIVE_SECS, "RUSTFS_INTERNODE_TCP_KEEPALIVE_SECS");
|
||||
assert_eq!(
|
||||
ENV_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS,
|
||||
"RUSTFS_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS"
|
||||
);
|
||||
assert_eq!(
|
||||
ENV_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS,
|
||||
"RUSTFS_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS"
|
||||
);
|
||||
assert_eq!(ENV_INTERNODE_RPC_TIMEOUT_SECS, "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS");
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,7 @@ pub(crate) mod drive;
|
||||
pub(crate) mod env;
|
||||
pub(crate) mod heal;
|
||||
pub(crate) mod health;
|
||||
pub(crate) mod internode;
|
||||
pub(crate) mod object;
|
||||
pub(crate) mod oidc;
|
||||
pub(crate) mod profiler;
|
||||
|
||||
@@ -33,6 +33,8 @@ pub use constants::heal::*;
|
||||
#[cfg(feature = "constants")]
|
||||
pub use constants::health::*;
|
||||
#[cfg(feature = "constants")]
|
||||
pub use constants::internode::*;
|
||||
#[cfg(feature = "constants")]
|
||||
pub use constants::object::*;
|
||||
#[cfg(feature = "constants")]
|
||||
pub use constants::profiler::*;
|
||||
|
||||
@@ -87,25 +87,70 @@ impl AsyncBatchProcessor {
|
||||
T: Send + 'static,
|
||||
F: Future<Output = Result<T>> + Send + 'static,
|
||||
{
|
||||
let results = self.execute_batch(tasks).await;
|
||||
let mut successes = Vec::new();
|
||||
if required_successes == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
for value in results.into_iter().flatten() {
|
||||
successes.push(value);
|
||||
if successes.len() >= required_successes {
|
||||
return Ok(successes);
|
||||
if tasks.is_empty() {
|
||||
return Err(Error::other(format!(
|
||||
"Insufficient successful results: got 0, needed {required_successes}"
|
||||
)));
|
||||
}
|
||||
|
||||
let semaphore = Arc::new(tokio::sync::Semaphore::new(self.max_concurrent));
|
||||
let mut join_set = JoinSet::new();
|
||||
let mut successes = Vec::new();
|
||||
let mut pending_tasks = tasks.len();
|
||||
let mut first_error = None;
|
||||
|
||||
for task in tasks {
|
||||
let sem = semaphore.clone();
|
||||
join_set.spawn(async move {
|
||||
let _permit = sem.acquire().await.map_err(|_| Error::other("Semaphore error"))?;
|
||||
task.await
|
||||
});
|
||||
}
|
||||
|
||||
while let Some(join_result) = join_set.join_next().await {
|
||||
pending_tasks = pending_tasks.saturating_sub(1);
|
||||
|
||||
match join_result {
|
||||
Ok(Ok(value)) => {
|
||||
successes.push(value);
|
||||
if successes.len() >= required_successes {
|
||||
return Ok(successes);
|
||||
}
|
||||
}
|
||||
Ok(Err(err)) => {
|
||||
if first_error.is_none() {
|
||||
first_error = Some(err);
|
||||
}
|
||||
}
|
||||
Err(join_error) => {
|
||||
if first_error.is_none() {
|
||||
first_error = Some(Error::other(format!("Task panicked in quorum batch processor: {join_error}")));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if successes.len() + pending_tasks < required_successes {
|
||||
return Err(first_error.unwrap_or_else(|| {
|
||||
Error::other(format!(
|
||||
"Insufficient successful results: got {}, needed {}",
|
||||
successes.len(),
|
||||
required_successes
|
||||
))
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
if successes.len() >= required_successes {
|
||||
Ok(successes)
|
||||
} else {
|
||||
Err(Error::other(format!(
|
||||
Err(first_error.unwrap_or_else(|| {
|
||||
Error::other(format!(
|
||||
"Insufficient successful results: got {}, needed {}",
|
||||
successes.len(),
|
||||
required_successes
|
||||
)))
|
||||
}
|
||||
))
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -228,4 +273,52 @@ mod tests {
|
||||
let successes = results.unwrap();
|
||||
assert!(successes.len() >= 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_batch_processor_quorum_returns_before_slow_tail() {
|
||||
let processor = AsyncBatchProcessor::new(4);
|
||||
let started = std::time::Instant::now();
|
||||
|
||||
let tasks: Vec<_> = [(10_u64, Ok(1_i32)), (15, Ok(2)), (250, Ok(3))]
|
||||
.into_iter()
|
||||
.map(|(delay_ms, outcome)| async move {
|
||||
tokio::time::sleep(Duration::from_millis(delay_ms)).await;
|
||||
outcome
|
||||
})
|
||||
.collect();
|
||||
|
||||
let results = processor
|
||||
.execute_batch_with_quorum(tasks, 2)
|
||||
.await
|
||||
.expect("quorum should succeed");
|
||||
assert_eq!(results.len(), 2);
|
||||
assert!(started.elapsed() < Duration::from_millis(100));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_batch_processor_quorum_fails_once_quorum_becomes_impossible() {
|
||||
let processor = AsyncBatchProcessor::new(4);
|
||||
let started = std::time::Instant::now();
|
||||
|
||||
let tasks: Vec<_> = vec![
|
||||
(10_u64, Ok(1_i32)),
|
||||
(15, Err(Error::other("first failure"))),
|
||||
(20, Err(Error::other("second failure"))),
|
||||
(250, Ok(4)),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(delay_ms, outcome)| async move {
|
||||
tokio::time::sleep(Duration::from_millis(delay_ms)).await;
|
||||
outcome
|
||||
})
|
||||
.collect();
|
||||
|
||||
let err = processor
|
||||
.execute_batch_with_quorum(tasks, 3)
|
||||
.await
|
||||
.expect_err("quorum should fail once it becomes impossible");
|
||||
|
||||
assert!(err.to_string().contains("first failure"));
|
||||
assert!(started.elapsed() < Duration::from_millis(120));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,8 +54,8 @@ use rustfs_filemeta::{
|
||||
use rustfs_s3_common::EventName;
|
||||
use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::strings_has_prefix_fold};
|
||||
use s3s::dto::{
|
||||
BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration, RestoreRequest, RestoreRequestType, RestoreStatus,
|
||||
Timestamp,
|
||||
BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, ReplicationConfiguration, RestoreRequest,
|
||||
RestoreRequestType, RestoreStatus, Timestamp,
|
||||
};
|
||||
use s3s::header::{X_AMZ_RESTORE, X_AMZ_SERVER_SIDE_ENCRYPTION};
|
||||
use sha2::{Digest, Sha256};
|
||||
@@ -92,6 +92,7 @@ const ENV_STALE_UPLOADS_EXPIRY: &str = "RUSTFS_API_STALE_UPLOADS_EXPIRY";
|
||||
const ENV_STALE_UPLOADS_CLEANUP_INTERVAL: &str = "RUSTFS_API_STALE_UPLOADS_CLEANUP_INTERVAL";
|
||||
const DEFAULT_STALE_UPLOADS_EXPIRY: StdDuration = StdDuration::from_secs(24 * 60 * 60);
|
||||
const DEFAULT_STALE_UPLOADS_CLEANUP_INTERVAL: StdDuration = StdDuration::from_secs(6 * 60 * 60);
|
||||
const DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS: i64 = 5;
|
||||
|
||||
lazy_static! {
|
||||
pub static ref GLOBAL_ExpiryState: Arc<RwLock<ExpiryState>> = ExpiryState::new();
|
||||
@@ -1241,6 +1242,29 @@ pub async fn enqueue_transition_for_existing_objects(api: Arc<ECStore>, bucket:
|
||||
}
|
||||
}
|
||||
|
||||
fn lifecycle_rule_has_date_expiration(lc: &BucketLifecycleConfiguration, rule_id: &str) -> bool {
|
||||
lc.rules.iter().any(|rule| {
|
||||
rule.status == ExpirationStatus::from_static(ExpirationStatus::ENABLED)
|
||||
&& rule.id.as_deref() == Some(rule_id)
|
||||
&& rule.expiration.as_ref().is_some_and(|expiration| expiration.date.is_some())
|
||||
})
|
||||
}
|
||||
|
||||
fn should_defer_date_expiry_for_recent_config_update(lc: &BucketLifecycleConfiguration, now: OffsetDateTime) -> bool {
|
||||
lc.expiry_updated_at.as_ref().is_some_and(|updated_at| {
|
||||
let updated_at = OffsetDateTime::from(updated_at.clone());
|
||||
now.unix_timestamp().saturating_sub(updated_at.unix_timestamp()) < DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS
|
||||
})
|
||||
}
|
||||
|
||||
async fn apply_existing_object_expiry(api: Arc<ECStore>, object: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) {
|
||||
if object.is_remote() {
|
||||
apply_expiry_on_transitioned_object(api, object, event, src).await;
|
||||
} else {
|
||||
apply_expiry_on_non_transitioned_objects(api, object, event, src).await;
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn enqueue_expiry_for_existing_objects(api: Arc<ECStore>, bucket: &str) -> Result<(), Error> {
|
||||
let Ok((lc, _)) = metadata_sys::get_lifecycle_config(bucket).await else {
|
||||
return Ok(());
|
||||
@@ -1253,6 +1277,8 @@ pub async fn enqueue_expiry_for_existing_objects(api: Arc<ECStore>, bucket: &str
|
||||
let mut marker = None;
|
||||
let mut version_marker = None;
|
||||
let src = LcEventSrc::Scanner;
|
||||
let defer_date_expiry_once = should_defer_date_expiry_for_recent_config_update(&lc, OffsetDateTime::now_utc());
|
||||
let mut date_expiry_deferred_once = false;
|
||||
|
||||
loop {
|
||||
let page = api
|
||||
@@ -1269,15 +1295,16 @@ pub async fn enqueue_expiry_for_existing_objects(api: Arc<ECStore>, bucket: &str
|
||||
| IlmAction::DeleteRestoredVersionAction
|
||||
| IlmAction::DeleteAllVersionsAction
|
||||
| IlmAction::DelMarkerDeleteAllVersionsAction => {
|
||||
if event
|
||||
.due
|
||||
.is_some_and(|due| due.unix_timestamp() <= OffsetDateTime::now_utc().unix_timestamp())
|
||||
{
|
||||
if object.is_remote() {
|
||||
apply_expiry_on_transitioned_object(api.clone(), object, &event, &src).await;
|
||||
} else {
|
||||
apply_expiry_on_non_transitioned_objects(api.clone(), object, &event, &src).await;
|
||||
let now = OffsetDateTime::now_utc();
|
||||
if event.due.is_some_and(|due| due.unix_timestamp() <= now.unix_timestamp()) {
|
||||
if defer_date_expiry_once
|
||||
&& !date_expiry_deferred_once
|
||||
&& lifecycle_rule_has_date_expiration(&lc, &event.rule_id)
|
||||
{
|
||||
tokio::time::sleep(StdDuration::from_secs(DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS as u64)).await;
|
||||
date_expiry_deferred_once = true;
|
||||
}
|
||||
apply_existing_object_expiry(api.clone(), object, &event, &src).await;
|
||||
} else {
|
||||
apply_expiry_rule(&event, &src, object).await;
|
||||
}
|
||||
@@ -1977,9 +2004,10 @@ pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc,
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
StaleMultipartUploadCandidate, cleanup_empty_multipart_sha_dirs_on_local_disks, cleanup_stale_multipart_uploads_once_at,
|
||||
lifecycle_deleted_object, lifecycle_version_purge_state_from_completed_targets,
|
||||
mark_delete_opts_skip_decommissioned_on_remote_success, merge_stale_multipart_candidate, replication_state_for_delete,
|
||||
DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS, StaleMultipartUploadCandidate, cleanup_empty_multipart_sha_dirs_on_local_disks,
|
||||
cleanup_stale_multipart_uploads_once_at, lifecycle_deleted_object, lifecycle_rule_has_date_expiration,
|
||||
lifecycle_version_purge_state_from_completed_targets, mark_delete_opts_skip_decommissioned_on_remote_success,
|
||||
merge_stale_multipart_candidate, replication_state_for_delete, should_defer_date_expiry_for_recent_config_update,
|
||||
should_reuse_lifecycle_delete_replication_state,
|
||||
};
|
||||
use crate::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
|
||||
@@ -1994,6 +2022,7 @@ mod tests {
|
||||
BucketOperations, BucketOptions, MakeBucketOptions, MultipartOperations, ObjectInfo, ObjectOptions, PutObjReader,
|
||||
};
|
||||
use rustfs_filemeta::{ReplicateDecision, VersionPurgeStatusType};
|
||||
use s3s::dto::{BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, Timestamp};
|
||||
use serial_test::serial;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashMap;
|
||||
@@ -2014,6 +2043,49 @@ mod tests {
|
||||
assert!(opts.skip_decommissioned);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lifecycle_rule_has_date_expiration_detects_enabled_date_rule() {
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: Some(LifecycleExpiration {
|
||||
date: Some(Timestamp::from(OffsetDateTime::now_utc())),
|
||||
..Default::default()
|
||||
}),
|
||||
id: Some("rule-date".to_string()),
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
filter: None,
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: None,
|
||||
}],
|
||||
};
|
||||
|
||||
assert!(lifecycle_rule_has_date_expiration(&lc, "rule-date"));
|
||||
assert!(!lifecycle_rule_has_date_expiration(&lc, "missing-rule"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_defer_date_expiry_for_recent_config_update_respects_grace_window() {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let recent = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: Some(Timestamp::from(now - time::Duration::seconds(1))),
|
||||
rules: Vec::new(),
|
||||
};
|
||||
let stale = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: Some(Timestamp::from(
|
||||
now - time::Duration::seconds(DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS + 1),
|
||||
)),
|
||||
rules: Vec::new(),
|
||||
};
|
||||
|
||||
assert!(should_defer_date_expiry_for_recent_config_update(&recent, now));
|
||||
assert!(!should_defer_date_expiry_for_recent_config_update(&stale, now));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mark_delete_opts_skip_decommissioned_on_remote_success_preserves_false_on_failure() {
|
||||
let mut opts = ObjectOptions::default();
|
||||
|
||||
@@ -46,9 +46,7 @@ const DISK_HEALTH_FAULTY: u32 = 1;
|
||||
|
||||
pub const ENV_RUSTFS_DRIVE_ACTIVE_MONITORING: &str = "RUSTFS_DRIVE_ACTIVE_MONITORING";
|
||||
pub const DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING: bool = true;
|
||||
pub const CHECK_EVERY: Duration = Duration::from_secs(15);
|
||||
pub const SKIP_IF_SUCCESS_BEFORE: Duration = Duration::from_secs(5);
|
||||
pub const CHECK_TIMEOUT_DURATION: Duration = Duration::from_secs(5);
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
static ref TEST_DATA: Bytes = Bytes::from(vec![42u8; 2048]);
|
||||
@@ -104,6 +102,20 @@ pub fn get_drive_walkdir_stall_timeout() -> Duration {
|
||||
)
|
||||
}
|
||||
|
||||
pub fn get_drive_active_check_interval() -> Duration {
|
||||
Duration::from_secs(rustfs_utils::get_env_u64(
|
||||
rustfs_config::ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS,
|
||||
rustfs_config::DEFAULT_DRIVE_ACTIVE_CHECK_INTERVAL_SECS,
|
||||
))
|
||||
}
|
||||
|
||||
pub fn get_drive_active_check_timeout() -> Duration {
|
||||
Duration::from_secs(rustfs_utils::get_env_u64(
|
||||
rustfs_config::ENV_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS,
|
||||
rustfs_config::DEFAULT_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS,
|
||||
))
|
||||
}
|
||||
|
||||
/// DiskHealthTracker tracks the health status of a disk.
|
||||
/// Similar to Go's diskHealthTracker.
|
||||
#[derive(Debug)]
|
||||
@@ -474,7 +486,7 @@ impl LocalDiskWrapper {
|
||||
async fn monitor_disk_writable(disk: Arc<LocalDisk>, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
|
||||
// TODO: config interval
|
||||
|
||||
let mut interval = time::interval(CHECK_EVERY);
|
||||
let mut interval = time::interval(get_drive_active_check_interval());
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
@@ -507,7 +519,16 @@ impl LocalDiskWrapper {
|
||||
|
||||
|
||||
let test_obj = format!("health-check-{}", Uuid::new_v4());
|
||||
if Self::perform_health_check(disk.clone(), &TEST_BUCKET, &test_obj, &TEST_DATA, true, CHECK_TIMEOUT_DURATION).await.is_err()
|
||||
if Self::perform_health_check(
|
||||
disk.clone(),
|
||||
&TEST_BUCKET,
|
||||
&test_obj,
|
||||
&TEST_DATA,
|
||||
true,
|
||||
get_drive_active_check_timeout(),
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
&& health.mark_failure(&disk.endpoint(), "active_health_check_failed")
|
||||
{
|
||||
// Health check failed, disk is considered faulty
|
||||
@@ -613,7 +634,16 @@ impl LocalDiskWrapper {
|
||||
}
|
||||
|
||||
let test_obj = format!("health-check-{}", Uuid::new_v4());
|
||||
match Self::perform_health_check(disk.clone(), &TEST_BUCKET, &test_obj, &TEST_DATA, false, CHECK_TIMEOUT_DURATION).await {
|
||||
match Self::perform_health_check(
|
||||
disk.clone(),
|
||||
&TEST_BUCKET,
|
||||
&test_obj,
|
||||
&TEST_DATA,
|
||||
false,
|
||||
get_drive_active_check_timeout(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
let state_before = health.runtime_state();
|
||||
let is_online = health.mark_recovery_success(&disk.endpoint(), "recovery_probe_success");
|
||||
@@ -1137,6 +1167,40 @@ mod tests {
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drive_active_check_interval_uses_default_when_unset() {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, || {
|
||||
assert_eq!(
|
||||
get_drive_active_check_interval(),
|
||||
Duration::from_secs(rustfs_config::DEFAULT_DRIVE_ACTIVE_CHECK_INTERVAL_SECS)
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drive_active_check_interval_reads_env_override() {
|
||||
temp_env::with_var(rustfs_config::ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, Some("3"), || {
|
||||
assert_eq!(get_drive_active_check_interval(), Duration::from_secs(3));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drive_active_check_timeout_uses_default_when_unset() {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS, || {
|
||||
assert_eq!(
|
||||
get_drive_active_check_timeout(),
|
||||
Duration::from_secs(rustfs_config::DEFAULT_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS)
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drive_active_check_timeout_reads_env_override() {
|
||||
temp_env::with_var(rustfs_config::ENV_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS, Some("1"), || {
|
||||
assert_eq!(get_drive_active_check_timeout(), Duration::from_secs(1));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_state_transitions_from_online_to_suspect_then_offline() {
|
||||
temp_env::with_var(rustfs_config::ENV_DRIVE_SUSPECT_FAILURE_THRESHOLD, Some("2"), || {
|
||||
|
||||
@@ -12,11 +12,12 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::disk::error::{DiskError, Error as DiskErrorType};
|
||||
use crate::rpc::{TONIC_RPC_PREFIX, gen_signature_headers};
|
||||
use http::Method;
|
||||
use rustfs_common::GLOBAL_CONN_MAP;
|
||||
use rustfs_protos::{create_new_channel, proto_gen::node_service::node_service_client::NodeServiceClient};
|
||||
use std::error::Error;
|
||||
use std::{error::Error, io::ErrorKind};
|
||||
use tonic::{service::interceptor::InterceptedService, transport::Channel};
|
||||
use tracing::debug;
|
||||
|
||||
@@ -51,6 +52,46 @@ pub async fn node_service_time_out_client_no_auth(
|
||||
node_service_time_out_client(addr, TonicInterceptor::NoOp(NoOpInterceptor)).await
|
||||
}
|
||||
|
||||
pub(crate) fn is_network_like_disk_error(err: &DiskErrorType) -> bool {
|
||||
match err {
|
||||
DiskError::Timeout => true,
|
||||
DiskError::Io(io_err) => {
|
||||
if matches!(
|
||||
io_err.kind(),
|
||||
ErrorKind::TimedOut
|
||||
| ErrorKind::ConnectionRefused
|
||||
| ErrorKind::ConnectionReset
|
||||
| ErrorKind::BrokenPipe
|
||||
| ErrorKind::NotConnected
|
||||
| ErrorKind::ConnectionAborted
|
||||
| ErrorKind::UnexpectedEof
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
let message = io_err.to_string().to_ascii_lowercase();
|
||||
[
|
||||
"transport error",
|
||||
"unavailable",
|
||||
"error trying to connect",
|
||||
"connection refused",
|
||||
"connection reset",
|
||||
"broken pipe",
|
||||
"not connected",
|
||||
"unexpected eof",
|
||||
"timed out",
|
||||
"deadline has elapsed",
|
||||
"connection closed",
|
||||
"connection aborted",
|
||||
"tcp connect error",
|
||||
]
|
||||
.iter()
|
||||
.any(|needle| message.contains(needle))
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TonicSignatureInterceptor;
|
||||
|
||||
impl tonic::service::Interceptor for TonicSignatureInterceptor {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -18,13 +18,15 @@ use crate::disk::error::{Error, Result};
|
||||
use crate::disk::error_reduce::{BUCKET_OP_IGNORED_ERRS, is_all_buckets_not_found, reduce_write_quorum_errs};
|
||||
use crate::disk::{DiskAPI, DiskStore, disk_store::get_max_timeout_duration};
|
||||
use crate::global::GLOBAL_LOCAL_DISK_MAP;
|
||||
use crate::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
|
||||
use crate::rpc::client::{
|
||||
TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
|
||||
};
|
||||
use crate::store::all_local_disk;
|
||||
use crate::store_utils::is_reserved_or_invalid_bucket;
|
||||
use crate::{
|
||||
disk::{
|
||||
self, VolumeInfo,
|
||||
disk_store::{CHECK_EVERY, CHECK_TIMEOUT_DURATION, DiskHealthTracker},
|
||||
disk_store::{DiskHealthTracker, get_drive_active_check_interval, get_drive_active_check_timeout},
|
||||
},
|
||||
endpoints::{EndpointServerPools, Node},
|
||||
store_api::{BucketInfo, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
|
||||
@@ -613,7 +615,7 @@ impl RemotePeerS3Client {
|
||||
|
||||
/// Monitor remote peer health periodically
|
||||
async fn monitor_remote_peer_health(addr: String, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
|
||||
let mut interval = time::interval(CHECK_EVERY);
|
||||
let mut interval = time::interval(get_drive_active_check_interval());
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
@@ -682,7 +684,7 @@ impl RemotePeerS3Client {
|
||||
let port = url.port_or_known_default().unwrap_or(80);
|
||||
|
||||
// Try to establish TCP connection
|
||||
match timeout(CHECK_TIMEOUT_DURATION, TcpStream::connect((host, port))).await {
|
||||
match timeout(get_drive_active_check_timeout(), TcpStream::connect((host, port))).await {
|
||||
Ok(Ok(_)) => Ok(()),
|
||||
_ => Err(Error::other(format!("Cannot connect to {host}:{port}"))),
|
||||
}
|
||||
@@ -717,16 +719,39 @@ impl RemotePeerS3Client {
|
||||
self.health.log_success();
|
||||
}
|
||||
self.health.decrement_waiting();
|
||||
if let Err(err) = &operation_result
|
||||
&& is_network_like_disk_error(err)
|
||||
{
|
||||
self.mark_faulty_and_start_recovery("operation_network_error").await;
|
||||
}
|
||||
operation_result
|
||||
}
|
||||
Err(_) => {
|
||||
// Timeout occurred, mark peer as potentially faulty
|
||||
self.health.decrement_waiting();
|
||||
self.mark_faulty_and_start_recovery("operation_timeout").await;
|
||||
warn!("Remote peer operation timeout after {:?}", timeout_duration);
|
||||
Err(Error::other(format!("Remote peer operation timeout after {timeout_duration:?}")))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn mark_faulty_and_start_recovery(&self, reason: &'static str) {
|
||||
if self.health.swap_ok_to_faulty() {
|
||||
warn!(
|
||||
addr = %self.addr,
|
||||
reason,
|
||||
"Remote peer marked faulty after network failure"
|
||||
);
|
||||
|
||||
let health = Arc::clone(&self.health);
|
||||
let cancel_token = self.cancel_token.clone();
|
||||
let addr = self.addr.clone();
|
||||
tokio::spawn(async move {
|
||||
Self::monitor_remote_peer_recovery(addr, health, cancel_token).await;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -1001,3 +1026,69 @@ pub async fn heal_bucket_local(bucket: &str, opts: &HealOpts) -> Result<HealResu
|
||||
async fn clone_drives() -> Vec<Option<DiskStore>> {
|
||||
GLOBAL_LOCAL_DISK_MAP.read().await.values().cloned().collect::<Vec<_>>()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn test_remote_peer(addr: &str) -> RemotePeerS3Client {
|
||||
let node = Node {
|
||||
url: url::Url::parse(addr).expect("test peer URL should parse"),
|
||||
pools: vec![0],
|
||||
is_local: false,
|
||||
grid_host: addr.to_string(),
|
||||
};
|
||||
|
||||
RemotePeerS3Client {
|
||||
node: Some(node),
|
||||
pools: Some(vec![0]),
|
||||
addr: addr.to_string(),
|
||||
health: Arc::new(DiskHealthTracker::new()),
|
||||
cancel_token: CancellationToken::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_with_timeout_marks_remote_peer_faulty_on_network_like_error() {
|
||||
let client = test_remote_peer("http://peer-network-error:9000");
|
||||
|
||||
let err = client
|
||||
.execute_with_timeout(
|
||||
|| async {
|
||||
Err::<(), Error>(DiskError::Io(std::io::Error::new(
|
||||
std::io::ErrorKind::ConnectionRefused,
|
||||
"connection refused",
|
||||
)))
|
||||
},
|
||||
Duration::from_secs(1),
|
||||
)
|
||||
.await
|
||||
.expect_err("network-like error should fail");
|
||||
|
||||
assert_eq!(
|
||||
match &err {
|
||||
DiskError::Io(io_err) => io_err.kind(),
|
||||
other => panic!("expected io network error, got {other:?}"),
|
||||
},
|
||||
std::io::ErrorKind::ConnectionRefused
|
||||
);
|
||||
assert!(client.health.is_faulty(), "network-like errors should mark remote peer faulty");
|
||||
|
||||
client.cancel_token.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_with_timeout_keeps_remote_peer_online_for_business_error() {
|
||||
let client = test_remote_peer("http://peer-business-error:9000");
|
||||
|
||||
let err = client
|
||||
.execute_with_timeout(|| async { Err::<(), Error>(DiskError::FileNotFound) }, Duration::from_secs(1))
|
||||
.await
|
||||
.expect_err("business error should fail");
|
||||
|
||||
assert_eq!(err, DiskError::FileNotFound);
|
||||
assert!(!client.health.is_faulty(), "business errors should not mark remote peer faulty");
|
||||
|
||||
client.cancel_token.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,15 +16,17 @@ use crate::disk::{
|
||||
CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, FileInfoVersions, FileReader,
|
||||
FileWriter, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
|
||||
disk_store::{
|
||||
CHECK_EVERY, CHECK_TIMEOUT_DURATION, DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING,
|
||||
SKIP_IF_SUCCESS_BEFORE, get_drive_disk_info_timeout, get_drive_list_dir_timeout, get_drive_metadata_timeout,
|
||||
get_drive_walkdir_stall_timeout, get_drive_walkdir_timeout, get_max_timeout_duration,
|
||||
DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, SKIP_IF_SUCCESS_BEFORE,
|
||||
get_drive_active_check_interval, get_drive_active_check_timeout, get_drive_disk_info_timeout, get_drive_list_dir_timeout,
|
||||
get_drive_metadata_timeout, get_drive_walkdir_stall_timeout, get_drive_walkdir_timeout, get_max_timeout_duration,
|
||||
},
|
||||
endpoint::Endpoint,
|
||||
health_state::{RuntimeDriveHealthState, get_drive_returning_probe_interval, record_drive_runtime_state},
|
||||
};
|
||||
use crate::disk::{disk_store::DiskHealthTracker, error::DiskError, local::ScanGuard};
|
||||
use crate::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
|
||||
use crate::rpc::client::{
|
||||
TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
|
||||
};
|
||||
use crate::set_disk::DEFAULT_READ_BUFFER_SIZE;
|
||||
use crate::{
|
||||
disk::error::{Error, Result},
|
||||
@@ -47,7 +49,7 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
use rustfs_rio::{HttpReader, HttpWriter};
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
use std::{
|
||||
io::{Cursor, ErrorKind},
|
||||
io::Cursor,
|
||||
path::PathBuf,
|
||||
sync::{
|
||||
Arc,
|
||||
@@ -176,7 +178,7 @@ impl RemoteDisk {
|
||||
health: Arc<DiskHealthTracker>,
|
||||
cancel_token: CancellationToken,
|
||||
) {
|
||||
let mut interval = time::interval(CHECK_EVERY);
|
||||
let mut interval = time::interval(get_drive_active_check_interval());
|
||||
|
||||
// Perform basic connectivity check
|
||||
if Self::perform_connectivity_check(&addr).await.is_err() && health.mark_offline(&endpoint, "connectivity_probe_failed") {
|
||||
@@ -281,7 +283,7 @@ impl RemoteDisk {
|
||||
let port = url.port_or_known_default().unwrap_or(80);
|
||||
|
||||
// Try to establish TCP connection
|
||||
match timeout(CHECK_TIMEOUT_DURATION, TcpStream::connect((host, port))).await {
|
||||
match timeout(get_drive_active_check_timeout(), TcpStream::connect((host, port))).await {
|
||||
Ok(Ok(stream)) => {
|
||||
drop(stream);
|
||||
Ok(())
|
||||
@@ -334,10 +336,10 @@ impl RemoteDisk {
|
||||
}
|
||||
self.health.decrement_waiting();
|
||||
if let Err(err) = &operation_result
|
||||
&& Self::is_timeout_like_error(err)
|
||||
&& is_network_like_disk_error(err)
|
||||
{
|
||||
counter!(
|
||||
"rustfs_drive_op_timeout_total",
|
||||
"rustfs_drive_op_network_error_total",
|
||||
"endpoint" => self.endpoint.to_string(),
|
||||
"op" => op.to_string()
|
||||
)
|
||||
@@ -347,9 +349,9 @@ impl RemoteDisk {
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = timeout_duration.as_millis(),
|
||||
"Remote disk operation returned a timeout-like error"
|
||||
"Remote disk operation returned a network-like error"
|
||||
);
|
||||
self.mark_faulty_and_evict("operation_timeout_error").await;
|
||||
self.mark_faulty_and_evict("operation_network_error").await;
|
||||
}
|
||||
operation_result
|
||||
}
|
||||
@@ -375,10 +377,6 @@ impl RemoteDisk {
|
||||
}
|
||||
}
|
||||
|
||||
fn is_timeout_like_error(err: &Error) -> bool {
|
||||
matches!(err, DiskError::Timeout) || matches!(err, DiskError::Io(io_err) if io_err.kind() == ErrorKind::TimedOut)
|
||||
}
|
||||
|
||||
async fn mark_faulty_and_evict(&self, reason: &'static str) {
|
||||
if self.health.mark_offline(&self.endpoint, reason) {
|
||||
self.spawn_recovery_monitor_if_needed();
|
||||
@@ -2073,6 +2071,96 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_with_timeout_marks_faulty_on_network_like_error() {
|
||||
let addr = "http://127.0.0.1:59993".to_string();
|
||||
let url = url::Url::parse(&format!("{addr}/data")).unwrap();
|
||||
let endpoint = Endpoint {
|
||||
url,
|
||||
is_local: false,
|
||||
pool_idx: 0,
|
||||
set_idx: 0,
|
||||
disk_idx: 0,
|
||||
};
|
||||
|
||||
let remote_disk = RemoteDisk::new(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let channel = TonicEndpoint::from_shared(addr.clone()).unwrap().connect_lazy();
|
||||
GLOBAL_CONN_MAP.write().await.insert(addr.clone(), channel);
|
||||
|
||||
let err = remote_disk
|
||||
.execute_with_timeout(
|
||||
|| async {
|
||||
Err::<(), Error>(DiskError::Io(std::io::Error::new(
|
||||
std::io::ErrorKind::ConnectionRefused,
|
||||
"connection refused",
|
||||
)))
|
||||
},
|
||||
Duration::from_secs(1),
|
||||
)
|
||||
.await
|
||||
.expect_err("network-like operation error should fail");
|
||||
|
||||
assert_eq!(
|
||||
match &err {
|
||||
DiskError::Io(io_err) => io_err.kind(),
|
||||
other => panic!("expected io network error, got {other:?}"),
|
||||
},
|
||||
std::io::ErrorKind::ConnectionRefused
|
||||
);
|
||||
assert!(!remote_disk.is_online().await, "network-like errors should mark remote disk faulty");
|
||||
assert!(
|
||||
!GLOBAL_CONN_MAP.read().await.contains_key(&addr),
|
||||
"network-like errors should evict cached connection"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_with_timeout_keeps_remote_disk_online_for_business_error() {
|
||||
let addr = "http://127.0.0.1:59994".to_string();
|
||||
let url = url::Url::parse(&format!("{addr}/data")).unwrap();
|
||||
let endpoint = Endpoint {
|
||||
url,
|
||||
is_local: false,
|
||||
pool_idx: 0,
|
||||
set_idx: 0,
|
||||
disk_idx: 0,
|
||||
};
|
||||
|
||||
let remote_disk = RemoteDisk::new(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let channel = TonicEndpoint::from_shared(addr.clone()).unwrap().connect_lazy();
|
||||
GLOBAL_CONN_MAP.write().await.insert(addr.clone(), channel);
|
||||
|
||||
let err = remote_disk
|
||||
.execute_with_timeout(|| async { Err::<(), Error>(DiskError::FileNotFound) }, Duration::from_secs(1))
|
||||
.await
|
||||
.expect_err("business error should still fail the operation");
|
||||
|
||||
assert_eq!(err, DiskError::FileNotFound);
|
||||
assert!(remote_disk.is_online().await, "business errors should not mark remote disk faulty");
|
||||
assert!(
|
||||
GLOBAL_CONN_MAP.read().await.contains_key(&addr),
|
||||
"business errors should not evict cached connection"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_remote_disk_sync_properties() {
|
||||
let url = url::Url::parse("https://secure-remote:9000/data").unwrap();
|
||||
|
||||
@@ -13,6 +13,87 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use std::future::Future;
|
||||
use std::time::Duration;
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
async fn collect_list_parts_results<F>(
|
||||
tasks: Vec<F>,
|
||||
read_quorum: usize,
|
||||
) -> disk::error::Result<(Vec<Option<DiskError>>, Vec<Vec<String>>)>
|
||||
where
|
||||
F: Future<Output = disk::error::Result<Vec<String>>> + Send + 'static,
|
||||
{
|
||||
let mut errs = vec![Some(DiskError::DiskNotFound); tasks.len()];
|
||||
let mut object_parts = vec![Vec::new(); tasks.len()];
|
||||
let mut successful_responses = 0usize;
|
||||
let mut pending = tasks.len();
|
||||
let mut join_set = JoinSet::new();
|
||||
|
||||
for (index, task) in tasks.into_iter().enumerate() {
|
||||
join_set.spawn(async move { (index, task.await) });
|
||||
}
|
||||
|
||||
while let Some(join_result) = join_set.join_next().await {
|
||||
pending = pending.saturating_sub(1);
|
||||
|
||||
match join_result {
|
||||
Ok((index, Ok(parts))) => {
|
||||
errs[index] = None;
|
||||
object_parts[index] = parts;
|
||||
successful_responses += 1;
|
||||
}
|
||||
Ok((index, Err(err))) => {
|
||||
errs[index] = Some(err);
|
||||
}
|
||||
Err(_) => {}
|
||||
}
|
||||
|
||||
if successful_responses + pending < read_quorum {
|
||||
return Err(DiskError::ErasureReadQuorum);
|
||||
}
|
||||
}
|
||||
|
||||
Ok((errs, object_parts))
|
||||
}
|
||||
|
||||
fn reduce_quorum_part_numbers(object_parts: Vec<Vec<String>>, read_quorum: usize) -> Vec<usize> {
|
||||
let mut part_quorum_map: HashMap<usize, usize> = HashMap::new();
|
||||
|
||||
for drive_parts in object_parts {
|
||||
let mut parts_with_meta_count: HashMap<usize, usize> = HashMap::new();
|
||||
|
||||
// part files can be either part.N or part.N.meta
|
||||
for part_path in drive_parts {
|
||||
if let Some(num_str) = part_path.strip_prefix("part.") {
|
||||
if let Some(meta_idx) = num_str.find(".meta") {
|
||||
if let Ok(part_num) = num_str[..meta_idx].parse::<usize>() {
|
||||
*parts_with_meta_count.entry(part_num).or_insert(0) += 1;
|
||||
}
|
||||
} else if let Ok(part_num) = num_str.parse::<usize>() {
|
||||
*parts_with_meta_count.entry(part_num).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Include only part.N.meta files with corresponding part.N
|
||||
for (&part_num, &cnt) in &parts_with_meta_count {
|
||||
if cnt >= 2 {
|
||||
*part_quorum_map.entry(part_num).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut part_numbers = Vec::with_capacity(part_quorum_map.len());
|
||||
for (part_num, count) in part_quorum_map {
|
||||
if count >= read_quorum {
|
||||
part_numbers.push(part_num);
|
||||
}
|
||||
}
|
||||
|
||||
part_numbers.sort();
|
||||
part_numbers
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
pub(super) async fn list_parts(
|
||||
@@ -21,10 +102,13 @@ impl SetDisks {
|
||||
read_quorum: usize,
|
||||
) -> disk::error::Result<Vec<usize>> {
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
for (i, disk) in disks.iter().enumerate() {
|
||||
let part_path = part_path.to_string();
|
||||
for disk in disks.iter() {
|
||||
let disk = disk.clone();
|
||||
let part_path = part_path.clone();
|
||||
futures.push(async move {
|
||||
if let Some(disk) = disk {
|
||||
disk.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, part_path, -1)
|
||||
disk.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, part_path.as_str(), -1)
|
||||
.await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
@@ -35,60 +119,15 @@ impl SetDisks {
|
||||
let mut errs = Vec::with_capacity(disks.len());
|
||||
let mut object_parts = Vec::with_capacity(disks.len());
|
||||
|
||||
let results = join_all(futures).await;
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(res) => {
|
||||
errs.push(None);
|
||||
object_parts.push(res);
|
||||
}
|
||||
Err(e) => {
|
||||
errs.push(Some(e));
|
||||
object_parts.push(vec![]);
|
||||
}
|
||||
}
|
||||
}
|
||||
let (collected_errs, collected_parts) = collect_list_parts_results(futures, read_quorum).await?;
|
||||
errs.extend(collected_errs);
|
||||
object_parts.extend(collected_parts);
|
||||
|
||||
if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let mut part_quorum_map: HashMap<usize, usize> = HashMap::new();
|
||||
|
||||
for drive_parts in object_parts {
|
||||
let mut parts_with_meta_count: HashMap<usize, usize> = HashMap::new();
|
||||
|
||||
// part files can be either part.N or part.N.meta
|
||||
for part_path in drive_parts {
|
||||
if let Some(num_str) = part_path.strip_prefix("part.") {
|
||||
if let Some(meta_idx) = num_str.find(".meta") {
|
||||
if let Ok(part_num) = num_str[..meta_idx].parse::<usize>() {
|
||||
*parts_with_meta_count.entry(part_num).or_insert(0) += 1;
|
||||
}
|
||||
} else if let Ok(part_num) = num_str.parse::<usize>() {
|
||||
*parts_with_meta_count.entry(part_num).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Include only part.N.meta files with corresponding part.N
|
||||
for (&part_num, &cnt) in &parts_with_meta_count {
|
||||
if cnt >= 2 {
|
||||
*part_quorum_map.entry(part_num).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut part_numbers = Vec::with_capacity(part_quorum_map.len());
|
||||
for (part_num, count) in part_quorum_map {
|
||||
if count >= read_quorum {
|
||||
part_numbers.push(part_num);
|
||||
}
|
||||
}
|
||||
|
||||
part_numbers.sort();
|
||||
|
||||
Ok(part_numbers)
|
||||
Ok(reduce_quorum_part_numbers(object_parts, read_quorum))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
@@ -145,3 +184,78 @@ impl SetDisks {
|
||||
Ok((fi, parts_metadata))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_list_parts_results_fails_early_when_quorum_is_impossible() {
|
||||
let started = std::time::Instant::now();
|
||||
let tasks: Vec<_> = vec![
|
||||
(10_u64, Err(DiskError::DiskNotFound)),
|
||||
(15, Err(DiskError::DiskNotFound)),
|
||||
(250, Ok::<Vec<String>, DiskError>(vec!["part.1".to_string(), "part.1.meta".to_string()])),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(delay_ms, outcome)| async move {
|
||||
tokio::time::sleep(Duration::from_millis(delay_ms)).await;
|
||||
outcome
|
||||
})
|
||||
.collect();
|
||||
|
||||
let err = collect_list_parts_results(tasks, 2)
|
||||
.await
|
||||
.expect_err("quorum should become impossible before slow tail completes");
|
||||
|
||||
assert_eq!(err, DiskError::ErasureReadQuorum);
|
||||
assert!(started.elapsed() < Duration::from_millis(120));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_list_parts_results_tolerates_single_panicked_task_when_quorum_is_met() {
|
||||
let tasks: Vec<_> = vec![(5_u64, true), (10, false), (12, false)]
|
||||
.into_iter()
|
||||
.map(|(delay_ms, should_panic)| async move {
|
||||
tokio::time::sleep(Duration::from_millis(delay_ms)).await;
|
||||
if should_panic {
|
||||
panic!("simulated task panic");
|
||||
}
|
||||
Ok::<Vec<String>, DiskError>(vec!["part.1".to_string(), "part.1.meta".to_string()])
|
||||
})
|
||||
.collect();
|
||||
|
||||
let (errs, object_parts) = collect_list_parts_results(tasks, 2)
|
||||
.await
|
||||
.expect("quorum should still succeed");
|
||||
assert_eq!(errs.iter().filter(|err| err.is_none()).count(), 2);
|
||||
assert_eq!(object_parts.iter().filter(|parts| !parts.is_empty()).count(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reduce_quorum_part_numbers_only_keeps_parts_present_on_quorum_of_drives() {
|
||||
let object_parts = vec![
|
||||
vec![
|
||||
"part.1".to_string(),
|
||||
"part.1.meta".to_string(),
|
||||
"part.2".to_string(),
|
||||
"part.2.meta".to_string(),
|
||||
],
|
||||
vec![
|
||||
"part.1".to_string(),
|
||||
"part.1.meta".to_string(),
|
||||
"part.3".to_string(),
|
||||
"part.3.meta".to_string(),
|
||||
],
|
||||
vec![
|
||||
"part.1".to_string(),
|
||||
"part.1.meta".to_string(),
|
||||
"part.2".to_string(),
|
||||
"part.2.meta".to_string(),
|
||||
],
|
||||
];
|
||||
|
||||
let parts = reduce_quorum_part_numbers(object_parts, 2);
|
||||
assert_eq!(parts, vec![1, 2]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,88 @@
|
||||
|
||||
use super::*;
|
||||
use rustfs_config::{DEFAULT_OBJECT_ZERO_COPY_ENABLE, ENV_OBJECT_ZERO_COPY_ENABLE};
|
||||
use std::future::Future;
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
async fn collect_read_multiple_results<F>(
|
||||
tasks: Vec<F>,
|
||||
read_quorum: usize,
|
||||
) -> std::result::Result<(Vec<Option<Vec<ReadMultipleResp>>>, Vec<Option<DiskError>>), ()>
|
||||
where
|
||||
F: Future<Output = disk::error::Result<Vec<ReadMultipleResp>>> + Send + 'static,
|
||||
{
|
||||
let mut responses = vec![None; tasks.len()];
|
||||
let mut errors = vec![Some(DiskError::DiskNotFound); tasks.len()];
|
||||
let mut successful_responses = 0usize;
|
||||
let mut pending = tasks.len();
|
||||
let mut join_set = JoinSet::new();
|
||||
|
||||
for (index, task) in tasks.into_iter().enumerate() {
|
||||
join_set.spawn(async move { (index, task.await) });
|
||||
}
|
||||
|
||||
while let Some(join_result) = join_set.join_next().await {
|
||||
pending = pending.saturating_sub(1);
|
||||
|
||||
match join_result {
|
||||
Ok((index, Ok(resp))) => {
|
||||
responses[index] = Some(resp);
|
||||
errors[index] = None;
|
||||
successful_responses += 1;
|
||||
}
|
||||
Ok((index, Err(err))) => {
|
||||
errors[index] = Some(err);
|
||||
}
|
||||
Err(_) => {}
|
||||
}
|
||||
|
||||
if successful_responses + pending < read_quorum {
|
||||
return Err(());
|
||||
}
|
||||
}
|
||||
|
||||
Ok((responses, errors))
|
||||
}
|
||||
|
||||
async fn collect_read_parts_results<F>(
|
||||
tasks: Vec<F>,
|
||||
read_quorum: usize,
|
||||
) -> std::result::Result<(Vec<Option<Vec<ObjectPartInfo>>>, Vec<Option<DiskError>>), ()>
|
||||
where
|
||||
F: Future<Output = disk::error::Result<Vec<ObjectPartInfo>>> + Send + 'static,
|
||||
{
|
||||
let mut responses = vec![None; tasks.len()];
|
||||
let mut errors = vec![Some(DiskError::DiskNotFound); tasks.len()];
|
||||
let mut successful_responses = 0usize;
|
||||
let mut pending = tasks.len();
|
||||
let mut join_set = JoinSet::new();
|
||||
|
||||
for (index, task) in tasks.into_iter().enumerate() {
|
||||
join_set.spawn(async move { (index, task.await) });
|
||||
}
|
||||
|
||||
while let Some(join_result) = join_set.join_next().await {
|
||||
pending = pending.saturating_sub(1);
|
||||
|
||||
match join_result {
|
||||
Ok((index, Ok(resp))) => {
|
||||
responses[index] = Some(resp);
|
||||
errors[index] = None;
|
||||
successful_responses += 1;
|
||||
}
|
||||
Ok((index, Err(err))) => {
|
||||
errors[index] = Some(err);
|
||||
}
|
||||
Err(_) => {}
|
||||
}
|
||||
|
||||
if successful_responses + pending < read_quorum {
|
||||
return Err(());
|
||||
}
|
||||
}
|
||||
|
||||
Ok((responses, errors))
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
pub(super) async fn read_parts(
|
||||
@@ -25,9 +107,6 @@ impl SetDisks {
|
||||
) -> disk::error::Result<Vec<ObjectPartInfo>> {
|
||||
let mut errs = Vec::with_capacity(disks.len());
|
||||
let mut object_parts = Vec::with_capacity(disks.len());
|
||||
|
||||
// Use batch processor for better performance
|
||||
let processor = get_global_processors().read_processor();
|
||||
let bucket = bucket.to_string();
|
||||
let part_meta_paths = part_meta_paths.to_vec();
|
||||
|
||||
@@ -48,19 +127,13 @@ impl SetDisks {
|
||||
})
|
||||
.collect();
|
||||
|
||||
let results = processor.execute_batch(tasks).await;
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(res) => {
|
||||
errs.push(None);
|
||||
object_parts.push(res);
|
||||
}
|
||||
Err(e) => {
|
||||
errs.push(Some(e));
|
||||
object_parts.push(vec![]);
|
||||
}
|
||||
}
|
||||
}
|
||||
let (responses, collected_errors) = match collect_read_parts_results(tasks, read_quorum).await {
|
||||
Ok(collected) => collected,
|
||||
Err(()) => return Err(DiskError::ErasureReadQuorum),
|
||||
};
|
||||
|
||||
errs.extend(collected_errors);
|
||||
object_parts.extend(responses.into_iter().map(|resp| resp.unwrap_or_default()));
|
||||
|
||||
if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
|
||||
return Err(err);
|
||||
@@ -384,10 +457,23 @@ impl SetDisks {
|
||||
read_quorum: usize,
|
||||
) -> Vec<ReadMultipleResp> {
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
let mut ress = Vec::with_capacity(disks.len());
|
||||
let mut errors = Vec::with_capacity(disks.len());
|
||||
let empty_quorum_result = || {
|
||||
req.files
|
||||
.iter()
|
||||
.map(|want| ReadMultipleResp {
|
||||
bucket: req.bucket.clone(),
|
||||
prefix: req.prefix.clone(),
|
||||
file: want.clone(),
|
||||
exists: false,
|
||||
error: Error::ErasureReadQuorum.to_string(),
|
||||
data: Vec::new(),
|
||||
mod_time: None,
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
|
||||
for disk in disks.iter() {
|
||||
let disk = disk.clone();
|
||||
let req = req.clone();
|
||||
futures.push(async move {
|
||||
if let Some(disk) = disk {
|
||||
@@ -398,19 +484,10 @@ impl SetDisks {
|
||||
});
|
||||
}
|
||||
|
||||
let results = join_all(futures).await;
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(res) => {
|
||||
ress.push(Some(res));
|
||||
errors.push(None);
|
||||
}
|
||||
Err(e) => {
|
||||
ress.push(None);
|
||||
errors.push(Some(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
let (ress, errors) = match collect_read_multiple_results(futures, read_quorum).await {
|
||||
Ok(collected) => collected,
|
||||
Err(()) => return empty_quorum_result(),
|
||||
};
|
||||
|
||||
// debug!("ReadMultipleResp ress {:?}", ress);
|
||||
// debug!("ReadMultipleResp errors {:?}", errors);
|
||||
@@ -839,3 +916,181 @@ impl SetDisks {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_read_multiple_results_fails_early_when_quorum_is_impossible() {
|
||||
let started = std::time::Instant::now();
|
||||
let resp = ReadMultipleResp {
|
||||
bucket: "bucket".to_string(),
|
||||
prefix: "prefix".to_string(),
|
||||
file: "file".to_string(),
|
||||
exists: true,
|
||||
error: String::new(),
|
||||
data: vec![1],
|
||||
mod_time: None,
|
||||
};
|
||||
|
||||
let tasks: Vec<_> = vec![
|
||||
(10_u64, Err(DiskError::DiskNotFound)),
|
||||
(15, Err(DiskError::DiskNotFound)),
|
||||
(250, Ok::<Vec<ReadMultipleResp>, DiskError>(vec![resp])),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(delay_ms, outcome)| async move {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
|
||||
outcome
|
||||
})
|
||||
.collect();
|
||||
|
||||
let result = collect_read_multiple_results(tasks, 2).await;
|
||||
assert!(result.is_err(), "quorum should become impossible before slow tail completes");
|
||||
assert!(started.elapsed() < std::time::Duration::from_millis(120));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_read_multiple_results_returns_collected_responses_on_quorum() {
|
||||
let resp = ReadMultipleResp {
|
||||
bucket: "bucket".to_string(),
|
||||
prefix: "prefix".to_string(),
|
||||
file: "file".to_string(),
|
||||
exists: true,
|
||||
error: String::new(),
|
||||
data: vec![1, 2, 3],
|
||||
mod_time: None,
|
||||
};
|
||||
|
||||
let tasks: Vec<_> = vec![
|
||||
(10_u64, Ok::<Vec<ReadMultipleResp>, DiskError>(vec![resp.clone()])),
|
||||
(15, Ok::<Vec<ReadMultipleResp>, DiskError>(vec![resp.clone()])),
|
||||
(250, Err(DiskError::DiskNotFound)),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(delay_ms, outcome)| async move {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
|
||||
outcome
|
||||
})
|
||||
.collect();
|
||||
|
||||
let (responses, errors) = collect_read_multiple_results(tasks, 2).await.expect("quorum should succeed");
|
||||
|
||||
assert_eq!(responses.iter().filter(|item| item.is_some()).count(), 2);
|
||||
assert_eq!(errors.iter().filter(|item| item.is_none()).count(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_read_multiple_results_tolerates_single_panicked_task_when_quorum_is_met() {
|
||||
let resp = ReadMultipleResp {
|
||||
bucket: "bucket".to_string(),
|
||||
prefix: "prefix".to_string(),
|
||||
file: "file".to_string(),
|
||||
exists: true,
|
||||
error: String::new(),
|
||||
data: vec![1, 2, 3],
|
||||
mod_time: None,
|
||||
};
|
||||
|
||||
let tasks: Vec<_> = vec![(5_u64, true), (10, false), (12, false)]
|
||||
.into_iter()
|
||||
.map(|(delay_ms, should_panic)| {
|
||||
let resp = resp.clone();
|
||||
async move {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
|
||||
if should_panic {
|
||||
panic!("simulated task panic");
|
||||
}
|
||||
Ok::<Vec<ReadMultipleResp>, DiskError>(vec![resp])
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
let (responses, errors) = collect_read_multiple_results(tasks, 2)
|
||||
.await
|
||||
.expect("quorum should still succeed");
|
||||
assert_eq!(responses.iter().filter(|item| item.is_some()).count(), 2);
|
||||
assert_eq!(errors.iter().filter(|item| item.is_none()).count(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_read_parts_results_fails_early_when_quorum_is_impossible() {
|
||||
let started = std::time::Instant::now();
|
||||
let part = ObjectPartInfo {
|
||||
number: 1,
|
||||
etag: "etag".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let tasks: Vec<_> = vec![
|
||||
(10_u64, Err(DiskError::DiskNotFound)),
|
||||
(15, Err(DiskError::DiskNotFound)),
|
||||
(250, Ok::<Vec<ObjectPartInfo>, DiskError>(vec![part])),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(delay_ms, outcome)| async move {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
|
||||
outcome
|
||||
})
|
||||
.collect();
|
||||
|
||||
let result = collect_read_parts_results(tasks, 2).await;
|
||||
assert!(result.is_err(), "quorum should become impossible before slow tail completes");
|
||||
assert!(started.elapsed() < std::time::Duration::from_millis(120));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_read_parts_results_returns_collected_responses_on_quorum() {
|
||||
let part = ObjectPartInfo {
|
||||
number: 1,
|
||||
etag: "etag".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let tasks: Vec<_> = vec![
|
||||
(10_u64, Ok::<Vec<ObjectPartInfo>, DiskError>(vec![part.clone()])),
|
||||
(15, Ok::<Vec<ObjectPartInfo>, DiskError>(vec![part.clone()])),
|
||||
(250, Err(DiskError::DiskNotFound)),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(delay_ms, outcome)| async move {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
|
||||
outcome
|
||||
})
|
||||
.collect();
|
||||
|
||||
let (responses, errors) = collect_read_parts_results(tasks, 2).await.expect("quorum should succeed");
|
||||
assert_eq!(responses.iter().filter(|item| item.is_some()).count(), 2);
|
||||
assert_eq!(errors.iter().filter(|item| item.is_none()).count(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn collect_read_parts_results_tolerates_single_panicked_task_when_quorum_is_met() {
|
||||
let part = ObjectPartInfo {
|
||||
number: 1,
|
||||
etag: "etag".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let tasks: Vec<_> = vec![(5_u64, true), (10, false), (12, false)]
|
||||
.into_iter()
|
||||
.map(|(delay_ms, should_panic)| {
|
||||
let part = part.clone();
|
||||
async move {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
|
||||
if should_panic {
|
||||
panic!("simulated task panic");
|
||||
}
|
||||
Ok::<Vec<ObjectPartInfo>, DiskError>(vec![part])
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
let (responses, errors) = collect_read_parts_results(tasks, 2)
|
||||
.await
|
||||
.expect("quorum should still succeed");
|
||||
assert_eq!(responses.iter().filter(|item| item.is_some()).count(), 2);
|
||||
assert_eq!(errors.iter().filter(|item| item.is_none()).count(), 2);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,6 +34,8 @@ path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
rustfs-common.workspace = true
|
||||
rustfs-config.workspace = true
|
||||
rustfs-utils.workspace = true
|
||||
flatbuffers = { workspace = true }
|
||||
prost = { workspace = true }
|
||||
tonic = { workspace = true, features = ["transport"] }
|
||||
|
||||
+52
-25
@@ -40,51 +40,78 @@ pub use generated::*;
|
||||
// Default 100 MB
|
||||
pub const DEFAULT_GRPC_SERVER_MESSAGE_LEN: usize = 100 * 1024 * 1024;
|
||||
|
||||
/// Timeout for connection establishment - reduced for faster failure detection
|
||||
const CONNECT_TIMEOUT_SECS: u64 = 3;
|
||||
|
||||
/// TCP keepalive interval - how often to probe the connection
|
||||
const TCP_KEEPALIVE_SECS: u64 = 10;
|
||||
|
||||
/// HTTP/2 keepalive interval - application-layer heartbeat
|
||||
const HTTP2_KEEPALIVE_INTERVAL_SECS: u64 = 5;
|
||||
|
||||
/// HTTP/2 keepalive timeout - how long to wait for PING ACK
|
||||
const HTTP2_KEEPALIVE_TIMEOUT_SECS: u64 = 3;
|
||||
|
||||
/// Overall RPC timeout - maximum time for any single RPC operation
|
||||
const RPC_TIMEOUT_SECS: u64 = 30;
|
||||
|
||||
/// Default HTTPS prefix for rustfs
|
||||
/// This is the default HTTPS prefix for rustfs.
|
||||
/// It is used to identify HTTPS URLs.
|
||||
/// Default value: https://
|
||||
const RUSTFS_HTTPS_PREFIX: &str = "https://";
|
||||
|
||||
fn internode_connect_timeout() -> Duration {
|
||||
Duration::from_secs(rustfs_utils::get_env_u64(
|
||||
rustfs_config::ENV_INTERNODE_CONNECT_TIMEOUT_SECS,
|
||||
rustfs_config::DEFAULT_INTERNODE_CONNECT_TIMEOUT_SECS,
|
||||
))
|
||||
}
|
||||
|
||||
fn internode_tcp_keepalive() -> Duration {
|
||||
Duration::from_secs(rustfs_utils::get_env_u64(
|
||||
rustfs_config::ENV_INTERNODE_TCP_KEEPALIVE_SECS,
|
||||
rustfs_config::DEFAULT_INTERNODE_TCP_KEEPALIVE_SECS,
|
||||
))
|
||||
}
|
||||
|
||||
fn internode_http2_keep_alive_interval() -> Duration {
|
||||
Duration::from_secs(rustfs_utils::get_env_u64(
|
||||
rustfs_config::ENV_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS,
|
||||
rustfs_config::DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS,
|
||||
))
|
||||
}
|
||||
|
||||
fn internode_http2_keep_alive_timeout() -> Duration {
|
||||
Duration::from_secs(rustfs_utils::get_env_u64(
|
||||
rustfs_config::ENV_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS,
|
||||
rustfs_config::DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS,
|
||||
))
|
||||
}
|
||||
|
||||
fn internode_rpc_timeout() -> Duration {
|
||||
Duration::from_secs(rustfs_utils::get_env_u64(
|
||||
rustfs_config::ENV_INTERNODE_RPC_TIMEOUT_SECS,
|
||||
rustfs_config::DEFAULT_INTERNODE_RPC_TIMEOUT_SECS,
|
||||
))
|
||||
}
|
||||
|
||||
/// Creates a new gRPC channel with optimized keepalive settings for cluster resilience.
|
||||
///
|
||||
/// This function is designed to detect dead peers quickly:
|
||||
/// - Fast connection timeout (3s instead of default 30s+)
|
||||
/// - Aggressive TCP keepalive (10s)
|
||||
/// - HTTP/2 PING every 5s, timeout at 3s
|
||||
/// - Overall RPC timeout of 30s (reduced from 60s)
|
||||
/// This function is designed to detect dead peers quickly using env-configurable
|
||||
/// internode transport settings. Defaults come from `rustfs_config` constants:
|
||||
/// - Connect timeout: `DEFAULT_INTERNODE_CONNECT_TIMEOUT_SECS` (3s)
|
||||
/// - TCP keepalive: `DEFAULT_INTERNODE_TCP_KEEPALIVE_SECS` (10s)
|
||||
/// - HTTP/2 keepalive interval: `DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS` (5s)
|
||||
/// - HTTP/2 keepalive timeout: `DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS` (3s)
|
||||
/// - RPC timeout: `DEFAULT_INTERNODE_RPC_TIMEOUT_SECS` (10s)
|
||||
pub async fn create_new_channel(addr: &str) -> Result<Channel, Box<dyn Error>> {
|
||||
debug!("Creating new gRPC channel to: {}", addr);
|
||||
let dial_started_at = Instant::now();
|
||||
let connect_timeout = internode_connect_timeout();
|
||||
let tcp_keepalive = internode_tcp_keepalive();
|
||||
let http2_keepalive_interval = internode_http2_keep_alive_interval();
|
||||
let http2_keepalive_timeout = internode_http2_keep_alive_timeout();
|
||||
let rpc_timeout = internode_rpc_timeout();
|
||||
|
||||
let mut connector = Endpoint::from_shared(addr.to_string())?
|
||||
// Fast connection timeout for dead peer detection
|
||||
.connect_timeout(Duration::from_secs(CONNECT_TIMEOUT_SECS))
|
||||
.connect_timeout(connect_timeout)
|
||||
// TCP-level keepalive - OS will probe connection
|
||||
.tcp_keepalive(Some(Duration::from_secs(TCP_KEEPALIVE_SECS)))
|
||||
.tcp_keepalive(Some(tcp_keepalive))
|
||||
// HTTP/2 PING frames for application-layer health check
|
||||
.http2_keep_alive_interval(Duration::from_secs(HTTP2_KEEPALIVE_INTERVAL_SECS))
|
||||
.http2_keep_alive_interval(http2_keepalive_interval)
|
||||
// How long to wait for PING ACK before considering connection dead
|
||||
.keep_alive_timeout(Duration::from_secs(HTTP2_KEEPALIVE_TIMEOUT_SECS))
|
||||
.keep_alive_timeout(http2_keepalive_timeout)
|
||||
// Send PINGs even when no active streams (critical for idle connections)
|
||||
.keep_alive_while_idle(true)
|
||||
// Overall timeout for any RPC - fail fast on unresponsive peers
|
||||
.timeout(Duration::from_secs(RPC_TIMEOUT_SECS));
|
||||
.timeout(rpc_timeout);
|
||||
|
||||
let root_cert = GLOBAL_ROOT_CERT.read().await;
|
||||
if addr.starts_with(RUSTFS_HTTPS_PREFIX) {
|
||||
|
||||
@@ -35,6 +35,9 @@ services:
|
||||
- RUSTFS_SECRET_KEY=rustfsadmin # CHANGEME
|
||||
- RUSTFS_OBS_LOGGER_LEVEL=info
|
||||
- RUSTFS_TLS_PATH=/opt/tls
|
||||
# Keep strict disk topology checks enabled by default.
|
||||
# For local testing only, set `RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true` explicitly.
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-false}
|
||||
|
||||
volumes:
|
||||
- rustfs_data_0:/data/rustfs0
|
||||
|
||||
@@ -1444,6 +1444,7 @@ impl DefaultBucketUsecase {
|
||||
return Err(s3_error!(InvalidArgument, "{err}"));
|
||||
}
|
||||
|
||||
input_cfg.expiry_updated_at = Some(Timestamp::from(time::OffsetDateTime::now_utc()));
|
||||
let data = serialize_config(&input_cfg)?;
|
||||
metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, data)
|
||||
.await
|
||||
|
||||
+766
@@ -0,0 +1,766 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
CLUSTER_COMPOSE="${CLUSTER_COMPOSE:-${PROJECT_ROOT}/.docker/compose/docker-compose.cluster.local-build.yml}"
|
||||
OBS_COMPOSE="${OBS_COMPOSE:-${PROJECT_ROOT}/.docker/observability/docker-compose.yml}"
|
||||
PROJECT_NAME="${PROJECT_NAME:-rustfs-four-node-test}"
|
||||
IMAGE_TAG="${IMAGE_TAG:-rustfs/rustfs:local-4node}"
|
||||
WITH_OBSERVABILITY="${WITH_OBSERVABILITY:-true}"
|
||||
BUILD_LOCAL_IMAGE="${BUILD_LOCAL_IMAGE:-true}"
|
||||
RUN_FAILOVER="${RUN_FAILOVER:-true}"
|
||||
RUN_BENCHMARK="${RUN_BENCHMARK:-true}"
|
||||
KEEP_UP="${KEEP_UP:-false}"
|
||||
PRECHECK_AUTO_CLEANUP="${PRECHECK_AUTO_CLEANUP:-true}"
|
||||
WAIT_PROBE_MODE="${WAIT_PROBE_MODE:-service}"
|
||||
|
||||
RUSTFS_ACCESS_KEY="${RUSTFS_ACCESS_KEY:-rustfsadmin}"
|
||||
RUSTFS_SECRET_KEY="${RUSTFS_SECRET_KEY:-rustfsadmin}"
|
||||
RUSTFS_OBS_ENDPOINT="${RUSTFS_OBS_ENDPOINT:-}"
|
||||
RUSTFS_UNSAFE_BYPASS_DISK_CHECK="${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}"
|
||||
|
||||
WAIT_TIMEOUT_SECS="${WAIT_TIMEOUT_SECS:-180}"
|
||||
BENCH_READY_TIMEOUT_SECS="${BENCH_READY_TIMEOUT_SECS:-180}"
|
||||
FAILOVER_NODE="${FAILOVER_NODE:-node4}"
|
||||
FAILOVER_WARMUP_SECS="${FAILOVER_WARMUP_SECS:-5}"
|
||||
FAILOVER_SAMPLE_SECS="${FAILOVER_SAMPLE_SECS:-60}"
|
||||
FAILOVER_INTERVAL_SECS="${FAILOVER_INTERVAL_SECS:-1}"
|
||||
BENCH_WAIT_MODE="${BENCH_WAIT_MODE:-ready}"
|
||||
|
||||
BENCH_ENDPOINT="${BENCH_ENDPOINT:-http://127.0.0.1:9000}"
|
||||
BENCH_BUCKET="${BENCH_BUCKET:-rustfs-four-node-bench}"
|
||||
BENCH_CONCURRENCY="${BENCH_CONCURRENCY:-}"
|
||||
BENCH_CONCURRENCIES="${BENCH_CONCURRENCIES:-}"
|
||||
BENCH_DURATION="${BENCH_DURATION:-60s}"
|
||||
BENCH_SIZES="${BENCH_SIZES:-1KiB,4KiB,11Mi}"
|
||||
|
||||
OUT_DIR="${OUT_DIR:-${PROJECT_ROOT}/target/bench/four-node-failover-$(date +%Y%m%d-%H%M%S)}"
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
scripts/run_four_node_cluster_failover_bench.sh [options]
|
||||
|
||||
Options:
|
||||
--cluster-compose <path> 4-node compose file
|
||||
--obs-compose <path> observability compose file
|
||||
--project-name <name> docker compose project name
|
||||
--image-tag <tag> image tag to build/use
|
||||
--with-observability bring up .docker/observability stack together
|
||||
--without-observability only bring up 4-node cluster
|
||||
--skip-build skip docker build from Dockerfile.source
|
||||
--skip-failover skip failover recovery validation
|
||||
--skip-bench skip benchmark phase
|
||||
--failover-node <nodeN> node to stop during failover test (default: node4)
|
||||
--obs-endpoint <url> RUSTFS_OBS_ENDPOINT (default: auto-select by mode)
|
||||
--bench-endpoint <url> benchmark endpoint (default: http://127.0.0.1:9000)
|
||||
--bench-sizes <sizes> comma list (default: 1KiB,4KiB,11Mi)
|
||||
--bench-concurrency <n> benchmark concurrency
|
||||
--bench-concurrencies <list> benchmark concurrency list (default: 8,16,32,64,128)
|
||||
--bench-duration <dur> benchmark duration
|
||||
--out-dir <path> output directory
|
||||
--keep-up keep compose services running after script exits
|
||||
-h, --help show help
|
||||
|
||||
Environment:
|
||||
CLUSTER_COMPOSE OBS_COMPOSE PROJECT_NAME IMAGE_TAG
|
||||
WITH_OBSERVABILITY BUILD_LOCAL_IMAGE RUN_FAILOVER RUN_BENCHMARK KEEP_UP
|
||||
RUSTFS_ACCESS_KEY RUSTFS_SECRET_KEY RUSTFS_OBS_ENDPOINT
|
||||
PRECHECK_AUTO_CLEANUP (true|false, default: true)
|
||||
WAIT_PROBE_MODE (service|ready, default: service)
|
||||
WAIT_TIMEOUT_SECS FAILOVER_NODE FAILOVER_WARMUP_SECS FAILOVER_SAMPLE_SECS
|
||||
FAILOVER_INTERVAL_SECS BENCH_ENDPOINT BENCH_BUCKET BENCH_CONCURRENCY
|
||||
BENCH_CONCURRENCIES BENCH_DURATION BENCH_SIZES OUT_DIR
|
||||
BENCH_WAIT_MODE (ready|service, default: ready)
|
||||
BENCH_READY_TIMEOUT_SECS (default: 180)
|
||||
USAGE
|
||||
}
|
||||
|
||||
log_info() {
|
||||
printf '[INFO] %s\n' "$*"
|
||||
}
|
||||
|
||||
log_warn() {
|
||||
printf '[WARN] %s\n' "$*"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
printf '[ERROR] %s\n' "$*" >&2
|
||||
}
|
||||
|
||||
require_cmd() {
|
||||
if ! command -v "$1" >/dev/null 2>&1; then
|
||||
log_error "command not found: $1"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
compose() {
|
||||
if [[ "${WITH_OBSERVABILITY}" == "true" ]]; then
|
||||
docker compose \
|
||||
--project-name "${PROJECT_NAME}" \
|
||||
-f "${OBS_COMPOSE}" \
|
||||
-f "${CLUSTER_COMPOSE}" \
|
||||
"$@"
|
||||
else
|
||||
docker compose \
|
||||
--project-name "${PROJECT_NAME}" \
|
||||
-f "${CLUSTER_COMPOSE}" \
|
||||
"$@"
|
||||
fi
|
||||
}
|
||||
|
||||
resolve_bool() {
|
||||
local key="$1"
|
||||
local value="$2"
|
||||
case "${value}" in
|
||||
true|false) ;;
|
||||
*)
|
||||
log_error "invalid ${key}: ${value} (expected true|false)"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
resolve_probe_mode() {
|
||||
case "${WAIT_PROBE_MODE}" in
|
||||
service|ready) ;;
|
||||
*)
|
||||
log_error "invalid WAIT_PROBE_MODE: ${WAIT_PROBE_MODE} (expected service|ready)"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
resolve_bench_wait_mode() {
|
||||
case "${BENCH_WAIT_MODE}" in
|
||||
ready|service) ;;
|
||||
*)
|
||||
log_error "invalid BENCH_WAIT_MODE: ${BENCH_WAIT_MODE} (expected ready|service)"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
resolve_bench_concurrency() {
|
||||
if [[ -n "${BENCH_CONCURRENCIES}" && -n "${BENCH_CONCURRENCY}" && "${BENCH_CONCURRENCIES}" != "${BENCH_CONCURRENCY}" ]]; then
|
||||
log_warn "BENCH_CONCURRENCY is ignored because BENCH_CONCURRENCIES is set"
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ -n "${BENCH_CONCURRENCIES}" ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ -n "${BENCH_CONCURRENCY}" ]]; then
|
||||
BENCH_CONCURRENCIES="${BENCH_CONCURRENCY}"
|
||||
return
|
||||
fi
|
||||
|
||||
# BENCH_CONCURRENCIES="8,16,32,64,128"
|
||||
BENCH_CONCURRENCIES="8,16"
|
||||
}
|
||||
|
||||
cluster_compose_uses_otel_network() {
|
||||
# Detect whether any service in cluster compose joins otel-network.
|
||||
grep -Eq '^[[:space:]]*-[[:space:]]*otel-network([[:space:]]*#.*)?$' "${CLUSTER_COMPOSE}"
|
||||
}
|
||||
|
||||
obs_compose_has_otel_collector() {
|
||||
grep -Eq '^[[:space:]]*otel-collector:[[:space:]]*$' "${OBS_COMPOSE}"
|
||||
}
|
||||
|
||||
resolve_default_obs_endpoint() {
|
||||
if [[ "${WITH_OBSERVABILITY}" != "true" ]]; then
|
||||
RUSTFS_OBS_ENDPOINT="http://host.docker.internal:4318"
|
||||
log_info "Auto-selected RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT} (observability stack disabled)"
|
||||
return
|
||||
fi
|
||||
|
||||
if cluster_compose_uses_otel_network && obs_compose_has_otel_collector; then
|
||||
RUSTFS_OBS_ENDPOINT="http://otel-collector:4318"
|
||||
log_info "Auto-selected RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT} (shared docker network detected)"
|
||||
return
|
||||
fi
|
||||
|
||||
RUSTFS_OBS_ENDPOINT="http://host.docker.internal:4318"
|
||||
log_info "Auto-selected RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT} (cross-network fallback)"
|
||||
}
|
||||
|
||||
docker_daemon_ready() {
|
||||
docker info >/dev/null 2>&1
|
||||
}
|
||||
|
||||
port_is_occupied() {
|
||||
local port="$1"
|
||||
|
||||
if command -v lsof >/dev/null 2>&1; then
|
||||
lsof -nP -iTCP:"${port}" -sTCP:LISTEN >/dev/null 2>&1
|
||||
return $?
|
||||
fi
|
||||
|
||||
if command -v ss >/dev/null 2>&1; then
|
||||
ss -ltn "sport = :${port}" 2>/dev/null | awk 'NR>1 {found=1} END{exit found?0:1}'
|
||||
return $?
|
||||
fi
|
||||
|
||||
if command -v netstat >/dev/null 2>&1; then
|
||||
netstat -an 2>/dev/null | grep -E "[\.\:]${port}[[:space:]].*LISTEN" >/dev/null 2>&1
|
||||
return $?
|
||||
fi
|
||||
|
||||
# Fallback: no tool available; treat as unknown (not occupied) and rely on compose failure.
|
||||
return 1
|
||||
}
|
||||
|
||||
print_port_owner() {
|
||||
local port="$1"
|
||||
|
||||
if command -v lsof >/dev/null 2>&1; then
|
||||
lsof -nP -iTCP:"${port}" -sTCP:LISTEN 2>/dev/null | awk 'NR==1 || NR==2 {print " " $0}'
|
||||
return
|
||||
fi
|
||||
|
||||
if command -v ss >/dev/null 2>&1; then
|
||||
ss -ltnp "sport = :${port}" 2>/dev/null | awk 'NR==1 || NR==2 {print " " $0}'
|
||||
fi
|
||||
}
|
||||
|
||||
cleanup_existing_project_containers() {
|
||||
local existing_ids
|
||||
existing_ids="$(docker ps -aq --filter "label=com.docker.compose.project=${PROJECT_NAME}")"
|
||||
|
||||
if [[ -z "${existing_ids}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
log_warn "Found existing containers for project ${PROJECT_NAME}."
|
||||
docker ps -a --filter "label=com.docker.compose.project=${PROJECT_NAME}" --format ' - {{.Names}} ({{.Status}})'
|
||||
|
||||
if [[ "${PRECHECK_AUTO_CLEANUP}" == "true" ]]; then
|
||||
log_info "PRECHECK_AUTO_CLEANUP=true, removing existing project containers."
|
||||
# shellcheck disable=SC2086
|
||||
docker rm -f ${existing_ids} >/dev/null
|
||||
else
|
||||
log_error "existing project containers detected and PRECHECK_AUTO_CLEANUP=false"
|
||||
log_error "run docker compose down --remove-orphans first, or set PRECHECK_AUTO_CLEANUP=true"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
check_required_ports_free() {
|
||||
local required_ports=(
|
||||
9000 9001 9002 9003
|
||||
)
|
||||
local occupied_ports=()
|
||||
local port
|
||||
|
||||
if [[ "${WITH_OBSERVABILITY}" == "true" ]]; then
|
||||
required_ports+=(
|
||||
1888 3000 3100 3200 4040 4317 4318 55679 8888 8889 9090 13133 14269 16686
|
||||
)
|
||||
fi
|
||||
|
||||
for port in "${required_ports[@]}"; do
|
||||
if port_is_occupied "${port}"; then
|
||||
occupied_ports+=("${port}")
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "${#occupied_ports[@]}" -gt 0 ]]; then
|
||||
log_error "required host ports are occupied: ${occupied_ports[*]}"
|
||||
for port in "${occupied_ports[@]}"; do
|
||||
print_port_owner "${port}" || true
|
||||
done
|
||||
log_error "free these ports or run with a different compose/profile before retrying"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
ensure_runtime_image_exists() {
|
||||
if ! docker image inspect "${IMAGE_TAG}" >/dev/null 2>&1; then
|
||||
log_error "image not found: ${IMAGE_TAG}"
|
||||
log_error "build it first or rerun without --skip-build"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
check_cluster_volumes_writable() {
|
||||
local node_idx
|
||||
local disk_idx
|
||||
local volume_name
|
||||
|
||||
log_info "Checking cluster data volumes writable"
|
||||
# Do not pre-create compose-managed volumes here.
|
||||
# If we create them via plain docker run, compose will warn:
|
||||
# "already exists but was not created by Docker Compose".
|
||||
for node_idx in 1 2 3 4; do
|
||||
for disk_idx in 1 2 3 4; do
|
||||
volume_name="${PROJECT_NAME}_node${node_idx}_data_${disk_idx}"
|
||||
if ! docker volume inspect "${volume_name}" >/dev/null 2>&1; then
|
||||
log_info "volume not present yet (will be created by compose): ${volume_name}"
|
||||
continue
|
||||
fi
|
||||
if ! docker run --rm --entrypoint sh -v "${volume_name}:/probe" "${IMAGE_TAG}" -c \
|
||||
'set -e; touch /probe/.rwtest; rm -f /probe/.rwtest' >/dev/null 2>&1; then
|
||||
log_error "volume write check failed: ${volume_name}"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
run_precheck_before_build() {
|
||||
log_info "Running precheck: docker daemon, residue containers, host ports"
|
||||
|
||||
if ! docker_daemon_ready; then
|
||||
log_error "cannot connect to docker daemon (permission or runtime not ready)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cleanup_existing_project_containers
|
||||
check_required_ports_free
|
||||
}
|
||||
|
||||
run_precheck_after_build() {
|
||||
log_info "Running precheck: image exists, cluster volumes writable"
|
||||
ensure_runtime_image_exists
|
||||
check_cluster_volumes_writable
|
||||
}
|
||||
|
||||
node_port() {
|
||||
case "$1" in
|
||||
node1) echo "9000" ;;
|
||||
node2) echo "9001" ;;
|
||||
node3) echo "9002" ;;
|
||||
node4) echo "9003" ;;
|
||||
*)
|
||||
log_error "unknown node name: $1 (expected node1..node4)"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
wait_http_ok() {
|
||||
local url="$1"
|
||||
local start now
|
||||
start="$(date +%s)"
|
||||
|
||||
while true; do
|
||||
if curl -fsS --connect-timeout 2 --max-time 3 "${url}" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
now="$(date +%s)"
|
||||
if (( now - start >= WAIT_TIMEOUT_SECS )); then
|
||||
log_error "timed out waiting for ${url}"
|
||||
return 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
}
|
||||
|
||||
probe_node_service_ok() {
|
||||
local port="$1"
|
||||
local health_code root_code
|
||||
|
||||
health_code="$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 2 --max-time 3 "http://127.0.0.1:${port}/health" || true)"
|
||||
if [[ "${health_code}" != "200" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ "${WAIT_PROBE_MODE}" == "ready" ]]; then
|
||||
local ready_code
|
||||
ready_code="$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 2 --max-time 3 "http://127.0.0.1:${port}/health/ready" || true)"
|
||||
[[ "${ready_code}" == "200" ]]
|
||||
return $?
|
||||
fi
|
||||
|
||||
# Service mode: keep startup probe permissive to avoid local false negatives.
|
||||
# Benchmark phase has its own stricter readiness gate via wait_bench_endpoint_ready.
|
||||
root_code="$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 2 --max-time 3 "http://127.0.0.1:${port}/" || true)"
|
||||
case "${root_code}" in
|
||||
[1-5][0-9][0-9]) return 0 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
probe_bench_endpoint_ok() {
|
||||
local endpoint health_url ready_url root_url
|
||||
local health_code ready_code root_code
|
||||
endpoint="${BENCH_ENDPOINT%/}"
|
||||
health_url="${endpoint}/health"
|
||||
ready_url="${endpoint}/health/ready"
|
||||
root_url="${endpoint}/"
|
||||
|
||||
health_code="$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 2 --max-time 3 "${health_url}" || true)"
|
||||
if [[ "${health_code}" != "200" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ "${BENCH_WAIT_MODE}" == "ready" ]]; then
|
||||
ready_code="$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 2 --max-time 3 "${ready_url}" || true)"
|
||||
[[ "${ready_code}" == "200" ]]
|
||||
return $?
|
||||
fi
|
||||
|
||||
root_code="$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 2 --max-time 3 "${root_url}" || true)"
|
||||
case "${root_code}" in
|
||||
2[0-9][0-9]|3[0-9][0-9]|401|403|404) return 0 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
wait_bench_endpoint_ready() {
|
||||
local start now
|
||||
start="$(date +%s)"
|
||||
|
||||
while true; do
|
||||
if probe_bench_endpoint_ok; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
now="$(date +%s)"
|
||||
if (( now - start >= BENCH_READY_TIMEOUT_SECS )); then
|
||||
log_error "timed out waiting for benchmark endpoint ${BENCH_ENDPOINT} (mode=${BENCH_WAIT_MODE})"
|
||||
return 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
}
|
||||
|
||||
wait_node_probe_ok() {
|
||||
local port="$1"
|
||||
local start now
|
||||
start="$(date +%s)"
|
||||
|
||||
while true; do
|
||||
if probe_node_service_ok "${port}"; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
now="$(date +%s)"
|
||||
if (( now - start >= WAIT_TIMEOUT_SECS )); then
|
||||
log_error "timed out waiting for node probe on 127.0.0.1:${port} (mode=${WAIT_PROBE_MODE})"
|
||||
return 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
}
|
||||
|
||||
wait_cluster_ready() {
|
||||
local port
|
||||
for port in 9000 9001 9002 9003; do
|
||||
wait_node_probe_ok "${port}"
|
||||
done
|
||||
}
|
||||
|
||||
probe_survivors_ready() {
|
||||
local failover_port="$1"
|
||||
local port
|
||||
for port in 9000 9001 9002 9003; do
|
||||
if [[ "${port}" == "${failover_port}" ]]; then
|
||||
continue
|
||||
fi
|
||||
if ! probe_node_service_ok "${port}"; then
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
run_failover_validation() {
|
||||
local failover_port
|
||||
local probe_file
|
||||
local summary_file
|
||||
local event_epoch
|
||||
local end_epoch
|
||||
local ts
|
||||
local first_fail
|
||||
local first_recover
|
||||
local recovery_secs
|
||||
|
||||
failover_port="$(node_port "${FAILOVER_NODE}")"
|
||||
probe_file="${OUT_DIR}/failover-probe.csv"
|
||||
summary_file="${OUT_DIR}/failover-summary.txt"
|
||||
mkdir -p "$(dirname "${probe_file}")"
|
||||
|
||||
log_info "Running failover validation: stopping ${FAILOVER_NODE}"
|
||||
sleep "${FAILOVER_WARMUP_SECS}"
|
||||
|
||||
compose stop "${FAILOVER_NODE}" >/dev/null
|
||||
event_epoch="$(date +%s)"
|
||||
end_epoch="$((event_epoch + FAILOVER_SAMPLE_SECS))"
|
||||
|
||||
echo "timestamp_epoch,status" > "${probe_file}"
|
||||
while (( "$(date +%s)" <= end_epoch )); do
|
||||
ts="$(date +%s)"
|
||||
if probe_survivors_ready "${failover_port}"; then
|
||||
echo "${ts},ok" >> "${probe_file}"
|
||||
else
|
||||
echo "${ts},fail" >> "${probe_file}"
|
||||
fi
|
||||
sleep "${FAILOVER_INTERVAL_SECS}"
|
||||
done
|
||||
|
||||
first_fail="$(awk -F',' 'NR>1 && $2=="fail" {print $1; exit}' "${probe_file}")"
|
||||
if [[ -z "${first_fail}" ]]; then
|
||||
recovery_secs="0"
|
||||
{
|
||||
echo "failover_node=${FAILOVER_NODE}"
|
||||
echo "outage_observed=false"
|
||||
echo "recovery_seconds=${recovery_secs}"
|
||||
echo "note=no survivor readiness interruption observed in probe window"
|
||||
} > "${summary_file}"
|
||||
else
|
||||
first_recover="$(awk -F',' -v fail_ts="${first_fail}" 'NR>1 && $1>fail_ts && $2=="ok" {print $1; exit}' "${probe_file}")"
|
||||
if [[ -z "${first_recover}" ]]; then
|
||||
{
|
||||
echo "failover_node=${FAILOVER_NODE}"
|
||||
echo "outage_observed=true"
|
||||
echo "recovery_seconds=unrecovered_within_${FAILOVER_SAMPLE_SECS}s"
|
||||
echo "first_fail_epoch=${first_fail}"
|
||||
} > "${summary_file}"
|
||||
else
|
||||
recovery_secs="$((first_recover - first_fail))"
|
||||
{
|
||||
echo "failover_node=${FAILOVER_NODE}"
|
||||
echo "outage_observed=true"
|
||||
echo "first_fail_epoch=${first_fail}"
|
||||
echo "first_recover_epoch=${first_recover}"
|
||||
echo "recovery_seconds=${recovery_secs}"
|
||||
} > "${summary_file}"
|
||||
fi
|
||||
fi
|
||||
|
||||
log_info "Restarting ${FAILOVER_NODE}"
|
||||
compose start "${FAILOVER_NODE}" >/dev/null
|
||||
wait_node_probe_ok "${failover_port}"
|
||||
wait_cluster_ready
|
||||
}
|
||||
|
||||
run_benchmark() {
|
||||
local bench_out_dir
|
||||
local conc
|
||||
local conc_dir
|
||||
bench_out_dir="${OUT_DIR}/benchmark"
|
||||
mkdir -p "${bench_out_dir}"
|
||||
|
||||
if ! command -v warp >/dev/null 2>&1; then
|
||||
log_error "warp is required for benchmark phase. Please install warp or run with --skip-bench."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log_info "Waiting for benchmark endpoint readiness (mode=${BENCH_WAIT_MODE})"
|
||||
wait_bench_endpoint_ready
|
||||
|
||||
IFS=',' read -r -a conc_list <<< "${BENCH_CONCURRENCIES}"
|
||||
for conc in "${conc_list[@]}"; do
|
||||
conc="$(echo "${conc}" | xargs)"
|
||||
if [[ -z "${conc}" ]]; then
|
||||
continue
|
||||
fi
|
||||
if ! [[ "${conc}" =~ ^[0-9]+$ ]] || [[ "${conc}" -le 0 ]]; then
|
||||
log_error "invalid concurrency in BENCH_CONCURRENCIES: ${conc}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
conc_dir="${bench_out_dir}/concurrency-${conc}"
|
||||
log_info "Running benchmark sequentially with concurrency=${conc}"
|
||||
(
|
||||
cd "${PROJECT_ROOT}"
|
||||
./scripts/run_object_batch_bench.sh \
|
||||
--tool warp \
|
||||
--endpoint "${BENCH_ENDPOINT}" \
|
||||
--access-key "${RUSTFS_ACCESS_KEY}" \
|
||||
--secret-key "${RUSTFS_SECRET_KEY}" \
|
||||
--bucket "${BENCH_BUCKET}" \
|
||||
--concurrency "${conc}" \
|
||||
--duration "${BENCH_DURATION}" \
|
||||
--sizes "${BENCH_SIZES}" \
|
||||
--out-dir "${conc_dir}"
|
||||
)
|
||||
done
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
if [[ "${KEEP_UP}" == "true" ]]; then
|
||||
log_info "KEEP_UP=true, leaving containers running"
|
||||
return
|
||||
fi
|
||||
|
||||
log_info "Stopping compose services"
|
||||
compose down --remove-orphans >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
parse_args() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--cluster-compose)
|
||||
CLUSTER_COMPOSE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--obs-compose)
|
||||
OBS_COMPOSE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--project-name)
|
||||
PROJECT_NAME="$2"
|
||||
shift 2
|
||||
;;
|
||||
--image-tag)
|
||||
IMAGE_TAG="$2"
|
||||
shift 2
|
||||
;;
|
||||
--with-observability)
|
||||
WITH_OBSERVABILITY=true
|
||||
shift
|
||||
;;
|
||||
--without-observability)
|
||||
WITH_OBSERVABILITY=false
|
||||
shift
|
||||
;;
|
||||
--skip-build)
|
||||
BUILD_LOCAL_IMAGE=false
|
||||
shift
|
||||
;;
|
||||
--skip-failover)
|
||||
RUN_FAILOVER=false
|
||||
shift
|
||||
;;
|
||||
--skip-bench)
|
||||
RUN_BENCHMARK=false
|
||||
shift
|
||||
;;
|
||||
--keep-up)
|
||||
KEEP_UP=true
|
||||
shift
|
||||
;;
|
||||
--failover-node)
|
||||
FAILOVER_NODE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--obs-endpoint)
|
||||
RUSTFS_OBS_ENDPOINT="$2"
|
||||
shift 2
|
||||
;;
|
||||
--bench-endpoint)
|
||||
BENCH_ENDPOINT="$2"
|
||||
shift 2
|
||||
;;
|
||||
--bench-sizes)
|
||||
BENCH_SIZES="$2"
|
||||
shift 2
|
||||
;;
|
||||
--bench-concurrency)
|
||||
BENCH_CONCURRENCY="$2"
|
||||
BENCH_CONCURRENCIES="$2"
|
||||
shift 2
|
||||
;;
|
||||
--bench-concurrencies)
|
||||
BENCH_CONCURRENCIES="$2"
|
||||
shift 2
|
||||
;;
|
||||
--bench-duration)
|
||||
BENCH_DURATION="$2"
|
||||
shift 2
|
||||
;;
|
||||
--out-dir)
|
||||
OUT_DIR="$2"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
log_error "unknown argument: $1"
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
main() {
|
||||
parse_args "$@"
|
||||
|
||||
resolve_bool "WITH_OBSERVABILITY" "${WITH_OBSERVABILITY}"
|
||||
resolve_bool "BUILD_LOCAL_IMAGE" "${BUILD_LOCAL_IMAGE}"
|
||||
resolve_bool "RUN_FAILOVER" "${RUN_FAILOVER}"
|
||||
resolve_bool "RUN_BENCHMARK" "${RUN_BENCHMARK}"
|
||||
resolve_bool "KEEP_UP" "${KEEP_UP}"
|
||||
resolve_bool "PRECHECK_AUTO_CLEANUP" "${PRECHECK_AUTO_CLEANUP}"
|
||||
resolve_probe_mode
|
||||
resolve_bench_wait_mode
|
||||
resolve_bench_concurrency
|
||||
|
||||
require_cmd docker
|
||||
require_cmd curl
|
||||
require_cmd awk
|
||||
|
||||
if [[ ! -f "${CLUSTER_COMPOSE}" ]]; then
|
||||
log_error "cluster compose file not found: ${CLUSTER_COMPOSE}"
|
||||
exit 1
|
||||
fi
|
||||
if [[ "${WITH_OBSERVABILITY}" == "true" && ! -f "${OBS_COMPOSE}" ]]; then
|
||||
log_error "observability compose file not found: ${OBS_COMPOSE}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "${RUSTFS_OBS_ENDPOINT}" ]]; then
|
||||
resolve_default_obs_endpoint
|
||||
fi
|
||||
|
||||
if [[ "${RUSTFS_OBS_ENDPOINT}" == "http://127.0.0.1:4318" ]]; then
|
||||
log_warn "RUSTFS_OBS_ENDPOINT is set to container loopback default (${RUSTFS_OBS_ENDPOINT})."
|
||||
log_warn "If you need host collector routing, consider: --obs-endpoint http://host.docker.internal:4318"
|
||||
fi
|
||||
|
||||
mkdir -p "${OUT_DIR}"
|
||||
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
export RUSTFS_IMAGE="${IMAGE_TAG}"
|
||||
export RUSTFS_ACCESS_KEY
|
||||
export RUSTFS_SECRET_KEY
|
||||
export RUSTFS_OBS_ENDPOINT
|
||||
export RUSTFS_UNSAFE_BYPASS_DISK_CHECK
|
||||
|
||||
run_precheck_before_build
|
||||
|
||||
if [[ "${BUILD_LOCAL_IMAGE}" == "true" ]]; then
|
||||
log_info "Building local image from Dockerfile.source: ${IMAGE_TAG}"
|
||||
docker build -f "${PROJECT_ROOT}/Dockerfile.source" -t "${IMAGE_TAG}" "${PROJECT_ROOT}"
|
||||
else
|
||||
log_info "Skipping image build"
|
||||
fi
|
||||
|
||||
run_precheck_after_build
|
||||
|
||||
log_info "Starting compose stack"
|
||||
compose up -d
|
||||
|
||||
log_info "Waiting for 4-node cluster readiness (mode=${WAIT_PROBE_MODE})"
|
||||
wait_cluster_ready
|
||||
|
||||
if [[ "${RUN_FAILOVER}" == "true" ]]; then
|
||||
run_failover_validation
|
||||
else
|
||||
log_info "Skipping failover validation"
|
||||
fi
|
||||
|
||||
if [[ "${RUN_BENCHMARK}" == "true" ]]; then
|
||||
run_benchmark
|
||||
else
|
||||
log_info "Skipping benchmark"
|
||||
fi
|
||||
|
||||
log_info "Validation finished"
|
||||
log_info "Artifacts directory: ${OUT_DIR}"
|
||||
log_info "Failover summary: ${OUT_DIR}/failover-summary.txt"
|
||||
log_info "Failover probe: ${OUT_DIR}/failover-probe.csv"
|
||||
log_info "Benchmark summary: ${OUT_DIR}/benchmark/summary.csv"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -74,6 +74,16 @@ require_cmd() {
|
||||
fi
|
||||
}
|
||||
|
||||
normalize_warp_host() {
|
||||
local raw="$1"
|
||||
raw="${raw#http://}"
|
||||
raw="${raw#https://}"
|
||||
raw="${raw%%/*}"
|
||||
raw="${raw%%\?*}"
|
||||
raw="${raw%%\#*}"
|
||||
echo "$raw"
|
||||
}
|
||||
|
||||
parse_args() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
@@ -127,6 +137,14 @@ validate_args() {
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
if [[ "$TOOL" == "warp" ]]; then
|
||||
local warp_host
|
||||
warp_host="$(normalize_warp_host "$ENDPOINT")"
|
||||
if [[ -z "$warp_host" ]]; then
|
||||
echo "ERROR: invalid --endpoint for warp: $ENDPOINT" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
setup_output() {
|
||||
@@ -141,7 +159,7 @@ setup_output() {
|
||||
extract_value() {
|
||||
local pattern="$1"
|
||||
local file="$2"
|
||||
rg -o "$pattern" "$file" | head -n1 | sed -E "s/$pattern/\\1/" || true
|
||||
rg -o "$pattern" "$file" | head -n1 || true
|
||||
}
|
||||
|
||||
collect_metrics() {
|
||||
@@ -161,9 +179,11 @@ run_one() {
|
||||
echo "==== [$TOOL] size=$size concurrency=$CONCURRENCY ===="
|
||||
|
||||
if [[ "$TOOL" == "warp" ]]; then
|
||||
local warp_host
|
||||
warp_host="$(normalize_warp_host "$ENDPOINT")"
|
||||
local cmd=(
|
||||
"$WARP_BIN" "$WARP_MODE"
|
||||
"--host" "$ENDPOINT"
|
||||
"--host" "$warp_host"
|
||||
"--access-key" "$ACCESS_KEY"
|
||||
"--secret-key" "$SECRET_KEY"
|
||||
"--bucket" "$BUCKET"
|
||||
@@ -175,7 +195,9 @@ run_one() {
|
||||
if [[ "$INSECURE" == "true" ]]; then
|
||||
cmd+=("--insecure")
|
||||
fi
|
||||
cmd+=("${EXTRA_ARGS[@]}")
|
||||
if [[ ${EXTRA_ARGS[@]+_} ]]; then
|
||||
cmd+=("${EXTRA_ARGS[@]}")
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
printf '[DRY-RUN] %q ' "${cmd[@]}"
|
||||
@@ -201,7 +223,9 @@ run_one() {
|
||||
if [[ "$INSECURE" == "true" ]]; then
|
||||
cmd+=("-insecure")
|
||||
fi
|
||||
cmd+=("${EXTRA_ARGS[@]}")
|
||||
if [[ ${EXTRA_ARGS[@]+_} ]]; then
|
||||
cmd+=("${EXTRA_ARGS[@]}")
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
printf '[DRY-RUN] %q ' "${cmd[@]}"
|
||||
@@ -255,4 +279,3 @@ main() {
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
|
||||
@@ -95,6 +95,16 @@ require_cmd() {
|
||||
fi
|
||||
}
|
||||
|
||||
normalize_warp_host() {
|
||||
local raw="$1"
|
||||
raw="${raw#http://}"
|
||||
raw="${raw#https://}"
|
||||
raw="${raw%%/*}"
|
||||
raw="${raw%%\?*}"
|
||||
raw="${raw%%\#*}"
|
||||
echo "$raw"
|
||||
}
|
||||
|
||||
parse_args() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
@@ -162,6 +172,14 @@ validate_args() {
|
||||
echo "ERROR: --baseline-csv does not exist: $BASELINE_CSV" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ "$TOOL" == "warp" ]]; then
|
||||
local warp_host
|
||||
warp_host="$(normalize_warp_host "$ENDPOINT")"
|
||||
if [[ -z "$warp_host" ]]; then
|
||||
echo "ERROR: invalid --endpoint for warp: $ENDPOINT" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
setup_output() {
|
||||
@@ -286,9 +304,11 @@ run_one_attempt() {
|
||||
local status="ok"
|
||||
|
||||
if [[ "$TOOL" == "warp" ]]; then
|
||||
local warp_host
|
||||
warp_host="$(normalize_warp_host "$ENDPOINT")"
|
||||
local cmd=(
|
||||
"$WARP_BIN" "$WARP_MODE"
|
||||
"--host" "$ENDPOINT"
|
||||
"--host" "$warp_host"
|
||||
"--access-key" "$ACCESS_KEY"
|
||||
"--secret-key" "$SECRET_KEY"
|
||||
"--bucket" "$BUCKET"
|
||||
@@ -300,7 +320,9 @@ run_one_attempt() {
|
||||
if [[ "$INSECURE" == "true" ]]; then
|
||||
cmd+=("--insecure")
|
||||
fi
|
||||
cmd+=("${EXTRA_ARGS[@]}")
|
||||
if [[ ${EXTRA_ARGS[@]+_} ]]; then
|
||||
cmd+=("${EXTRA_ARGS[@]}")
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
printf '[DRY-RUN] %q ' "${cmd[@]}"
|
||||
@@ -326,7 +348,9 @@ run_one_attempt() {
|
||||
if [[ "$INSECURE" == "true" ]]; then
|
||||
cmd+=("-insecure")
|
||||
fi
|
||||
cmd+=("${EXTRA_ARGS[@]}")
|
||||
if [[ ${EXTRA_ARGS[@]+_} ]]; then
|
||||
cmd+=("${EXTRA_ARGS[@]}")
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
printf '[DRY-RUN] %q ' "${cmd[@]}"
|
||||
@@ -484,4 +508,3 @@ main() {
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
|
||||
Executable
+177
@@ -0,0 +1,177 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
COMPOSE_FILE="${COMPOSE_FILE:-docker-compose-simple.yml}"
|
||||
WAIT_TIMEOUT_SECS="${WAIT_TIMEOUT_SECS:-120}"
|
||||
KEEP_UP="${KEEP_UP:-false}"
|
||||
RUN_S3_TESTS="${RUN_S3_TESTS:-true}"
|
||||
BUILD_LOCAL_IMAGE="${BUILD_LOCAL_IMAGE:-true}"
|
||||
S3_HOST="${S3_HOST:-127.0.0.1}"
|
||||
S3_PORT="${S3_PORT:-9000}"
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
scripts/validate_issue_1365_docker.sh [options]
|
||||
|
||||
Options:
|
||||
--compose-file <path> docker compose file (default: docker-compose-simple.yml)
|
||||
--wait-timeout <secs> health wait timeout (default: 120)
|
||||
--keep-up keep compose services up after the script exits
|
||||
--skip-s3-tests skip scripts/s3-tests/run.sh
|
||||
--skip-build skip local Dockerfile.source image build
|
||||
-h, --help show help
|
||||
|
||||
Environment:
|
||||
COMPOSE_FILE
|
||||
WAIT_TIMEOUT_SECS
|
||||
KEEP_UP
|
||||
RUN_S3_TESTS
|
||||
BUILD_LOCAL_IMAGE
|
||||
S3_HOST
|
||||
S3_PORT
|
||||
USAGE
|
||||
}
|
||||
|
||||
log_info() {
|
||||
printf '[INFO] %s\n' "$*"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
printf '[ERROR] %s\n' "$*" >&2
|
||||
}
|
||||
|
||||
require_cmd() {
|
||||
if ! command -v "$1" >/dev/null 2>&1; then
|
||||
log_error "command not found: $1"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
compose() {
|
||||
local compose_path
|
||||
compose_path="$(resolve_compose_file)"
|
||||
docker compose -f "${compose_path}" "$@"
|
||||
}
|
||||
|
||||
resolve_compose_file() {
|
||||
if [[ "${COMPOSE_FILE}" = /* ]]; then
|
||||
printf '%s\n' "${COMPOSE_FILE}"
|
||||
else
|
||||
printf '%s\n' "${PROJECT_ROOT}/${COMPOSE_FILE}"
|
||||
fi
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
if [[ "${KEEP_UP}" == "true" ]]; then
|
||||
log_info "KEEP_UP=true, leaving compose services running"
|
||||
return
|
||||
fi
|
||||
|
||||
log_info "Stopping docker compose services"
|
||||
compose down -v >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
parse_args() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--compose-file)
|
||||
COMPOSE_FILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--wait-timeout)
|
||||
WAIT_TIMEOUT_SECS="$2"
|
||||
shift 2
|
||||
;;
|
||||
--keep-up)
|
||||
KEEP_UP=true
|
||||
shift
|
||||
;;
|
||||
--skip-s3-tests)
|
||||
RUN_S3_TESTS=false
|
||||
shift
|
||||
;;
|
||||
--skip-build)
|
||||
BUILD_LOCAL_IMAGE=false
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
log_error "unknown argument: $1"
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
wait_for_endpoint() {
|
||||
local url="$1"
|
||||
local start now
|
||||
start="$(date +%s)"
|
||||
|
||||
while true; do
|
||||
if curl -fsS --connect-timeout 2 --max-time 3 "${url}" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
now="$(date +%s)"
|
||||
if (( now - start >= WAIT_TIMEOUT_SECS )); then
|
||||
log_error "timed out waiting for ${url}"
|
||||
compose ps || true
|
||||
compose logs rustfs --tail 200 || true
|
||||
return 1
|
||||
fi
|
||||
|
||||
sleep 2
|
||||
done
|
||||
}
|
||||
|
||||
main() {
|
||||
parse_args "$@"
|
||||
require_cmd docker
|
||||
require_cmd curl
|
||||
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
if [[ "${BUILD_LOCAL_IMAGE}" == "true" ]]; then
|
||||
log_info "Building rustfs/rustfs:latest from Dockerfile.source"
|
||||
docker build -f "${PROJECT_ROOT}/Dockerfile.source" -t rustfs/rustfs:latest "${PROJECT_ROOT}"
|
||||
else
|
||||
log_info "Skipping local image build"
|
||||
fi
|
||||
|
||||
if [[ -z "${RUSTFS_UNSAFE_BYPASS_DISK_CHECK+x}" ]]; then
|
||||
export RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true
|
||||
log_info "RUSTFS_UNSAFE_BYPASS_DISK_CHECK not set; defaulting to true for local validation"
|
||||
fi
|
||||
|
||||
log_info "Starting docker compose from $(resolve_compose_file)"
|
||||
compose up -d
|
||||
|
||||
log_info "Waiting for RustFS health endpoint"
|
||||
wait_for_endpoint "http://${S3_HOST}:${S3_PORT}/health"
|
||||
|
||||
log_info "Waiting for RustFS readiness endpoint"
|
||||
wait_for_endpoint "http://${S3_HOST}:${S3_PORT}/health/ready"
|
||||
|
||||
log_info "Docker health checks passed"
|
||||
|
||||
if [[ "${RUN_S3_TESTS}" == "true" ]]; then
|
||||
log_info "Running S3 compatibility tests against the running dockerized service"
|
||||
(
|
||||
cd "${PROJECT_ROOT}"
|
||||
DEPLOY_MODE=existing S3_HOST="${S3_HOST}" S3_PORT="${S3_PORT}" ./scripts/s3-tests/run.sh
|
||||
)
|
||||
else
|
||||
log_info "Skipping S3 compatibility tests"
|
||||
fi
|
||||
|
||||
log_info "Issue 1365 docker validation completed successfully"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user