From 2953558f414df849b5fdc9ac98e6ed83524b961b Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 28 Apr 2026 18:26:14 +0800 Subject: [PATCH] fix(lifecycle): prevent eager date-expiry deletion on config update (#2708) --- .docker/compose/README.md | 18 + .../docker-compose.cluster.local-build.yml | 144 +++ .../compose/docker-compose.observability.yaml | 68 +- Cargo.lock | 191 ++-- Cargo.toml | 2 +- Dockerfile.decommission-local | 2 +- Dockerfile.source | 27 +- crates/config/src/constants/drive.rs | 8 + crates/config/src/constants/internode.rs | 62 ++ crates/config/src/constants/mod.rs | 1 + crates/config/src/lib.rs | 2 + crates/ecstore/src/batch_processor.rs | 117 ++- .../bucket/lifecycle/bucket_lifecycle_ops.rs | 98 +- crates/ecstore/src/disk/disk_store.rs | 74 +- crates/ecstore/src/rpc/client.rs | 43 +- crates/ecstore/src/rpc/peer_rest_client.rs | 965 ++++++++++++------ crates/ecstore/src/rpc/peer_s3_client.rs | 99 +- crates/ecstore/src/rpc/remote_disk.rs | 118 ++- crates/ecstore/src/set_disk/multipart.rs | 216 +++- crates/ecstore/src/set_disk/read.rs | 317 +++++- crates/protos/Cargo.toml | 2 + crates/protos/src/lib.rs | 77 +- docker-compose-simple.yml | 3 + rustfs/src/app/bucket_usecase.rs | 1 + .../run_four_node_cluster_failover_bench.sh | 766 ++++++++++++++ scripts/run_object_batch_bench.sh | 33 +- scripts/run_object_batch_bench_enhanced.sh | 31 +- scripts/validate_issue_1365_docker.sh | 177 ++++ 28 files changed, 3069 insertions(+), 593 deletions(-) create mode 100644 .docker/compose/docker-compose.cluster.local-build.yml create mode 100644 crates/config/src/constants/internode.rs create mode 100755 scripts/run_four_node_cluster_failover_bench.sh create mode 100755 scripts/validate_issue_1365_docker.sh diff --git a/.docker/compose/README.md b/.docker/compose/README.md index 06a91beed..1065c46e4 100644 --- a/.docker/compose/README.md +++ b/.docker/compose/README.md @@ -36,6 +36,24 @@ To start a 4-node cluster for distributed testing: docker compose -f .docker/compose/docker-compose.cluster.yaml up -d ``` +### Script-Based 4-Node Validation (Recommended) + +Use the local validation script when you need local-source image build, failover checks, +and benchmark workflow in one command: + +```bash +# Default mode: WAIT_PROBE_MODE=service +# This avoids false negatives where /health/ready remains 503 locally +# while the service path is already available. +./scripts/run_four_node_cluster_failover_bench.sh +``` + +Strict mode is available when you explicitly want `/health/ready == 200` as the gate: + +```bash +WAIT_PROBE_MODE=ready ./scripts/run_four_node_cluster_failover_bench.sh +``` + ### (Deprecated) Minimal Observability ```bash diff --git a/.docker/compose/docker-compose.cluster.local-build.yml b/.docker/compose/docker-compose.cluster.local-build.yml new file mode 100644 index 000000000..173f9ba97 --- /dev/null +++ b/.docker/compose/docker-compose.cluster.local-build.yml @@ -0,0 +1,144 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +services: + node1: + image: ${RUSTFS_IMAGE:-rustfs/rustfs:local-4node} + build: + context: ../.. + dockerfile: Dockerfile.source + hostname: node1 + environment: + - RUSTFS_VOLUMES=http://node{1...4}:9000/data/rustfs{0...3} + - RUSTFS_ADDRESS=:9000 + - RUSTFS_CONSOLE_ENABLE=true + - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin} + - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfsadmin} + - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318} + - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info} + - RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true} + extra_hosts: + - "host.docker.internal:host-gateway" + volumes: + - node1_data_0:/data/rustfs0 + - node1_data_1:/data/rustfs1 + - node1_data_2:/data/rustfs2 + - node1_data_3:/data/rustfs3 + ports: + - "9000:9000" + networks: + - rustfs-cluster-net + + node2: + image: ${RUSTFS_IMAGE:-rustfs/rustfs:local-4node} + build: + context: ../.. + dockerfile: Dockerfile.source + hostname: node2 + environment: + - RUSTFS_VOLUMES=http://node{1...4}:9000/data/rustfs{0...3} + - RUSTFS_ADDRESS=:9000 + - RUSTFS_CONSOLE_ENABLE=true + - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin} + - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfsadmin} + - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318} + - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info} + - RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true} + extra_hosts: + - "host.docker.internal:host-gateway" + volumes: + - node2_data_0:/data/rustfs0 + - node2_data_1:/data/rustfs1 + - node2_data_2:/data/rustfs2 + - node2_data_3:/data/rustfs3 + ports: + - "9001:9000" + networks: + - rustfs-cluster-net + + node3: + image: ${RUSTFS_IMAGE:-rustfs/rustfs:local-4node} + build: + context: ../.. + dockerfile: Dockerfile.source + hostname: node3 + environment: + - RUSTFS_VOLUMES=http://node{1...4}:9000/data/rustfs{0...3} + - RUSTFS_ADDRESS=:9000 + - RUSTFS_CONSOLE_ENABLE=true + - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin} + - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfsadmin} + - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318} + - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info} + - RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true} + extra_hosts: + - "host.docker.internal:host-gateway" + volumes: + - node3_data_0:/data/rustfs0 + - node3_data_1:/data/rustfs1 + - node3_data_2:/data/rustfs2 + - node3_data_3:/data/rustfs3 + ports: + - "9002:9000" + networks: + - rustfs-cluster-net + + node4: + image: ${RUSTFS_IMAGE:-rustfs/rustfs:local-4node} + build: + context: ../.. + dockerfile: Dockerfile.source + hostname: node4 + environment: + - RUSTFS_VOLUMES=http://node{1...4}:9000/data/rustfs{0...3} + - RUSTFS_ADDRESS=:9000 + - RUSTFS_CONSOLE_ENABLE=true + - RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin} + - RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfsadmin} + - RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318} + - RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info} + - RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true} + extra_hosts: + - "host.docker.internal:host-gateway" + volumes: + - node4_data_0:/data/rustfs0 + - node4_data_1:/data/rustfs1 + - node4_data_2:/data/rustfs2 + - node4_data_3:/data/rustfs3 + ports: + - "9003:9000" + networks: + - rustfs-cluster-net + +volumes: + node1_data_0: + node1_data_1: + node1_data_2: + node1_data_3: + node2_data_0: + node2_data_1: + node2_data_2: + node2_data_3: + node3_data_0: + node3_data_1: + node3_data_2: + node3_data_3: + node4_data_0: + node4_data_1: + node4_data_2: + node4_data_3: + +networks: + rustfs-cluster-net: + driver: bridge diff --git a/.docker/compose/docker-compose.observability.yaml b/.docker/compose/docker-compose.observability.yaml index 9c8c3cd07..03b4d765d 100644 --- a/.docker/compose/docker-compose.observability.yaml +++ b/.docker/compose/docker-compose.observability.yaml @@ -26,7 +26,7 @@ services: restart: "no" tempo: - image: grafana/tempo:latest + image: grafana/tempo:2.10.5 user: "10001" command: [ "-config.file=/etc/tempo.yaml" ] volumes: @@ -36,9 +36,19 @@ services: - "3200:3200" # tempo - "4317" # otlp grpc - "4318" # otlp http + - "7946" # memberlist restart: unless-stopped networks: - rustfs-network + depends_on: + tempo-init: + condition: service_completed_successfully + healthcheck: + test: [ "CMD", "/tempo", "-version" ] + interval: 10s + timeout: 5s + retries: 3 + start_period: 15s otel-collector: image: otel/opentelemetry-collector-contrib:latest @@ -61,6 +71,12 @@ services: - jaeger - prometheus - loki + restart: unless-stopped + healthcheck: + test: [ "CMD", "/otelcol-contrib", "--version" ] + interval: 10s + timeout: 5s + retries: 3 jaeger: image: jaegertracing/jaeger:latest @@ -72,12 +88,23 @@ services: - BADGER_DIRECTORY_KEY=/badger/key - COLLECTOR_OTLP_ENABLED=true volumes: + - ../../.docker/observability/jaeger.yaml:/etc/jaeger/config.yml:ro - jaeger-data:/badger ports: - "16686:16686" # Web UI - "14269:14269" # Admin/Metrics + - "4317" # otlp grpc + - "4318" # otlp http + command: [ "--config", "/etc/jaeger/config.yml" ] networks: - rustfs-network + restart: unless-stopped + healthcheck: + test: [ "CMD", "wget", "--spider", "-q", "http://localhost:14269" ] + interval: 10s + timeout: 5s + retries: 3 + start_period: 15s prometheus: image: prom/prometheus:latest @@ -85,6 +112,7 @@ services: - TZ=Asia/Shanghai volumes: - ../../.docker/observability/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ../../.docker/observability/prometheus-rules:/etc/prometheus/rules:ro - prometheus-data:/prometheus ports: - "9090:9090" @@ -94,23 +122,45 @@ services: - '--web.enable-remote-write-receiver' - '--enable-feature=promql-experimental-functions' - '--storage.tsdb.path=/prometheus' - - '--web.console.libraries=/usr/share/prometheus/console_libraries' - - '--web.console.templates=/usr/share/prometheus/consoles' + - '--storage.tsdb.retention.time=30d' networks: - rustfs-network + restart: unless-stopped + healthcheck: + test: [ "CMD", "wget", "--spider", "-q", "http://localhost:9090/-/healthy" ] + interval: 10s + timeout: 5s + retries: 3 loki: image: grafana/loki:latest environment: - TZ=Asia/Shanghai volumes: - - ../../.docker/observability/loki.yaml:/etc/loki/local-config.yaml:ro + - ../../.docker/observability/loki.yaml:/etc/loki/loki.yaml:ro - loki-data:/loki ports: - "3100:3100" - command: -config.file=/etc/loki/local-config.yaml + command: -config.file=/etc/loki/loki.yaml networks: - rustfs-network + restart: unless-stopped + healthcheck: + test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3100/ready" ] + interval: 15s + timeout: 10s + retries: 5 + start_period: 60s + + pyroscope: + image: grafana/pyroscope:latest + ports: + - "4040:4040" + command: + - -self-profiling.disable-push=true + networks: + - rustfs-network + restart: unless-stopped grafana: image: grafana/grafana:latest @@ -127,10 +177,17 @@ services: volumes: - ../../.docker/observability/grafana/provisioning:/etc/grafana/provisioning:ro - ../../.docker/observability/grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana-data:/var/lib/grafana depends_on: - prometheus - tempo - loki + restart: unless-stopped + healthcheck: + test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health" ] + interval: 10s + timeout: 5s + retries: 3 # --- RustFS Cluster --- @@ -215,6 +272,7 @@ volumes: tempo-data: loki-data: jaeger-data: + grafana-data: networks: rustfs-network: diff --git a/Cargo.lock b/Cargo.lock index 1508a3f2b..ea15d6132 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1473,12 +1473,6 @@ dependencies = [ "shlex", ] -[[package]] -name = "cesu8" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" - [[package]] name = "cfg-if" version = "1.0.4" @@ -3196,7 +3190,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3475,7 +3469,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4607,9 +4601,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" dependencies = [ "icu_normalizer", "icu_properties", @@ -4746,7 +4740,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4823,7 +4817,7 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4854,27 +4848,32 @@ dependencies = [ [[package]] name = "jni" -version = "0.21.1" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" dependencies = [ - "cesu8", "cfg-if", "combine", - "jni-sys 0.3.1", + "jni-macros", + "jni-sys", "log", - "thiserror 1.0.69", + "simd_cesu8", + "thiserror 2.0.18", "walkdir", - "windows-sys 0.45.0", + "windows-link", ] [[package]] -name = "jni-sys" -version = "0.3.1" +name = "jni-macros" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" dependencies = [ - "jni-sys 0.4.1", + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.117", ] [[package]] @@ -5793,7 +5792,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5965,7 +5964,7 @@ version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "chrono", "getrandom 0.2.17", "http 1.4.0", @@ -6600,9 +6599,9 @@ dependencies = [ [[package]] name = "pkcs8" -version = "0.11.0-rc.11" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12922b6296c06eb741b02d7b5161e3aaa22864af38dfa025a1a3ba3f68c84577" +checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" dependencies = [ "der 0.8.0", "spki 0.8.0", @@ -7181,7 +7180,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -7690,16 +7689,16 @@ dependencies = [ [[package]] name = "rsa" -version = "0.10.0-rc.17" +version = "0.10.0-rc.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ed3e93fc7e473e464b9726f4759659e72bc8665e4b8ea227547024f416d905" +checksum = "30b2aa4ba0d89f73d1e332df05be0eeab8840351c36ca5654341dfdb57bb3caf" dependencies = [ "const-oid 0.10.2", "crypto-bigint 0.7.3", "crypto-primes", "digest 0.11.2", "pkcs1 0.8.0-rc.4", - "pkcs8 0.11.0-rc.11", + "pkcs8 0.11.0", "rand_core 0.10.1", "signature 3.0.0-rc.10", "spki 0.8.0", @@ -7936,7 +7935,7 @@ version = "0.0.5" dependencies = [ "base64-simd", "rand 0.10.1", - "rsa 0.10.0-rc.17", + "rsa 0.10.0-rc.18", "serde", "serde_json", ] @@ -8569,6 +8568,8 @@ dependencies = [ "flatbuffers", "prost 0.14.3", "rustfs-common", + "rustfs-config", + "rustfs-utils", "tonic", "tonic-prost", "tonic-prost-build", @@ -8889,7 +8890,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -8932,9 +8933,9 @@ dependencies = [ [[package]] name = "rustls-platform-verifier" -version = "0.6.2" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" dependencies = [ "core-foundation 0.10.1", "core-foundation-sys", @@ -8948,7 +8949,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -9473,6 +9474,16 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "simd_cesu8" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +dependencies = [ + "rustc_version", + "simdutf8", +] + [[package]] name = "simdutf8" version = "0.1.5" @@ -9955,7 +9966,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -10945,7 +10956,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -11066,15 +11077,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - [[package]] name = "windows-sys" version = "0.52.0" @@ -11093,6 +11095,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -11102,21 +11113,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - [[package]] name = "windows-targets" version = "0.52.6" @@ -11126,13 +11122,30 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + [[package]] name = "windows-threading" version = "0.2.1" @@ -11142,12 +11155,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -11155,10 +11162,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" +name = "windows_aarch64_gnullvm" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" [[package]] name = "windows_aarch64_msvc" @@ -11167,10 +11174,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] -name = "windows_i686_gnu" -version = "0.42.2" +name = "windows_aarch64_msvc" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" [[package]] name = "windows_i686_gnu" @@ -11178,6 +11185,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" @@ -11185,10 +11198,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] -name = "windows_i686_msvc" -version = "0.42.2" +name = "windows_i686_gnullvm" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" [[package]] name = "windows_i686_msvc" @@ -11197,10 +11210,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" +name = "windows_i686_msvc" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" [[package]] name = "windows_x86_64_gnu" @@ -11209,10 +11222,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" +name = "windows_x86_64_gnu" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" [[package]] name = "windows_x86_64_gnullvm" @@ -11221,10 +11234,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" +name = "windows_x86_64_gnullvm" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" [[package]] name = "windows_x86_64_msvc" @@ -11232,6 +11245,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + [[package]] name = "wit-bindgen" version = "0.51.0" diff --git a/Cargo.toml b/Cargo.toml index 5bf58e6b3..0ee6974bf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -169,7 +169,7 @@ hmac = { version = "0.13.0" } jsonwebtoken = { version = "10.3.0", features = ["aws_lc_rs"] } openidconnect = { version = "4.0", default-features = false } pbkdf2 = "0.13.0" -rsa = { version = "0.10.0-rc.17" } +rsa = { version = "0.10.0-rc.18" } rustls = { version = "0.23.39", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] } rustls-pki-types = "1.14.1" sha1 = "0.11.0" diff --git a/Dockerfile.decommission-local b/Dockerfile.decommission-local index 6a3bcbd96..1ccf3ce70 100644 --- a/Dockerfile.decommission-local +++ b/Dockerfile.decommission-local @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM rust:1.91-trixie +FROM rust:1.93-trixie RUN set -eux; \ export DEBIAN_FRONTEND=noninteractive; \ diff --git a/Dockerfile.source b/Dockerfile.source index d2d3e6f37..4b61d5a50 100644 --- a/Dockerfile.source +++ b/Dockerfile.source @@ -30,7 +30,7 @@ ARG BUILDPLATFORM # ----------------------------- # Build stage # ----------------------------- -FROM rust:1.91-trixie AS builder +FROM rust:1.93-trixie AS builder # Re-declare args after FROM ARG TARGETPLATFORM @@ -60,7 +60,15 @@ RUN set -eux; \ # Optional: cross toolchain for aarch64 (only when targeting linux/arm64) RUN set -eux; \ - if [ "${TARGETPLATFORM:-linux/amd64}" = "linux/arm64" ]; then \ + target_platform="${TARGETPLATFORM:-}"; \ + if [ -z "${target_platform}" ]; then \ + case "$(uname -m)" in \ + x86_64) target_platform="linux/amd64" ;; \ + aarch64|arm64) target_platform="linux/arm64" ;; \ + *) target_platform="linux/amd64" ;; \ + esac; \ + fi; \ + if [ "${target_platform}" = "linux/arm64" ]; then \ export DEBIAN_FRONTEND=noninteractive; \ apt-get update; \ apt-get install -y --no-install-recommends gcc-aarch64-linux-gnu; \ @@ -120,7 +128,16 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/usr/local/cargo/git \ --mount=type=cache,target=/usr/src/rustfs/target \ set -eux; \ - case "${TARGETPLATFORM:-linux/amd64}" in \ + rustup target add x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu; \ + target_platform="${TARGETPLATFORM:-}"; \ + if [ -z "${target_platform}" ]; then \ + case "$(uname -m)" in \ + x86_64) target_platform="linux/amd64" ;; \ + aarch64|arm64) target_platform="linux/arm64" ;; \ + *) target_platform="linux/amd64" ;; \ + esac; \ + fi; \ + case "${target_platform}" in \ linux/amd64) \ echo "Building for x86_64-unknown-linux-gnu"; \ cargo build --release --locked --target x86_64-unknown-linux-gnu --bin rustfs -j "$(nproc)"; \ @@ -132,7 +149,7 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \ install -m 0755 target/aarch64-unknown-linux-gnu/release/rustfs /usr/local/bin/rustfs \ ;; \ *) \ - echo "Unsupported TARGETPLATFORM=${TARGETPLATFORM}" >&2; exit 1 \ + echo "Unsupported target platform=${target_platform}" >&2; exit 1 \ ;; \ esac @@ -178,7 +195,7 @@ CMD ["cargo", "run", "--bin", "rustfs", "--"] # ----------------------------- # Runtime stage (Ubuntu minimal) # ----------------------------- -FROM ubuntu:22.04 +FROM ubuntu:24.04 ARG BUILD_DATE ARG VCS_REF diff --git a/crates/config/src/constants/drive.rs b/crates/config/src/constants/drive.rs index b8a4c4719..f3d6dcd14 100644 --- a/crates/config/src/constants/drive.rs +++ b/crates/config/src/constants/drive.rs @@ -39,6 +39,14 @@ pub const DEFAULT_DRIVE_WALKDIR_TIMEOUT_SECS: u64 = 5; pub const ENV_DRIVE_WALKDIR_STALL_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_WALKDIR_STALL_TIMEOUT_SECS"; pub const DEFAULT_DRIVE_WALKDIR_STALL_TIMEOUT_SECS: u64 = 5; +/// Interval in seconds between active health probes for local and remote drives. +pub const ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS: &str = "RUSTFS_DRIVE_ACTIVE_CHECK_INTERVAL_SECS"; +pub const DEFAULT_DRIVE_ACTIVE_CHECK_INTERVAL_SECS: u64 = 15; + +/// Timeout in seconds for a single active health probe. +pub const ENV_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS"; +pub const DEFAULT_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS: u64 = 5; + /// Number of consecutive failures before a suspect drive is classified as offline. pub const ENV_DRIVE_SUSPECT_FAILURE_THRESHOLD: &str = "RUSTFS_DRIVE_SUSPECT_FAILURE_THRESHOLD"; pub const DEFAULT_DRIVE_SUSPECT_FAILURE_THRESHOLD: u64 = 2; diff --git a/crates/config/src/constants/internode.rs b/crates/config/src/constants/internode.rs new file mode 100644 index 000000000..aa9b861e6 --- /dev/null +++ b/crates/config/src/constants/internode.rs @@ -0,0 +1,62 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// Timeout for establishing a new internode gRPC connection. +pub const ENV_INTERNODE_CONNECT_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_CONNECT_TIMEOUT_SECS"; +pub const DEFAULT_INTERNODE_CONNECT_TIMEOUT_SECS: u64 = 3; + +/// TCP keepalive interval for internode gRPC channels. +pub const ENV_INTERNODE_TCP_KEEPALIVE_SECS: &str = "RUSTFS_INTERNODE_TCP_KEEPALIVE_SECS"; +pub const DEFAULT_INTERNODE_TCP_KEEPALIVE_SECS: u64 = 10; + +/// HTTP/2 keepalive interval for internode gRPC channels. +pub const ENV_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS: &str = "RUSTFS_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS"; +pub const DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS: u64 = 5; + +/// HTTP/2 keepalive timeout for internode gRPC channels. +pub const ENV_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS"; +pub const DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS: u64 = 3; + +/// Overall timeout for a single internode gRPC request. +pub const ENV_INTERNODE_RPC_TIMEOUT_SECS: &str = "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS"; +pub const DEFAULT_INTERNODE_RPC_TIMEOUT_SECS: u64 = 10; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn internode_timeout_defaults_stay_in_expected_bounds() { + assert_eq!(DEFAULT_INTERNODE_CONNECT_TIMEOUT_SECS, 3); + assert_eq!(DEFAULT_INTERNODE_TCP_KEEPALIVE_SECS, 10); + assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS, 5); + assert_eq!(DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS, 3); + assert_eq!(DEFAULT_INTERNODE_RPC_TIMEOUT_SECS, 10); + } + + #[test] + fn internode_timeout_env_names_are_stable() { + assert_eq!(ENV_INTERNODE_CONNECT_TIMEOUT_SECS, "RUSTFS_INTERNODE_CONNECT_TIMEOUT_SECS"); + assert_eq!(ENV_INTERNODE_TCP_KEEPALIVE_SECS, "RUSTFS_INTERNODE_TCP_KEEPALIVE_SECS"); + assert_eq!( + ENV_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS, + "RUSTFS_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS" + ); + assert_eq!( + ENV_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS, + "RUSTFS_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS" + ); + assert_eq!(ENV_INTERNODE_RPC_TIMEOUT_SECS, "RUSTFS_INTERNODE_RPC_TIMEOUT_SECS"); + } +} diff --git a/crates/config/src/constants/mod.rs b/crates/config/src/constants/mod.rs index 6ca7ac738..a7eb8aa1e 100644 --- a/crates/config/src/constants/mod.rs +++ b/crates/config/src/constants/mod.rs @@ -21,6 +21,7 @@ pub(crate) mod drive; pub(crate) mod env; pub(crate) mod heal; pub(crate) mod health; +pub(crate) mod internode; pub(crate) mod object; pub(crate) mod oidc; pub(crate) mod profiler; diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index 871f621be..ed7c1b203 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -33,6 +33,8 @@ pub use constants::heal::*; #[cfg(feature = "constants")] pub use constants::health::*; #[cfg(feature = "constants")] +pub use constants::internode::*; +#[cfg(feature = "constants")] pub use constants::object::*; #[cfg(feature = "constants")] pub use constants::profiler::*; diff --git a/crates/ecstore/src/batch_processor.rs b/crates/ecstore/src/batch_processor.rs index c3c3ee7ad..7d2cc11d0 100644 --- a/crates/ecstore/src/batch_processor.rs +++ b/crates/ecstore/src/batch_processor.rs @@ -87,25 +87,70 @@ impl AsyncBatchProcessor { T: Send + 'static, F: Future> + Send + 'static, { - let results = self.execute_batch(tasks).await; - let mut successes = Vec::new(); + if required_successes == 0 { + return Ok(Vec::new()); + } - for value in results.into_iter().flatten() { - successes.push(value); - if successes.len() >= required_successes { - return Ok(successes); + if tasks.is_empty() { + return Err(Error::other(format!( + "Insufficient successful results: got 0, needed {required_successes}" + ))); + } + + let semaphore = Arc::new(tokio::sync::Semaphore::new(self.max_concurrent)); + let mut join_set = JoinSet::new(); + let mut successes = Vec::new(); + let mut pending_tasks = tasks.len(); + let mut first_error = None; + + for task in tasks { + let sem = semaphore.clone(); + join_set.spawn(async move { + let _permit = sem.acquire().await.map_err(|_| Error::other("Semaphore error"))?; + task.await + }); + } + + while let Some(join_result) = join_set.join_next().await { + pending_tasks = pending_tasks.saturating_sub(1); + + match join_result { + Ok(Ok(value)) => { + successes.push(value); + if successes.len() >= required_successes { + return Ok(successes); + } + } + Ok(Err(err)) => { + if first_error.is_none() { + first_error = Some(err); + } + } + Err(join_error) => { + if first_error.is_none() { + first_error = Some(Error::other(format!("Task panicked in quorum batch processor: {join_error}"))); + } + } + } + + if successes.len() + pending_tasks < required_successes { + return Err(first_error.unwrap_or_else(|| { + Error::other(format!( + "Insufficient successful results: got {}, needed {}", + successes.len(), + required_successes + )) + })); } } - if successes.len() >= required_successes { - Ok(successes) - } else { - Err(Error::other(format!( + Err(first_error.unwrap_or_else(|| { + Error::other(format!( "Insufficient successful results: got {}, needed {}", successes.len(), required_successes - ))) - } + )) + })) } } @@ -228,4 +273,52 @@ mod tests { let successes = results.unwrap(); assert!(successes.len() >= 2); } + + #[tokio::test] + async fn test_batch_processor_quorum_returns_before_slow_tail() { + let processor = AsyncBatchProcessor::new(4); + let started = std::time::Instant::now(); + + let tasks: Vec<_> = [(10_u64, Ok(1_i32)), (15, Ok(2)), (250, Ok(3))] + .into_iter() + .map(|(delay_ms, outcome)| async move { + tokio::time::sleep(Duration::from_millis(delay_ms)).await; + outcome + }) + .collect(); + + let results = processor + .execute_batch_with_quorum(tasks, 2) + .await + .expect("quorum should succeed"); + assert_eq!(results.len(), 2); + assert!(started.elapsed() < Duration::from_millis(100)); + } + + #[tokio::test] + async fn test_batch_processor_quorum_fails_once_quorum_becomes_impossible() { + let processor = AsyncBatchProcessor::new(4); + let started = std::time::Instant::now(); + + let tasks: Vec<_> = vec![ + (10_u64, Ok(1_i32)), + (15, Err(Error::other("first failure"))), + (20, Err(Error::other("second failure"))), + (250, Ok(4)), + ] + .into_iter() + .map(|(delay_ms, outcome)| async move { + tokio::time::sleep(Duration::from_millis(delay_ms)).await; + outcome + }) + .collect(); + + let err = processor + .execute_batch_with_quorum(tasks, 3) + .await + .expect_err("quorum should fail once it becomes impossible"); + + assert!(err.to_string().contains("first failure")); + assert!(started.elapsed() < Duration::from_millis(120)); + } } diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 39e94c64b..0d5666d8d 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -54,8 +54,8 @@ use rustfs_filemeta::{ use rustfs_s3_common::EventName; use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::strings_has_prefix_fold}; use s3s::dto::{ - BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration, RestoreRequest, RestoreRequestType, RestoreStatus, - Timestamp, + BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, ReplicationConfiguration, RestoreRequest, + RestoreRequestType, RestoreStatus, Timestamp, }; use s3s::header::{X_AMZ_RESTORE, X_AMZ_SERVER_SIDE_ENCRYPTION}; use sha2::{Digest, Sha256}; @@ -92,6 +92,7 @@ const ENV_STALE_UPLOADS_EXPIRY: &str = "RUSTFS_API_STALE_UPLOADS_EXPIRY"; const ENV_STALE_UPLOADS_CLEANUP_INTERVAL: &str = "RUSTFS_API_STALE_UPLOADS_CLEANUP_INTERVAL"; const DEFAULT_STALE_UPLOADS_EXPIRY: StdDuration = StdDuration::from_secs(24 * 60 * 60); const DEFAULT_STALE_UPLOADS_CLEANUP_INTERVAL: StdDuration = StdDuration::from_secs(6 * 60 * 60); +const DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS: i64 = 5; lazy_static! { pub static ref GLOBAL_ExpiryState: Arc> = ExpiryState::new(); @@ -1241,6 +1242,29 @@ pub async fn enqueue_transition_for_existing_objects(api: Arc, bucket: } } +fn lifecycle_rule_has_date_expiration(lc: &BucketLifecycleConfiguration, rule_id: &str) -> bool { + lc.rules.iter().any(|rule| { + rule.status == ExpirationStatus::from_static(ExpirationStatus::ENABLED) + && rule.id.as_deref() == Some(rule_id) + && rule.expiration.as_ref().is_some_and(|expiration| expiration.date.is_some()) + }) +} + +fn should_defer_date_expiry_for_recent_config_update(lc: &BucketLifecycleConfiguration, now: OffsetDateTime) -> bool { + lc.expiry_updated_at.as_ref().is_some_and(|updated_at| { + let updated_at = OffsetDateTime::from(updated_at.clone()); + now.unix_timestamp().saturating_sub(updated_at.unix_timestamp()) < DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS + }) +} + +async fn apply_existing_object_expiry(api: Arc, object: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) { + if object.is_remote() { + apply_expiry_on_transitioned_object(api, object, event, src).await; + } else { + apply_expiry_on_non_transitioned_objects(api, object, event, src).await; + } +} + pub async fn enqueue_expiry_for_existing_objects(api: Arc, bucket: &str) -> Result<(), Error> { let Ok((lc, _)) = metadata_sys::get_lifecycle_config(bucket).await else { return Ok(()); @@ -1253,6 +1277,8 @@ pub async fn enqueue_expiry_for_existing_objects(api: Arc, bucket: &str let mut marker = None; let mut version_marker = None; let src = LcEventSrc::Scanner; + let defer_date_expiry_once = should_defer_date_expiry_for_recent_config_update(&lc, OffsetDateTime::now_utc()); + let mut date_expiry_deferred_once = false; loop { let page = api @@ -1269,15 +1295,16 @@ pub async fn enqueue_expiry_for_existing_objects(api: Arc, bucket: &str | IlmAction::DeleteRestoredVersionAction | IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction => { - if event - .due - .is_some_and(|due| due.unix_timestamp() <= OffsetDateTime::now_utc().unix_timestamp()) - { - if object.is_remote() { - apply_expiry_on_transitioned_object(api.clone(), object, &event, &src).await; - } else { - apply_expiry_on_non_transitioned_objects(api.clone(), object, &event, &src).await; + let now = OffsetDateTime::now_utc(); + if event.due.is_some_and(|due| due.unix_timestamp() <= now.unix_timestamp()) { + if defer_date_expiry_once + && !date_expiry_deferred_once + && lifecycle_rule_has_date_expiration(&lc, &event.rule_id) + { + tokio::time::sleep(StdDuration::from_secs(DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS as u64)).await; + date_expiry_deferred_once = true; } + apply_existing_object_expiry(api.clone(), object, &event, &src).await; } else { apply_expiry_rule(&event, &src, object).await; } @@ -1977,9 +2004,10 @@ pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc, #[cfg(test)] mod tests { use super::{ - StaleMultipartUploadCandidate, cleanup_empty_multipart_sha_dirs_on_local_disks, cleanup_stale_multipart_uploads_once_at, - lifecycle_deleted_object, lifecycle_version_purge_state_from_completed_targets, - mark_delete_opts_skip_decommissioned_on_remote_success, merge_stale_multipart_candidate, replication_state_for_delete, + DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS, StaleMultipartUploadCandidate, cleanup_empty_multipart_sha_dirs_on_local_disks, + cleanup_stale_multipart_uploads_once_at, lifecycle_deleted_object, lifecycle_rule_has_date_expiration, + lifecycle_version_purge_state_from_completed_targets, mark_delete_opts_skip_decommissioned_on_remote_success, + merge_stale_multipart_candidate, replication_state_for_delete, should_defer_date_expiry_for_recent_config_update, should_reuse_lifecycle_delete_replication_state, }; use crate::bucket::metadata::BUCKET_LIFECYCLE_CONFIG; @@ -1994,6 +2022,7 @@ mod tests { BucketOperations, BucketOptions, MakeBucketOptions, MultipartOperations, ObjectInfo, ObjectOptions, PutObjReader, }; use rustfs_filemeta::{ReplicateDecision, VersionPurgeStatusType}; + use s3s::dto::{BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, Timestamp}; use serial_test::serial; use sha2::{Digest, Sha256}; use std::collections::HashMap; @@ -2014,6 +2043,49 @@ mod tests { assert!(opts.skip_decommissioned); } + #[test] + fn lifecycle_rule_has_date_expiration_detects_enabled_date_rule() { + let lc = BucketLifecycleConfiguration { + expiry_updated_at: None, + rules: vec![LifecycleRule { + status: ExpirationStatus::from_static(ExpirationStatus::ENABLED), + expiration: Some(LifecycleExpiration { + date: Some(Timestamp::from(OffsetDateTime::now_utc())), + ..Default::default() + }), + id: Some("rule-date".to_string()), + abort_incomplete_multipart_upload: None, + del_marker_expiration: None, + filter: None, + noncurrent_version_expiration: None, + noncurrent_version_transitions: None, + prefix: None, + transitions: None, + }], + }; + + assert!(lifecycle_rule_has_date_expiration(&lc, "rule-date")); + assert!(!lifecycle_rule_has_date_expiration(&lc, "missing-rule")); + } + + #[test] + fn should_defer_date_expiry_for_recent_config_update_respects_grace_window() { + let now = OffsetDateTime::now_utc(); + let recent = BucketLifecycleConfiguration { + expiry_updated_at: Some(Timestamp::from(now - time::Duration::seconds(1))), + rules: Vec::new(), + }; + let stale = BucketLifecycleConfiguration { + expiry_updated_at: Some(Timestamp::from( + now - time::Duration::seconds(DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS + 1), + )), + rules: Vec::new(), + }; + + assert!(should_defer_date_expiry_for_recent_config_update(&recent, now)); + assert!(!should_defer_date_expiry_for_recent_config_update(&stale, now)); + } + #[test] fn mark_delete_opts_skip_decommissioned_on_remote_success_preserves_false_on_failure() { let mut opts = ObjectOptions::default(); diff --git a/crates/ecstore/src/disk/disk_store.rs b/crates/ecstore/src/disk/disk_store.rs index 1045163e5..fe02dd811 100644 --- a/crates/ecstore/src/disk/disk_store.rs +++ b/crates/ecstore/src/disk/disk_store.rs @@ -46,9 +46,7 @@ const DISK_HEALTH_FAULTY: u32 = 1; pub const ENV_RUSTFS_DRIVE_ACTIVE_MONITORING: &str = "RUSTFS_DRIVE_ACTIVE_MONITORING"; pub const DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING: bool = true; -pub const CHECK_EVERY: Duration = Duration::from_secs(15); pub const SKIP_IF_SUCCESS_BEFORE: Duration = Duration::from_secs(5); -pub const CHECK_TIMEOUT_DURATION: Duration = Duration::from_secs(5); lazy_static::lazy_static! { static ref TEST_DATA: Bytes = Bytes::from(vec![42u8; 2048]); @@ -104,6 +102,20 @@ pub fn get_drive_walkdir_stall_timeout() -> Duration { ) } +pub fn get_drive_active_check_interval() -> Duration { + Duration::from_secs(rustfs_utils::get_env_u64( + rustfs_config::ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, + rustfs_config::DEFAULT_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, + )) +} + +pub fn get_drive_active_check_timeout() -> Duration { + Duration::from_secs(rustfs_utils::get_env_u64( + rustfs_config::ENV_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS, + rustfs_config::DEFAULT_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS, + )) +} + /// DiskHealthTracker tracks the health status of a disk. /// Similar to Go's diskHealthTracker. #[derive(Debug)] @@ -474,7 +486,7 @@ impl LocalDiskWrapper { async fn monitor_disk_writable(disk: Arc, health: Arc, cancel_token: CancellationToken) { // TODO: config interval - let mut interval = time::interval(CHECK_EVERY); + let mut interval = time::interval(get_drive_active_check_interval()); loop { tokio::select! { @@ -507,7 +519,16 @@ impl LocalDiskWrapper { let test_obj = format!("health-check-{}", Uuid::new_v4()); - if Self::perform_health_check(disk.clone(), &TEST_BUCKET, &test_obj, &TEST_DATA, true, CHECK_TIMEOUT_DURATION).await.is_err() + if Self::perform_health_check( + disk.clone(), + &TEST_BUCKET, + &test_obj, + &TEST_DATA, + true, + get_drive_active_check_timeout(), + ) + .await + .is_err() && health.mark_failure(&disk.endpoint(), "active_health_check_failed") { // Health check failed, disk is considered faulty @@ -613,7 +634,16 @@ impl LocalDiskWrapper { } let test_obj = format!("health-check-{}", Uuid::new_v4()); - match Self::perform_health_check(disk.clone(), &TEST_BUCKET, &test_obj, &TEST_DATA, false, CHECK_TIMEOUT_DURATION).await { + match Self::perform_health_check( + disk.clone(), + &TEST_BUCKET, + &test_obj, + &TEST_DATA, + false, + get_drive_active_check_timeout(), + ) + .await + { Ok(_) => { let state_before = health.runtime_state(); let is_online = health.mark_recovery_success(&disk.endpoint(), "recovery_probe_success"); @@ -1137,6 +1167,40 @@ mod tests { }); } + #[test] + fn drive_active_check_interval_uses_default_when_unset() { + temp_env::with_var_unset(rustfs_config::ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, || { + assert_eq!( + get_drive_active_check_interval(), + Duration::from_secs(rustfs_config::DEFAULT_DRIVE_ACTIVE_CHECK_INTERVAL_SECS) + ); + }); + } + + #[test] + fn drive_active_check_interval_reads_env_override() { + temp_env::with_var(rustfs_config::ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, Some("3"), || { + assert_eq!(get_drive_active_check_interval(), Duration::from_secs(3)); + }); + } + + #[test] + fn drive_active_check_timeout_uses_default_when_unset() { + temp_env::with_var_unset(rustfs_config::ENV_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS, || { + assert_eq!( + get_drive_active_check_timeout(), + Duration::from_secs(rustfs_config::DEFAULT_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS) + ); + }); + } + + #[test] + fn drive_active_check_timeout_reads_env_override() { + temp_env::with_var(rustfs_config::ENV_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS, Some("1"), || { + assert_eq!(get_drive_active_check_timeout(), Duration::from_secs(1)); + }); + } + #[test] fn runtime_state_transitions_from_online_to_suspect_then_offline() { temp_env::with_var(rustfs_config::ENV_DRIVE_SUSPECT_FAILURE_THRESHOLD, Some("2"), || { diff --git a/crates/ecstore/src/rpc/client.rs b/crates/ecstore/src/rpc/client.rs index 6314c6571..43a1c37f0 100644 --- a/crates/ecstore/src/rpc/client.rs +++ b/crates/ecstore/src/rpc/client.rs @@ -12,11 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::disk::error::{DiskError, Error as DiskErrorType}; use crate::rpc::{TONIC_RPC_PREFIX, gen_signature_headers}; use http::Method; use rustfs_common::GLOBAL_CONN_MAP; use rustfs_protos::{create_new_channel, proto_gen::node_service::node_service_client::NodeServiceClient}; -use std::error::Error; +use std::{error::Error, io::ErrorKind}; use tonic::{service::interceptor::InterceptedService, transport::Channel}; use tracing::debug; @@ -51,6 +52,46 @@ pub async fn node_service_time_out_client_no_auth( node_service_time_out_client(addr, TonicInterceptor::NoOp(NoOpInterceptor)).await } +pub(crate) fn is_network_like_disk_error(err: &DiskErrorType) -> bool { + match err { + DiskError::Timeout => true, + DiskError::Io(io_err) => { + if matches!( + io_err.kind(), + ErrorKind::TimedOut + | ErrorKind::ConnectionRefused + | ErrorKind::ConnectionReset + | ErrorKind::BrokenPipe + | ErrorKind::NotConnected + | ErrorKind::ConnectionAborted + | ErrorKind::UnexpectedEof + ) { + return true; + } + + let message = io_err.to_string().to_ascii_lowercase(); + [ + "transport error", + "unavailable", + "error trying to connect", + "connection refused", + "connection reset", + "broken pipe", + "not connected", + "unexpected eof", + "timed out", + "deadline has elapsed", + "connection closed", + "connection aborted", + "tcp connect error", + ] + .iter() + .any(|needle| message.contains(needle)) + } + _ => false, + } +} + pub struct TonicSignatureInterceptor; impl tonic::service::Interceptor for TonicSignatureInterceptor { diff --git a/crates/ecstore/src/rpc/peer_rest_client.rs b/crates/ecstore/src/rpc/peer_rest_client.rs index 18a66d42e..46dae7005 100644 --- a/crates/ecstore/src/rpc/peer_rest_client.rs +++ b/crates/ecstore/src/rpc/peer_rest_client.rs @@ -15,6 +15,7 @@ use crate::error::{Error, Result}; use crate::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client}; use crate::{ + disk::disk_store::{get_drive_active_check_interval, get_drive_active_check_timeout}, endpoints::EndpointServerPools, global::is_dist_erasure, metrics_realtime::{CollectMetricsOpts, MetricType}, @@ -38,7 +39,16 @@ use rustfs_protos::proto_gen::node_service::{ }; use rustfs_utils::XHost; use serde::{Deserialize, Serialize as _}; -use std::{collections::HashMap, io::Cursor, time::SystemTime}; +use std::{ + collections::HashMap, + io::Cursor, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::SystemTime, +}; +use tokio::{net::TcpStream, time::Duration}; use tonic::Request; use tonic::service::interceptor::InterceptedService; use tonic::transport::Channel; @@ -47,6 +57,8 @@ use tracing::warn; pub const PEER_RESTSIGNAL: &str = "signal"; pub const PEER_RESTSUB_SYS: &str = "sub-sys"; pub const PEER_RESTDRY_RUN: &str = "dry-run"; +const PEER_REST_RECOVERY_MAX_ATTEMPTS: u32 = 60; +const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30); #[derive(Clone, Debug)] pub struct PeerLiveEventsBatch { @@ -59,11 +71,18 @@ pub struct PeerLiveEventsBatch { pub struct PeerRestClient { pub host: XHost, pub grid_host: String, + offline: Arc, + recovery_running: Arc, } impl PeerRestClient { pub fn new(host: XHost, grid_host: String) -> Self { - Self { host, grid_host } + Self { + host, + grid_host, + offline: Arc::new(AtomicBool::new(false)), + recovery_running: Arc::new(AtomicBool::new(false)), + } } pub async fn new_clients(eps: EndpointServerPools) -> (Vec>, Vec>) { if !is_dist_erasure().await { @@ -93,9 +112,20 @@ impl PeerRestClient { } pub async fn get_client(&self) -> Result>> { + if self.offline.load(Ordering::Acquire) { + self.mark_offline_and_spawn_recovery(); + return Err(Error::other(format!("peer {} is temporarily offline", self.grid_host))); + } + node_service_time_out_client(&self.grid_host, TonicInterceptor::Signature(gen_tonic_signature_interceptor())) .await - .map_err(|err| Error::other(format!("can not get client, err: {err}"))) + .map_err(|err| { + let storage_err = Error::other(format!("can not get client, err: {err}")); + if Self::is_network_like_error(&storage_err) { + self.mark_offline_and_spawn_recovery(); + } + storage_err + }) } /// Evict the connection to this peer from the global cache. @@ -103,16 +133,98 @@ impl PeerRestClient { pub async fn evict_connection(&self) { evict_failed_connection(&self.grid_host).await; } + + fn is_network_like_error(err: &Error) -> bool { + let message = err.to_string().to_ascii_lowercase(); + [ + "temporarily offline", + "transport error", + "unavailable", + "error trying to connect", + "connection refused", + "connection reset", + "broken pipe", + "not connected", + "unexpected eof", + "timed out", + "deadline has elapsed", + "connection closed", + "connection aborted", + "tcp connect error", + ] + .iter() + .any(|needle| message.contains(needle)) + } + + fn mark_offline_and_spawn_recovery(&self) { + self.offline.store(true, Ordering::Release); + + if self + .recovery_running + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) + .is_err() + { + return; + } + + let grid_host = self.grid_host.clone(); + let offline = Arc::clone(&self.offline); + let recovery_running = Arc::clone(&self.recovery_running); + tokio::spawn(async move { + let mut delay = get_drive_active_check_interval(); + let connect_timeout = get_drive_active_check_timeout(); + + for _ in 0..PEER_REST_RECOVERY_MAX_ATTEMPTS { + tokio::time::sleep(delay).await; + if Self::perform_connectivity_check(&grid_host, connect_timeout).await.is_ok() { + offline.store(false, Ordering::Release); + recovery_running.store(false, Ordering::Release); + return; + } + + delay = std::cmp::min(delay.saturating_mul(2), PEER_REST_RECOVERY_MAX_BACKOFF); + } + + warn!( + grid_host = %grid_host, + attempts = PEER_REST_RECOVERY_MAX_ATTEMPTS, + "peer recovery monitor reached max attempts; will retry on next request" + ); + recovery_running.store(false, Ordering::Release); + }); + } + + async fn perform_connectivity_check(addr: &str, timeout_duration: Duration) -> Result<()> { + let url = url::Url::parse(addr).map_err(|e| Error::other(format!("Invalid URL: {e}")))?; + let Some(host) = url.host_str() else { + return Err(Error::other("No host in URL".to_string())); + }; + + let port = url.port_or_known_default().unwrap_or(80); + match tokio::time::timeout(timeout_duration, TcpStream::connect((host, port))).await { + Ok(Ok(stream)) => { + drop(stream); + Ok(()) + } + _ => Err(Error::other(format!("Cannot connect to {host}:{port}"))), + } + } + + async fn finalize_result(&self, result: Result) -> Result { + if let Err(err) = &result + && Self::is_network_like_error(err) + { + self.mark_offline_and_spawn_recovery(); + self.evict_connection().await; + } + + result + } } impl PeerRestClient { pub async fn local_storage_info(&self) -> Result { - let result = self.local_storage_info_inner().await; - if result.is_err() { - // Evict stale connection on any error for cluster recovery - self.evict_connection().await; - } - result + self.finalize_result(self.local_storage_info_inner().await).await } async fn local_storage_info_inner(&self) -> Result { @@ -135,12 +247,7 @@ impl PeerRestClient { } pub async fn server_info(&self) -> Result { - let result = self.server_info_inner().await; - if result.is_err() { - // Evict stale connection on any error for cluster recovery - self.evict_connection().await; - } - result + self.finalize_result(self.server_info_inner().await).await } async fn server_info_inner(&self) -> Result { @@ -163,6 +270,10 @@ impl PeerRestClient { } pub async fn get_cpus(&self) -> Result { + self.finalize_result(self.get_cpus_inner().await).await + } + + async fn get_cpus_inner(&self) -> Result { let mut client = self.get_client().await?; let request = Request::new(GetCpusRequest {}); @@ -182,6 +293,10 @@ impl PeerRestClient { } pub async fn get_net_info(&self) -> Result { + self.finalize_result(self.get_net_info_inner().await).await + } + + async fn get_net_info_inner(&self) -> Result { let mut client = self.get_client().await?; let request = Request::new(GetNetInfoRequest {}); @@ -201,6 +316,10 @@ impl PeerRestClient { } pub async fn get_partitions(&self) -> Result { + self.finalize_result(self.get_partitions_inner().await).await + } + + async fn get_partitions_inner(&self) -> Result { let mut client = self.get_client().await?; let request = Request::new(GetPartitionsRequest {}); @@ -220,6 +339,10 @@ impl PeerRestClient { } pub async fn get_os_info(&self) -> Result { + self.finalize_result(self.get_os_info_inner().await).await + } + + async fn get_os_info_inner(&self) -> Result { let mut client = self.get_client().await?; let request = Request::new(GetOsInfoRequest {}); @@ -239,159 +362,207 @@ impl PeerRestClient { } pub async fn get_se_linux_info(&self) -> Result { - let mut client = self.get_client().await?; - let request = Request::new(GetSeLinuxInfoRequest {}); + self.finalize_result( + async { + let mut client = self.get_client().await?; + let request = Request::new(GetSeLinuxInfoRequest {}); - let response = client.get_se_linux_info(request).await?.into_inner(); - if !response.success { - if let Some(msg) = response.error_info { - return Err(Error::other(msg)); + let response = client.get_se_linux_info(request).await?.into_inner(); + if !response.success { + if let Some(msg) = response.error_info { + return Err(Error::other(msg)); + } + return Err(Error::other("")); + } + let data = response.sys_services; + + let mut buf = Deserializer::new(Cursor::new(data)); + let sys_services: SysService = Deserialize::deserialize(&mut buf)?; + + Ok(sys_services) } - return Err(Error::other("")); - } - let data = response.sys_services; - - let mut buf = Deserializer::new(Cursor::new(data)); - let sys_services: SysService = Deserialize::deserialize(&mut buf)?; - - Ok(sys_services) + .await, + ) + .await } pub async fn get_sys_config(&self) -> Result { - let mut client = self.get_client().await?; - let request = Request::new(GetSysConfigRequest {}); + self.finalize_result( + async { + let mut client = self.get_client().await?; + let request = Request::new(GetSysConfigRequest {}); - let response = client.get_sys_config(request).await?.into_inner(); - if !response.success { - if let Some(msg) = response.error_info { - return Err(Error::other(msg)); + let response = client.get_sys_config(request).await?.into_inner(); + if !response.success { + if let Some(msg) = response.error_info { + return Err(Error::other(msg)); + } + return Err(Error::other("")); + } + let data = response.sys_config; + + let mut buf = Deserializer::new(Cursor::new(data)); + let sys_config: SysConfig = Deserialize::deserialize(&mut buf)?; + + Ok(sys_config) } - return Err(Error::other("")); - } - let data = response.sys_config; - - let mut buf = Deserializer::new(Cursor::new(data)); - let sys_config: SysConfig = Deserialize::deserialize(&mut buf)?; - - Ok(sys_config) + .await, + ) + .await } pub async fn get_sys_errors(&self) -> Result { - let mut client = self.get_client().await?; - let request = Request::new(GetSysErrorsRequest {}); + self.finalize_result( + async { + let mut client = self.get_client().await?; + let request = Request::new(GetSysErrorsRequest {}); - let response = client.get_sys_errors(request).await?.into_inner(); - if !response.success { - if let Some(msg) = response.error_info { - return Err(Error::other(msg)); + let response = client.get_sys_errors(request).await?.into_inner(); + if !response.success { + if let Some(msg) = response.error_info { + return Err(Error::other(msg)); + } + return Err(Error::other("")); + } + let data = response.sys_errors; + + let mut buf = Deserializer::new(Cursor::new(data)); + let sys_errors: SysErrors = Deserialize::deserialize(&mut buf)?; + + Ok(sys_errors) } - return Err(Error::other("")); - } - let data = response.sys_errors; - - let mut buf = Deserializer::new(Cursor::new(data)); - let sys_errors: SysErrors = Deserialize::deserialize(&mut buf)?; - - Ok(sys_errors) + .await, + ) + .await } pub async fn get_mem_info(&self) -> Result { - let mut client = self.get_client().await?; - let request = Request::new(GetMemInfoRequest {}); + self.finalize_result( + async { + let mut client = self.get_client().await?; + let request = Request::new(GetMemInfoRequest {}); - let response = client.get_mem_info(request).await?.into_inner(); - if !response.success { - if let Some(msg) = response.error_info { - return Err(Error::other(msg)); + let response = client.get_mem_info(request).await?.into_inner(); + if !response.success { + if let Some(msg) = response.error_info { + return Err(Error::other(msg)); + } + return Err(Error::other("")); + } + let data = response.mem_info; + + let mut buf = Deserializer::new(Cursor::new(data)); + let mem_info: MemInfo = Deserialize::deserialize(&mut buf)?; + + Ok(mem_info) } - return Err(Error::other("")); - } - let data = response.mem_info; - - let mut buf = Deserializer::new(Cursor::new(data)); - let mem_info: MemInfo = Deserialize::deserialize(&mut buf)?; - - Ok(mem_info) + .await, + ) + .await } pub async fn get_metrics(&self, t: MetricType, opts: &CollectMetricsOpts) -> Result { - let mut client = self.get_client().await?; - let mut buf_t = Vec::new(); - t.serialize(&mut Serializer::new(&mut buf_t))?; - let mut buf_o = Vec::new(); - opts.serialize(&mut Serializer::new(&mut buf_o))?; - let request = Request::new(GetMetricsRequest { - metric_type: buf_t.into(), - opts: buf_o.into(), - }); + self.finalize_result( + async { + let mut client = self.get_client().await?; + let mut buf_t = Vec::new(); + t.serialize(&mut Serializer::new(&mut buf_t))?; + let mut buf_o = Vec::new(); + opts.serialize(&mut Serializer::new(&mut buf_o))?; + let request = Request::new(GetMetricsRequest { + metric_type: buf_t.into(), + opts: buf_o.into(), + }); - let response = client.get_metrics(request).await?.into_inner(); - if !response.success { - if let Some(msg) = response.error_info { - return Err(Error::other(msg)); + let response = client.get_metrics(request).await?.into_inner(); + if !response.success { + if let Some(msg) = response.error_info { + return Err(Error::other(msg)); + } + return Err(Error::other("")); + } + let data = response.realtime_metrics; + + let mut buf = Deserializer::new(Cursor::new(data)); + let realtime_metrics: RealtimeMetrics = Deserialize::deserialize(&mut buf)?; + + Ok(realtime_metrics) } - return Err(Error::other("")); - } - let data = response.realtime_metrics; - - let mut buf = Deserializer::new(Cursor::new(data)); - let realtime_metrics: RealtimeMetrics = Deserialize::deserialize(&mut buf)?; - - Ok(realtime_metrics) + .await, + ) + .await } pub async fn get_live_events(&self, after_sequence: u64, limit: u32) -> Result { - let mut client = self.get_client().await?; - let request = Request::new(GetLiveEventsRequest { after_sequence, limit }); + self.finalize_result( + async { + let mut client = self.get_client().await?; + let request = Request::new(GetLiveEventsRequest { after_sequence, limit }); - let response = client.get_live_events(request).await?.into_inner(); - if !response.success { - if let Some(msg) = response.error_info { - return Err(Error::other(msg)); + let response = client.get_live_events(request).await?.into_inner(); + if !response.success { + if let Some(msg) = response.error_info { + return Err(Error::other(msg)); + } + return Err(Error::other("")); + } + + Ok(PeerLiveEventsBatch { + events: response.events.to_vec(), + next_sequence: response.next_sequence, + truncated: response.truncated, + }) } - return Err(Error::other("")); - } - - Ok(PeerLiveEventsBatch { - events: response.events.to_vec(), - next_sequence: response.next_sequence, - truncated: response.truncated, - }) + .await, + ) + .await } pub async fn get_proc_info(&self) -> Result { - let mut client = self.get_client().await?; - let request = Request::new(GetProcInfoRequest {}); + self.finalize_result( + async { + let mut client = self.get_client().await?; + let request = Request::new(GetProcInfoRequest {}); - let response = client.get_proc_info(request).await?.into_inner(); - if !response.success { - if let Some(msg) = response.error_info { - return Err(Error::other(msg)); + let response = client.get_proc_info(request).await?.into_inner(); + if !response.success { + if let Some(msg) = response.error_info { + return Err(Error::other(msg)); + } + return Err(Error::other("")); + } + let data = response.proc_info; + + let mut buf = Deserializer::new(Cursor::new(data)); + let proc_info: ProcInfo = Deserialize::deserialize(&mut buf)?; + + Ok(proc_info) } - return Err(Error::other("")); - } - let data = response.proc_info; - - let mut buf = Deserializer::new(Cursor::new(data)); - let proc_info: ProcInfo = Deserialize::deserialize(&mut buf)?; - - Ok(proc_info) + .await, + ) + .await } pub async fn start_profiling(&self, profiler: &str) -> Result<()> { - let mut client = self.get_client().await?; - let request = Request::new(StartProfilingRequest { - profiler: profiler.to_string(), - }); + self.finalize_result( + async { + let mut client = self.get_client().await?; + let request = Request::new(StartProfilingRequest { + profiler: profiler.to_string(), + }); - let response = client.start_profiling(request).await?.into_inner(); - if !response.success { - if let Some(msg) = response.error_info { - return Err(Error::other(msg)); + let response = client.start_profiling(request).await?.into_inner(); + if !response.success { + if let Some(msg) = response.error_info { + return Err(Error::other(msg)); + } + return Err(Error::other("")); + } + Ok(()) } - return Err(Error::other("")); - } - Ok(()) + .await, + ) + .await } pub async fn download_profile_data(&self) -> Result<()> { @@ -415,220 +586,272 @@ impl PeerRestClient { } pub async fn load_bucket_metadata(&self, bucket: &str) -> Result<()> { - let mut client = self.get_client().await?; - let request = Request::new(LoadBucketMetadataRequest { - bucket: bucket.to_string(), - }); + self.finalize_result( + async { + let mut client = self.get_client().await?; + let request = Request::new(LoadBucketMetadataRequest { + bucket: bucket.to_string(), + }); - let response = client.load_bucket_metadata(request).await?.into_inner(); - if !response.success { - if let Some(msg) = response.error_info { - return Err(Error::other(msg)); + let response = client.load_bucket_metadata(request).await?.into_inner(); + if !response.success { + if let Some(msg) = response.error_info { + return Err(Error::other(msg)); + } + return Err(Error::other("")); + } + Ok(()) } - return Err(Error::other("")); - } - Ok(()) + .await, + ) + .await } pub async fn delete_bucket_metadata(&self, bucket: &str) -> Result<()> { - let mut client = self.get_client().await?; - let request = Request::new(DeleteBucketMetadataRequest { - bucket: bucket.to_string(), - }); + self.finalize_result( + async { + let mut client = self.get_client().await?; + let request = Request::new(DeleteBucketMetadataRequest { + bucket: bucket.to_string(), + }); - let response = client.delete_bucket_metadata(request).await?.into_inner(); - if !response.success { - if let Some(msg) = response.error_info { - return Err(Error::other(msg)); + let response = client.delete_bucket_metadata(request).await?.into_inner(); + if !response.success { + if let Some(msg) = response.error_info { + return Err(Error::other(msg)); + } + return Err(Error::other("")); + } + Ok(()) } - return Err(Error::other("")); - } - Ok(()) + .await, + ) + .await } pub async fn delete_policy(&self, policy: &str) -> Result<()> { - let mut client = self.get_client().await?; - let request = Request::new(DeletePolicyRequest { - policy_name: policy.to_string(), - }); + self.finalize_result( + async { + let mut client = self.get_client().await?; + let request = Request::new(DeletePolicyRequest { + policy_name: policy.to_string(), + }); - let response = client.delete_policy(request).await?.into_inner(); - if !response.success { - if let Some(msg) = response.error_info { - return Err(Error::other(msg)); + let response = client.delete_policy(request).await?.into_inner(); + if !response.success { + if let Some(msg) = response.error_info { + return Err(Error::other(msg)); + } + return Err(Error::other("")); + } + Ok(()) } - return Err(Error::other("")); - } - Ok(()) + .await, + ) + .await } pub async fn load_policy(&self, policy: &str) -> Result<()> { - let mut client = self.get_client().await?; - let request = Request::new(LoadPolicyRequest { - policy_name: policy.to_string(), - }); + self.finalize_result( + async { + let mut client = self.get_client().await?; + let request = Request::new(LoadPolicyRequest { + policy_name: policy.to_string(), + }); - let response = client.load_policy(request).await?.into_inner(); - if !response.success { - if let Some(msg) = response.error_info { - return Err(Error::other(msg)); + let response = client.load_policy(request).await?.into_inner(); + if !response.success { + if let Some(msg) = response.error_info { + return Err(Error::other(msg)); + } + return Err(Error::other("")); + } + Ok(()) } - return Err(Error::other("")); - } - Ok(()) + .await, + ) + .await } pub async fn load_policy_mapping(&self, user_or_group: &str, user_type: u64, is_group: bool) -> Result<()> { - let mut client = self.get_client().await?; - let request = Request::new(LoadPolicyMappingRequest { - user_or_group: user_or_group.to_string(), - user_type, - is_group, - }); + self.finalize_result( + async { + let mut client = self.get_client().await?; + let request = Request::new(LoadPolicyMappingRequest { + user_or_group: user_or_group.to_string(), + user_type, + is_group, + }); - let response = client.load_policy_mapping(request).await?.into_inner(); - if !response.success { - if let Some(msg) = response.error_info { - return Err(Error::other(msg)); + let response = client.load_policy_mapping(request).await?.into_inner(); + if !response.success { + if let Some(msg) = response.error_info { + return Err(Error::other(msg)); + } + return Err(Error::other("")); + } + Ok(()) } - return Err(Error::other("")); - } - Ok(()) + .await, + ) + .await } pub async fn delete_user(&self, access_key: &str) -> Result<()> { - let mut client = self.get_client().await?; - let request = Request::new(DeleteUserRequest { - access_key: access_key.to_string(), - }); + self.finalize_result( + async { + let mut client = self.get_client().await?; + let request = Request::new(DeleteUserRequest { + access_key: access_key.to_string(), + }); - let result = client.delete_user(request).await; - if result.is_err() { - self.evict_connection().await; - } - let response = result?.into_inner(); - if !response.success { - if let Some(msg) = response.error_info { - return Err(Error::other(msg)); + let response = client.delete_user(request).await?.into_inner(); + if !response.success { + if let Some(msg) = response.error_info { + return Err(Error::other(msg)); + } + return Err(Error::other("")); + } + Ok(()) } - return Err(Error::other("")); - } - Ok(()) + .await, + ) + .await } pub async fn delete_service_account(&self, access_key: &str) -> Result<()> { - let mut client = self.get_client().await?; - let request = Request::new(DeleteServiceAccountRequest { - access_key: access_key.to_string(), - }); + self.finalize_result( + async { + let mut client = self.get_client().await?; + let request = Request::new(DeleteServiceAccountRequest { + access_key: access_key.to_string(), + }); - let result = client.delete_service_account(request).await; - if result.is_err() { - self.evict_connection().await; - } - let response = result?.into_inner(); - if !response.success { - if let Some(msg) = response.error_info { - return Err(Error::other(msg)); + let response = client.delete_service_account(request).await?.into_inner(); + if !response.success { + if let Some(msg) = response.error_info { + return Err(Error::other(msg)); + } + return Err(Error::other("")); + } + Ok(()) } - return Err(Error::other("")); - } - Ok(()) + .await, + ) + .await } pub async fn load_user(&self, access_key: &str, temp: bool) -> Result<()> { - let mut client = self.get_client().await?; - let request = Request::new(LoadUserRequest { - access_key: access_key.to_string(), - temp, - }); + self.finalize_result( + async { + let mut client = self.get_client().await?; + let request = Request::new(LoadUserRequest { + access_key: access_key.to_string(), + temp, + }); - let result = client.load_user(request).await; - if result.is_err() { - self.evict_connection().await; - } - let response = result?.into_inner(); - if !response.success { - if let Some(msg) = response.error_info { - return Err(Error::other(msg)); + let response = client.load_user(request).await?.into_inner(); + if !response.success { + if let Some(msg) = response.error_info { + return Err(Error::other(msg)); + } + return Err(Error::other("")); + } + Ok(()) } - return Err(Error::other("")); - } - Ok(()) + .await, + ) + .await } pub async fn load_service_account(&self, access_key: &str) -> Result<()> { - let mut client = self.get_client().await?; - let request = Request::new(LoadServiceAccountRequest { - access_key: access_key.to_string(), - }); + self.finalize_result( + async { + let mut client = self.get_client().await?; + let request = Request::new(LoadServiceAccountRequest { + access_key: access_key.to_string(), + }); - let result = client.load_service_account(request).await; - if result.is_err() { - self.evict_connection().await; - } - let response = result?.into_inner(); - if !response.success { - if let Some(msg) = response.error_info { - return Err(Error::other(msg)); + let response = client.load_service_account(request).await?.into_inner(); + if !response.success { + if let Some(msg) = response.error_info { + return Err(Error::other(msg)); + } + return Err(Error::other("")); + } + Ok(()) } - return Err(Error::other("")); - } - Ok(()) + .await, + ) + .await } pub async fn load_group(&self, group: &str) -> Result<()> { - let mut client = self.get_client().await?; - let request = Request::new(LoadGroupRequest { - group: group.to_string(), - }); + self.finalize_result( + async { + let mut client = self.get_client().await?; + let request = Request::new(LoadGroupRequest { + group: group.to_string(), + }); - let result = client.load_group(request).await; - if result.is_err() { - self.evict_connection().await; - } - let response = result?.into_inner(); - if !response.success { - if let Some(msg) = response.error_info { - return Err(Error::other(msg)); + let response = client.load_group(request).await?.into_inner(); + if !response.success { + if let Some(msg) = response.error_info { + return Err(Error::other(msg)); + } + return Err(Error::other("")); + } + Ok(()) } - return Err(Error::other("")); - } - Ok(()) + .await, + ) + .await } pub async fn reload_site_replication_config(&self) -> Result<()> { - let mut client = self.get_client().await?; - let request = Request::new(ReloadSiteReplicationConfigRequest {}); + self.finalize_result( + async { + let mut client = self.get_client().await?; + let request = Request::new(ReloadSiteReplicationConfigRequest {}); - let response = client.reload_site_replication_config(request).await?.into_inner(); - if !response.success { - if let Some(msg) = response.error_info { - return Err(Error::other(msg)); + let response = client.reload_site_replication_config(request).await?.into_inner(); + if !response.success { + if let Some(msg) = response.error_info { + return Err(Error::other(msg)); + } + return Err(Error::other("")); + } + Ok(()) } - return Err(Error::other("")); - } - Ok(()) + .await, + ) + .await } pub async fn signal_service(&self, sig: u64, sub_sys: &str, dry_run: bool, _exec_at: SystemTime) -> Result<()> { - let mut client = self.get_client().await?; - let mut vars = HashMap::new(); - vars.insert(PEER_RESTSIGNAL.to_string(), sig.to_string()); - vars.insert(PEER_RESTSUB_SYS.to_string(), sub_sys.to_string()); - vars.insert(PEER_RESTDRY_RUN.to_string(), dry_run.to_string()); - let request = Request::new(SignalServiceRequest { - vars: Some(Mss { value: vars }), - }); + self.finalize_result( + async { + let mut client = self.get_client().await?; + let mut vars = HashMap::new(); + vars.insert(PEER_RESTSIGNAL.to_string(), sig.to_string()); + vars.insert(PEER_RESTSUB_SYS.to_string(), sub_sys.to_string()); + vars.insert(PEER_RESTDRY_RUN.to_string(), dry_run.to_string()); + let request = Request::new(SignalServiceRequest { + vars: Some(Mss { value: vars }), + }); - let response = client.signal_service(request).await?.into_inner(); - if !response.success { - if let Some(msg) = response.error_info { - return Err(Error::other(msg)); + let response = client.signal_service(request).await?.into_inner(); + if !response.success { + if let Some(msg) = response.error_info { + return Err(Error::other(msg)); + } + return Err(Error::other("")); + } + Ok(()) } - return Err(Error::other("")); - } - Ok(()) + .await, + ) + .await } pub async fn get_metacache_listing(&self) -> Result<()> { @@ -642,64 +865,148 @@ impl PeerRestClient { } pub async fn reload_pool_meta(&self) -> Result<()> { - let mut client = self.get_client().await?; - let request = Request::new(ReloadPoolMetaRequest {}); + self.finalize_result( + async { + let mut client = self.get_client().await?; + let request = Request::new(ReloadPoolMetaRequest {}); - let response = client.reload_pool_meta(request).await?.into_inner(); - if !response.success { - if let Some(msg) = response.error_info { - return Err(Error::other(msg)); + let response = client.reload_pool_meta(request).await?.into_inner(); + if !response.success { + if let Some(msg) = response.error_info { + return Err(Error::other(msg)); + } + return Err(Error::other("")); + } + + Ok(()) } - return Err(Error::other("")); - } - - Ok(()) + .await, + ) + .await } pub async fn stop_rebalance(&self) -> Result<()> { - let mut client = self.get_client().await?; - let request = Request::new(StopRebalanceRequest {}); + self.finalize_result( + async { + let mut client = self.get_client().await?; + let request = Request::new(StopRebalanceRequest {}); - let response = client.stop_rebalance(request).await?.into_inner(); - if !response.success { - if let Some(msg) = response.error_info { - return Err(Error::other(msg)); + let response = client.stop_rebalance(request).await?.into_inner(); + if !response.success { + if let Some(msg) = response.error_info { + return Err(Error::other(msg)); + } + return Err(Error::other("")); + } + + Ok(()) } - return Err(Error::other("")); - } - - Ok(()) + .await, + ) + .await } pub async fn load_rebalance_meta(&self, start_rebalance: bool) -> Result<()> { - let mut client = self.get_client().await?; - let request = Request::new(LoadRebalanceMetaRequest { start_rebalance }); + self.finalize_result( + async { + let mut client = self.get_client().await?; + let request = Request::new(LoadRebalanceMetaRequest { start_rebalance }); - let response = client.load_rebalance_meta(request).await?.into_inner(); + let response = client.load_rebalance_meta(request).await?.into_inner(); - warn!("load_rebalance_meta response {:?}, grid_host: {:?}", response, &self.grid_host); - if !response.success { - if let Some(msg) = response.error_info { - return Err(Error::other(msg)); + warn!("load_rebalance_meta response {:?}, grid_host: {:?}", response, &self.grid_host); + if !response.success { + if let Some(msg) = response.error_info { + return Err(Error::other(msg)); + } + return Err(Error::other("")); + } + + Ok(()) } - return Err(Error::other("")); - } - - Ok(()) + .await, + ) + .await } pub async fn load_transition_tier_config(&self) -> Result<()> { - let mut client = self.get_client().await?; - let request = Request::new(LoadTransitionTierConfigRequest {}); + self.finalize_result( + async { + let mut client = self.get_client().await?; + let request = Request::new(LoadTransitionTierConfigRequest {}); - let response = client.load_transition_tier_config(request).await?.into_inner(); - if !response.success { - if let Some(msg) = response.error_info { - return Err(Error::other(msg)); + let response = client.load_transition_tier_config(request).await?.into_inner(); + if !response.success { + if let Some(msg) = response.error_info { + return Err(Error::other(msg)); + } + return Err(Error::other("")); + } + + Ok(()) } - return Err(Error::other("")); - } - - Ok(()) + .await, + ) + .await + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_peer_client() -> PeerRestClient { + PeerRestClient::new( + XHost { + name: "127.0.0.1".to_string(), + port: 9000, + is_port_set: true, + }, + "http://127.0.0.1:9000".to_string(), + ) + } + + #[test] + fn peer_rest_client_marks_network_like_errors() { + assert!(PeerRestClient::is_network_like_error(&Error::other("transport error"))); + assert!(PeerRestClient::is_network_like_error(&Error::other("connection refused"))); + assert!(!PeerRestClient::is_network_like_error(&Error::NotImplemented)); + } + + #[tokio::test] + async fn peer_rest_client_fast_fails_when_marked_offline() { + let client = test_peer_client(); + client.offline.store(true, Ordering::Release); + + let err = client + .get_client() + .await + .expect_err("offline peer should fast-fail before dialing"); + + assert!(err.to_string().contains("temporarily offline")); + } + + #[tokio::test] + async fn peer_rest_client_finalize_result_marks_offline_for_network_errors() { + let client = test_peer_client(); + let err = client + .finalize_result::<()>(Err(Error::other("transport error"))) + .await + .expect_err("network error should still be returned"); + + assert!(err.to_string().contains("transport error")); + assert!(client.offline.load(Ordering::Acquire)); + } + + #[tokio::test] + async fn peer_rest_client_finalize_result_keeps_online_for_business_errors() { + let client = test_peer_client(); + let err = client + .finalize_result::<()>(Err(Error::VolumeNotFound)) + .await + .expect_err("business error should still be returned"); + + assert!(matches!(err, Error::VolumeNotFound)); + assert!(!client.offline.load(Ordering::Acquire)); } } diff --git a/crates/ecstore/src/rpc/peer_s3_client.rs b/crates/ecstore/src/rpc/peer_s3_client.rs index a52c8fd48..d4dd4e8ff 100644 --- a/crates/ecstore/src/rpc/peer_s3_client.rs +++ b/crates/ecstore/src/rpc/peer_s3_client.rs @@ -18,13 +18,15 @@ use crate::disk::error::{Error, Result}; use crate::disk::error_reduce::{BUCKET_OP_IGNORED_ERRS, is_all_buckets_not_found, reduce_write_quorum_errs}; use crate::disk::{DiskAPI, DiskStore, disk_store::get_max_timeout_duration}; use crate::global::GLOBAL_LOCAL_DISK_MAP; -use crate::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client}; +use crate::rpc::client::{ + TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client, +}; use crate::store::all_local_disk; use crate::store_utils::is_reserved_or_invalid_bucket; use crate::{ disk::{ self, VolumeInfo, - disk_store::{CHECK_EVERY, CHECK_TIMEOUT_DURATION, DiskHealthTracker}, + disk_store::{DiskHealthTracker, get_drive_active_check_interval, get_drive_active_check_timeout}, }, endpoints::{EndpointServerPools, Node}, store_api::{BucketInfo, BucketOptions, DeleteBucketOptions, MakeBucketOptions}, @@ -613,7 +615,7 @@ impl RemotePeerS3Client { /// Monitor remote peer health periodically async fn monitor_remote_peer_health(addr: String, health: Arc, cancel_token: CancellationToken) { - let mut interval = time::interval(CHECK_EVERY); + let mut interval = time::interval(get_drive_active_check_interval()); loop { tokio::select! { @@ -682,7 +684,7 @@ impl RemotePeerS3Client { let port = url.port_or_known_default().unwrap_or(80); // Try to establish TCP connection - match timeout(CHECK_TIMEOUT_DURATION, TcpStream::connect((host, port))).await { + match timeout(get_drive_active_check_timeout(), TcpStream::connect((host, port))).await { Ok(Ok(_)) => Ok(()), _ => Err(Error::other(format!("Cannot connect to {host}:{port}"))), } @@ -717,16 +719,39 @@ impl RemotePeerS3Client { self.health.log_success(); } self.health.decrement_waiting(); + if let Err(err) = &operation_result + && is_network_like_disk_error(err) + { + self.mark_faulty_and_start_recovery("operation_network_error").await; + } operation_result } Err(_) => { // Timeout occurred, mark peer as potentially faulty self.health.decrement_waiting(); + self.mark_faulty_and_start_recovery("operation_timeout").await; warn!("Remote peer operation timeout after {:?}", timeout_duration); Err(Error::other(format!("Remote peer operation timeout after {timeout_duration:?}"))) } } } + + async fn mark_faulty_and_start_recovery(&self, reason: &'static str) { + if self.health.swap_ok_to_faulty() { + warn!( + addr = %self.addr, + reason, + "Remote peer marked faulty after network failure" + ); + + let health = Arc::clone(&self.health); + let cancel_token = self.cancel_token.clone(); + let addr = self.addr.clone(); + tokio::spawn(async move { + Self::monitor_remote_peer_recovery(addr, health, cancel_token).await; + }); + } + } } #[async_trait] @@ -1001,3 +1026,69 @@ pub async fn heal_bucket_local(bucket: &str, opts: &HealOpts) -> Result Vec> { GLOBAL_LOCAL_DISK_MAP.read().await.values().cloned().collect::>() } + +#[cfg(test)] +mod tests { + use super::*; + + fn test_remote_peer(addr: &str) -> RemotePeerS3Client { + let node = Node { + url: url::Url::parse(addr).expect("test peer URL should parse"), + pools: vec![0], + is_local: false, + grid_host: addr.to_string(), + }; + + RemotePeerS3Client { + node: Some(node), + pools: Some(vec![0]), + addr: addr.to_string(), + health: Arc::new(DiskHealthTracker::new()), + cancel_token: CancellationToken::new(), + } + } + + #[tokio::test] + async fn test_execute_with_timeout_marks_remote_peer_faulty_on_network_like_error() { + let client = test_remote_peer("http://peer-network-error:9000"); + + let err = client + .execute_with_timeout( + || async { + Err::<(), Error>(DiskError::Io(std::io::Error::new( + std::io::ErrorKind::ConnectionRefused, + "connection refused", + ))) + }, + Duration::from_secs(1), + ) + .await + .expect_err("network-like error should fail"); + + assert_eq!( + match &err { + DiskError::Io(io_err) => io_err.kind(), + other => panic!("expected io network error, got {other:?}"), + }, + std::io::ErrorKind::ConnectionRefused + ); + assert!(client.health.is_faulty(), "network-like errors should mark remote peer faulty"); + + client.cancel_token.cancel(); + } + + #[tokio::test] + async fn test_execute_with_timeout_keeps_remote_peer_online_for_business_error() { + let client = test_remote_peer("http://peer-business-error:9000"); + + let err = client + .execute_with_timeout(|| async { Err::<(), Error>(DiskError::FileNotFound) }, Duration::from_secs(1)) + .await + .expect_err("business error should fail"); + + assert_eq!(err, DiskError::FileNotFound); + assert!(!client.health.is_faulty(), "business errors should not mark remote peer faulty"); + + client.cancel_token.cancel(); + } +} diff --git a/crates/ecstore/src/rpc/remote_disk.rs b/crates/ecstore/src/rpc/remote_disk.rs index faf2e6a66..eb58b0fd6 100644 --- a/crates/ecstore/src/rpc/remote_disk.rs +++ b/crates/ecstore/src/rpc/remote_disk.rs @@ -16,15 +16,17 @@ use crate::disk::{ CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, FileInfoVersions, FileReader, FileWriter, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, disk_store::{ - CHECK_EVERY, CHECK_TIMEOUT_DURATION, DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, - SKIP_IF_SUCCESS_BEFORE, get_drive_disk_info_timeout, get_drive_list_dir_timeout, get_drive_metadata_timeout, - get_drive_walkdir_stall_timeout, get_drive_walkdir_timeout, get_max_timeout_duration, + DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, SKIP_IF_SUCCESS_BEFORE, + get_drive_active_check_interval, get_drive_active_check_timeout, get_drive_disk_info_timeout, get_drive_list_dir_timeout, + get_drive_metadata_timeout, get_drive_walkdir_stall_timeout, get_drive_walkdir_timeout, get_max_timeout_duration, }, endpoint::Endpoint, health_state::{RuntimeDriveHealthState, get_drive_returning_probe_interval, record_drive_runtime_state}, }; use crate::disk::{disk_store::DiskHealthTracker, error::DiskError, local::ScanGuard}; -use crate::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client}; +use crate::rpc::client::{ + TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client, +}; use crate::set_disk::DEFAULT_READ_BUFFER_SIZE; use crate::{ disk::error::{Error, Result}, @@ -47,7 +49,7 @@ use rustfs_protos::proto_gen::node_service::{ use rustfs_rio::{HttpReader, HttpWriter}; use serde::{Serialize, de::DeserializeOwned}; use std::{ - io::{Cursor, ErrorKind}, + io::Cursor, path::PathBuf, sync::{ Arc, @@ -176,7 +178,7 @@ impl RemoteDisk { health: Arc, cancel_token: CancellationToken, ) { - let mut interval = time::interval(CHECK_EVERY); + let mut interval = time::interval(get_drive_active_check_interval()); // Perform basic connectivity check if Self::perform_connectivity_check(&addr).await.is_err() && health.mark_offline(&endpoint, "connectivity_probe_failed") { @@ -281,7 +283,7 @@ impl RemoteDisk { let port = url.port_or_known_default().unwrap_or(80); // Try to establish TCP connection - match timeout(CHECK_TIMEOUT_DURATION, TcpStream::connect((host, port))).await { + match timeout(get_drive_active_check_timeout(), TcpStream::connect((host, port))).await { Ok(Ok(stream)) => { drop(stream); Ok(()) @@ -334,10 +336,10 @@ impl RemoteDisk { } self.health.decrement_waiting(); if let Err(err) = &operation_result - && Self::is_timeout_like_error(err) + && is_network_like_disk_error(err) { counter!( - "rustfs_drive_op_timeout_total", + "rustfs_drive_op_network_error_total", "endpoint" => self.endpoint.to_string(), "op" => op.to_string() ) @@ -347,9 +349,9 @@ impl RemoteDisk { addr = %self.addr, op, timeout_ms = timeout_duration.as_millis(), - "Remote disk operation returned a timeout-like error" + "Remote disk operation returned a network-like error" ); - self.mark_faulty_and_evict("operation_timeout_error").await; + self.mark_faulty_and_evict("operation_network_error").await; } operation_result } @@ -375,10 +377,6 @@ impl RemoteDisk { } } - fn is_timeout_like_error(err: &Error) -> bool { - matches!(err, DiskError::Timeout) || matches!(err, DiskError::Io(io_err) if io_err.kind() == ErrorKind::TimedOut) - } - async fn mark_faulty_and_evict(&self, reason: &'static str) { if self.health.mark_offline(&self.endpoint, reason) { self.spawn_recovery_monitor_if_needed(); @@ -2073,6 +2071,96 @@ mod tests { ); } + #[tokio::test] + async fn test_execute_with_timeout_marks_faulty_on_network_like_error() { + let addr = "http://127.0.0.1:59993".to_string(); + let url = url::Url::parse(&format!("{addr}/data")).unwrap(); + let endpoint = Endpoint { + url, + is_local: false, + pool_idx: 0, + set_idx: 0, + disk_idx: 0, + }; + + let remote_disk = RemoteDisk::new( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .unwrap(); + + let channel = TonicEndpoint::from_shared(addr.clone()).unwrap().connect_lazy(); + GLOBAL_CONN_MAP.write().await.insert(addr.clone(), channel); + + let err = remote_disk + .execute_with_timeout( + || async { + Err::<(), Error>(DiskError::Io(std::io::Error::new( + std::io::ErrorKind::ConnectionRefused, + "connection refused", + ))) + }, + Duration::from_secs(1), + ) + .await + .expect_err("network-like operation error should fail"); + + assert_eq!( + match &err { + DiskError::Io(io_err) => io_err.kind(), + other => panic!("expected io network error, got {other:?}"), + }, + std::io::ErrorKind::ConnectionRefused + ); + assert!(!remote_disk.is_online().await, "network-like errors should mark remote disk faulty"); + assert!( + !GLOBAL_CONN_MAP.read().await.contains_key(&addr), + "network-like errors should evict cached connection" + ); + } + + #[tokio::test] + async fn test_execute_with_timeout_keeps_remote_disk_online_for_business_error() { + let addr = "http://127.0.0.1:59994".to_string(); + let url = url::Url::parse(&format!("{addr}/data")).unwrap(); + let endpoint = Endpoint { + url, + is_local: false, + pool_idx: 0, + set_idx: 0, + disk_idx: 0, + }; + + let remote_disk = RemoteDisk::new( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .unwrap(); + + let channel = TonicEndpoint::from_shared(addr.clone()).unwrap().connect_lazy(); + GLOBAL_CONN_MAP.write().await.insert(addr.clone(), channel); + + let err = remote_disk + .execute_with_timeout(|| async { Err::<(), Error>(DiskError::FileNotFound) }, Duration::from_secs(1)) + .await + .expect_err("business error should still fail the operation"); + + assert_eq!(err, DiskError::FileNotFound); + assert!(remote_disk.is_online().await, "business errors should not mark remote disk faulty"); + assert!( + GLOBAL_CONN_MAP.read().await.contains_key(&addr), + "business errors should not evict cached connection" + ); + } + #[test] fn test_remote_disk_sync_properties() { let url = url::Url::parse("https://secure-remote:9000/data").unwrap(); diff --git a/crates/ecstore/src/set_disk/multipart.rs b/crates/ecstore/src/set_disk/multipart.rs index 491180827..983dc65e7 100644 --- a/crates/ecstore/src/set_disk/multipart.rs +++ b/crates/ecstore/src/set_disk/multipart.rs @@ -13,6 +13,87 @@ // limitations under the License. use super::*; +use std::future::Future; +use std::time::Duration; +use tokio::task::JoinSet; + +async fn collect_list_parts_results( + tasks: Vec, + read_quorum: usize, +) -> disk::error::Result<(Vec>, Vec>)> +where + F: Future>> + Send + 'static, +{ + let mut errs = vec![Some(DiskError::DiskNotFound); tasks.len()]; + let mut object_parts = vec![Vec::new(); tasks.len()]; + let mut successful_responses = 0usize; + let mut pending = tasks.len(); + let mut join_set = JoinSet::new(); + + for (index, task) in tasks.into_iter().enumerate() { + join_set.spawn(async move { (index, task.await) }); + } + + while let Some(join_result) = join_set.join_next().await { + pending = pending.saturating_sub(1); + + match join_result { + Ok((index, Ok(parts))) => { + errs[index] = None; + object_parts[index] = parts; + successful_responses += 1; + } + Ok((index, Err(err))) => { + errs[index] = Some(err); + } + Err(_) => {} + } + + if successful_responses + pending < read_quorum { + return Err(DiskError::ErasureReadQuorum); + } + } + + Ok((errs, object_parts)) +} + +fn reduce_quorum_part_numbers(object_parts: Vec>, read_quorum: usize) -> Vec { + let mut part_quorum_map: HashMap = HashMap::new(); + + for drive_parts in object_parts { + let mut parts_with_meta_count: HashMap = HashMap::new(); + + // part files can be either part.N or part.N.meta + for part_path in drive_parts { + if let Some(num_str) = part_path.strip_prefix("part.") { + if let Some(meta_idx) = num_str.find(".meta") { + if let Ok(part_num) = num_str[..meta_idx].parse::() { + *parts_with_meta_count.entry(part_num).or_insert(0) += 1; + } + } else if let Ok(part_num) = num_str.parse::() { + *parts_with_meta_count.entry(part_num).or_insert(0) += 1; + } + } + } + + // Include only part.N.meta files with corresponding part.N + for (&part_num, &cnt) in &parts_with_meta_count { + if cnt >= 2 { + *part_quorum_map.entry(part_num).or_insert(0) += 1; + } + } + } + + let mut part_numbers = Vec::with_capacity(part_quorum_map.len()); + for (part_num, count) in part_quorum_map { + if count >= read_quorum { + part_numbers.push(part_num); + } + } + + part_numbers.sort(); + part_numbers +} impl SetDisks { pub(super) async fn list_parts( @@ -21,10 +102,13 @@ impl SetDisks { read_quorum: usize, ) -> disk::error::Result> { let mut futures = Vec::with_capacity(disks.len()); - for (i, disk) in disks.iter().enumerate() { + let part_path = part_path.to_string(); + for disk in disks.iter() { + let disk = disk.clone(); + let part_path = part_path.clone(); futures.push(async move { if let Some(disk) = disk { - disk.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, part_path, -1) + disk.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, part_path.as_str(), -1) .await } else { Err(DiskError::DiskNotFound) @@ -35,60 +119,15 @@ impl SetDisks { let mut errs = Vec::with_capacity(disks.len()); let mut object_parts = Vec::with_capacity(disks.len()); - let results = join_all(futures).await; - for result in results { - match result { - Ok(res) => { - errs.push(None); - object_parts.push(res); - } - Err(e) => { - errs.push(Some(e)); - object_parts.push(vec![]); - } - } - } + let (collected_errs, collected_parts) = collect_list_parts_results(futures, read_quorum).await?; + errs.extend(collected_errs); + object_parts.extend(collected_parts); if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) { return Err(err); } - let mut part_quorum_map: HashMap = HashMap::new(); - - for drive_parts in object_parts { - let mut parts_with_meta_count: HashMap = HashMap::new(); - - // part files can be either part.N or part.N.meta - for part_path in drive_parts { - if let Some(num_str) = part_path.strip_prefix("part.") { - if let Some(meta_idx) = num_str.find(".meta") { - if let Ok(part_num) = num_str[..meta_idx].parse::() { - *parts_with_meta_count.entry(part_num).or_insert(0) += 1; - } - } else if let Ok(part_num) = num_str.parse::() { - *parts_with_meta_count.entry(part_num).or_insert(0) += 1; - } - } - } - - // Include only part.N.meta files with corresponding part.N - for (&part_num, &cnt) in &parts_with_meta_count { - if cnt >= 2 { - *part_quorum_map.entry(part_num).or_insert(0) += 1; - } - } - } - - let mut part_numbers = Vec::with_capacity(part_quorum_map.len()); - for (part_num, count) in part_quorum_map { - if count >= read_quorum { - part_numbers.push(part_num); - } - } - - part_numbers.sort(); - - Ok(part_numbers) + Ok(reduce_quorum_part_numbers(object_parts, read_quorum)) } #[tracing::instrument(level = "debug", skip(self))] @@ -145,3 +184,78 @@ impl SetDisks { Ok((fi, parts_metadata)) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn collect_list_parts_results_fails_early_when_quorum_is_impossible() { + let started = std::time::Instant::now(); + let tasks: Vec<_> = vec![ + (10_u64, Err(DiskError::DiskNotFound)), + (15, Err(DiskError::DiskNotFound)), + (250, Ok::, DiskError>(vec!["part.1".to_string(), "part.1.meta".to_string()])), + ] + .into_iter() + .map(|(delay_ms, outcome)| async move { + tokio::time::sleep(Duration::from_millis(delay_ms)).await; + outcome + }) + .collect(); + + let err = collect_list_parts_results(tasks, 2) + .await + .expect_err("quorum should become impossible before slow tail completes"); + + assert_eq!(err, DiskError::ErasureReadQuorum); + assert!(started.elapsed() < Duration::from_millis(120)); + } + + #[tokio::test] + async fn collect_list_parts_results_tolerates_single_panicked_task_when_quorum_is_met() { + let tasks: Vec<_> = vec![(5_u64, true), (10, false), (12, false)] + .into_iter() + .map(|(delay_ms, should_panic)| async move { + tokio::time::sleep(Duration::from_millis(delay_ms)).await; + if should_panic { + panic!("simulated task panic"); + } + Ok::, DiskError>(vec!["part.1".to_string(), "part.1.meta".to_string()]) + }) + .collect(); + + let (errs, object_parts) = collect_list_parts_results(tasks, 2) + .await + .expect("quorum should still succeed"); + assert_eq!(errs.iter().filter(|err| err.is_none()).count(), 2); + assert_eq!(object_parts.iter().filter(|parts| !parts.is_empty()).count(), 2); + } + + #[test] + fn reduce_quorum_part_numbers_only_keeps_parts_present_on_quorum_of_drives() { + let object_parts = vec![ + vec![ + "part.1".to_string(), + "part.1.meta".to_string(), + "part.2".to_string(), + "part.2.meta".to_string(), + ], + vec![ + "part.1".to_string(), + "part.1.meta".to_string(), + "part.3".to_string(), + "part.3.meta".to_string(), + ], + vec![ + "part.1".to_string(), + "part.1.meta".to_string(), + "part.2".to_string(), + "part.2.meta".to_string(), + ], + ]; + + let parts = reduce_quorum_part_numbers(object_parts, 2); + assert_eq!(parts, vec![1, 2]); + } +} diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index 1efc663d0..35a2f9102 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -14,6 +14,88 @@ use super::*; use rustfs_config::{DEFAULT_OBJECT_ZERO_COPY_ENABLE, ENV_OBJECT_ZERO_COPY_ENABLE}; +use std::future::Future; +use tokio::task::JoinSet; + +async fn collect_read_multiple_results( + tasks: Vec, + read_quorum: usize, +) -> std::result::Result<(Vec>>, Vec>), ()> +where + F: Future>> + Send + 'static, +{ + let mut responses = vec![None; tasks.len()]; + let mut errors = vec![Some(DiskError::DiskNotFound); tasks.len()]; + let mut successful_responses = 0usize; + let mut pending = tasks.len(); + let mut join_set = JoinSet::new(); + + for (index, task) in tasks.into_iter().enumerate() { + join_set.spawn(async move { (index, task.await) }); + } + + while let Some(join_result) = join_set.join_next().await { + pending = pending.saturating_sub(1); + + match join_result { + Ok((index, Ok(resp))) => { + responses[index] = Some(resp); + errors[index] = None; + successful_responses += 1; + } + Ok((index, Err(err))) => { + errors[index] = Some(err); + } + Err(_) => {} + } + + if successful_responses + pending < read_quorum { + return Err(()); + } + } + + Ok((responses, errors)) +} + +async fn collect_read_parts_results( + tasks: Vec, + read_quorum: usize, +) -> std::result::Result<(Vec>>, Vec>), ()> +where + F: Future>> + Send + 'static, +{ + let mut responses = vec![None; tasks.len()]; + let mut errors = vec![Some(DiskError::DiskNotFound); tasks.len()]; + let mut successful_responses = 0usize; + let mut pending = tasks.len(); + let mut join_set = JoinSet::new(); + + for (index, task) in tasks.into_iter().enumerate() { + join_set.spawn(async move { (index, task.await) }); + } + + while let Some(join_result) = join_set.join_next().await { + pending = pending.saturating_sub(1); + + match join_result { + Ok((index, Ok(resp))) => { + responses[index] = Some(resp); + errors[index] = None; + successful_responses += 1; + } + Ok((index, Err(err))) => { + errors[index] = Some(err); + } + Err(_) => {} + } + + if successful_responses + pending < read_quorum { + return Err(()); + } + } + + Ok((responses, errors)) +} impl SetDisks { pub(super) async fn read_parts( @@ -25,9 +107,6 @@ impl SetDisks { ) -> disk::error::Result> { let mut errs = Vec::with_capacity(disks.len()); let mut object_parts = Vec::with_capacity(disks.len()); - - // Use batch processor for better performance - let processor = get_global_processors().read_processor(); let bucket = bucket.to_string(); let part_meta_paths = part_meta_paths.to_vec(); @@ -48,19 +127,13 @@ impl SetDisks { }) .collect(); - let results = processor.execute_batch(tasks).await; - for result in results { - match result { - Ok(res) => { - errs.push(None); - object_parts.push(res); - } - Err(e) => { - errs.push(Some(e)); - object_parts.push(vec![]); - } - } - } + let (responses, collected_errors) = match collect_read_parts_results(tasks, read_quorum).await { + Ok(collected) => collected, + Err(()) => return Err(DiskError::ErasureReadQuorum), + }; + + errs.extend(collected_errors); + object_parts.extend(responses.into_iter().map(|resp| resp.unwrap_or_default())); if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) { return Err(err); @@ -384,10 +457,23 @@ impl SetDisks { read_quorum: usize, ) -> Vec { let mut futures = Vec::with_capacity(disks.len()); - let mut ress = Vec::with_capacity(disks.len()); - let mut errors = Vec::with_capacity(disks.len()); + let empty_quorum_result = || { + req.files + .iter() + .map(|want| ReadMultipleResp { + bucket: req.bucket.clone(), + prefix: req.prefix.clone(), + file: want.clone(), + exists: false, + error: Error::ErasureReadQuorum.to_string(), + data: Vec::new(), + mod_time: None, + }) + .collect::>() + }; for disk in disks.iter() { + let disk = disk.clone(); let req = req.clone(); futures.push(async move { if let Some(disk) = disk { @@ -398,19 +484,10 @@ impl SetDisks { }); } - let results = join_all(futures).await; - for result in results { - match result { - Ok(res) => { - ress.push(Some(res)); - errors.push(None); - } - Err(e) => { - ress.push(None); - errors.push(Some(e)); - } - } - } + let (ress, errors) = match collect_read_multiple_results(futures, read_quorum).await { + Ok(collected) => collected, + Err(()) => return empty_quorum_result(), + }; // debug!("ReadMultipleResp ress {:?}", ress); // debug!("ReadMultipleResp errors {:?}", errors); @@ -839,3 +916,181 @@ impl SetDisks { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn collect_read_multiple_results_fails_early_when_quorum_is_impossible() { + let started = std::time::Instant::now(); + let resp = ReadMultipleResp { + bucket: "bucket".to_string(), + prefix: "prefix".to_string(), + file: "file".to_string(), + exists: true, + error: String::new(), + data: vec![1], + mod_time: None, + }; + + let tasks: Vec<_> = vec![ + (10_u64, Err(DiskError::DiskNotFound)), + (15, Err(DiskError::DiskNotFound)), + (250, Ok::, DiskError>(vec![resp])), + ] + .into_iter() + .map(|(delay_ms, outcome)| async move { + tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await; + outcome + }) + .collect(); + + let result = collect_read_multiple_results(tasks, 2).await; + assert!(result.is_err(), "quorum should become impossible before slow tail completes"); + assert!(started.elapsed() < std::time::Duration::from_millis(120)); + } + + #[tokio::test] + async fn collect_read_multiple_results_returns_collected_responses_on_quorum() { + let resp = ReadMultipleResp { + bucket: "bucket".to_string(), + prefix: "prefix".to_string(), + file: "file".to_string(), + exists: true, + error: String::new(), + data: vec![1, 2, 3], + mod_time: None, + }; + + let tasks: Vec<_> = vec![ + (10_u64, Ok::, DiskError>(vec![resp.clone()])), + (15, Ok::, DiskError>(vec![resp.clone()])), + (250, Err(DiskError::DiskNotFound)), + ] + .into_iter() + .map(|(delay_ms, outcome)| async move { + tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await; + outcome + }) + .collect(); + + let (responses, errors) = collect_read_multiple_results(tasks, 2).await.expect("quorum should succeed"); + + assert_eq!(responses.iter().filter(|item| item.is_some()).count(), 2); + assert_eq!(errors.iter().filter(|item| item.is_none()).count(), 2); + } + + #[tokio::test] + async fn collect_read_multiple_results_tolerates_single_panicked_task_when_quorum_is_met() { + let resp = ReadMultipleResp { + bucket: "bucket".to_string(), + prefix: "prefix".to_string(), + file: "file".to_string(), + exists: true, + error: String::new(), + data: vec![1, 2, 3], + mod_time: None, + }; + + let tasks: Vec<_> = vec![(5_u64, true), (10, false), (12, false)] + .into_iter() + .map(|(delay_ms, should_panic)| { + let resp = resp.clone(); + async move { + tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await; + if should_panic { + panic!("simulated task panic"); + } + Ok::, DiskError>(vec![resp]) + } + }) + .collect(); + + let (responses, errors) = collect_read_multiple_results(tasks, 2) + .await + .expect("quorum should still succeed"); + assert_eq!(responses.iter().filter(|item| item.is_some()).count(), 2); + assert_eq!(errors.iter().filter(|item| item.is_none()).count(), 2); + } + + #[tokio::test] + async fn collect_read_parts_results_fails_early_when_quorum_is_impossible() { + let started = std::time::Instant::now(); + let part = ObjectPartInfo { + number: 1, + etag: "etag".to_string(), + ..Default::default() + }; + + let tasks: Vec<_> = vec![ + (10_u64, Err(DiskError::DiskNotFound)), + (15, Err(DiskError::DiskNotFound)), + (250, Ok::, DiskError>(vec![part])), + ] + .into_iter() + .map(|(delay_ms, outcome)| async move { + tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await; + outcome + }) + .collect(); + + let result = collect_read_parts_results(tasks, 2).await; + assert!(result.is_err(), "quorum should become impossible before slow tail completes"); + assert!(started.elapsed() < std::time::Duration::from_millis(120)); + } + + #[tokio::test] + async fn collect_read_parts_results_returns_collected_responses_on_quorum() { + let part = ObjectPartInfo { + number: 1, + etag: "etag".to_string(), + ..Default::default() + }; + + let tasks: Vec<_> = vec![ + (10_u64, Ok::, DiskError>(vec![part.clone()])), + (15, Ok::, DiskError>(vec![part.clone()])), + (250, Err(DiskError::DiskNotFound)), + ] + .into_iter() + .map(|(delay_ms, outcome)| async move { + tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await; + outcome + }) + .collect(); + + let (responses, errors) = collect_read_parts_results(tasks, 2).await.expect("quorum should succeed"); + assert_eq!(responses.iter().filter(|item| item.is_some()).count(), 2); + assert_eq!(errors.iter().filter(|item| item.is_none()).count(), 2); + } + + #[tokio::test] + async fn collect_read_parts_results_tolerates_single_panicked_task_when_quorum_is_met() { + let part = ObjectPartInfo { + number: 1, + etag: "etag".to_string(), + ..Default::default() + }; + + let tasks: Vec<_> = vec![(5_u64, true), (10, false), (12, false)] + .into_iter() + .map(|(delay_ms, should_panic)| { + let part = part.clone(); + async move { + tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await; + if should_panic { + panic!("simulated task panic"); + } + Ok::, DiskError>(vec![part]) + } + }) + .collect(); + + let (responses, errors) = collect_read_parts_results(tasks, 2) + .await + .expect("quorum should still succeed"); + assert_eq!(responses.iter().filter(|item| item.is_some()).count(), 2); + assert_eq!(errors.iter().filter(|item| item.is_none()).count(), 2); + } +} diff --git a/crates/protos/Cargo.toml b/crates/protos/Cargo.toml index 58b735a64..a0251c63a 100644 --- a/crates/protos/Cargo.toml +++ b/crates/protos/Cargo.toml @@ -34,6 +34,8 @@ path = "src/main.rs" [dependencies] rustfs-common.workspace = true +rustfs-config.workspace = true +rustfs-utils.workspace = true flatbuffers = { workspace = true } prost = { workspace = true } tonic = { workspace = true, features = ["transport"] } diff --git a/crates/protos/src/lib.rs b/crates/protos/src/lib.rs index 60073bb18..6a021ed93 100644 --- a/crates/protos/src/lib.rs +++ b/crates/protos/src/lib.rs @@ -40,51 +40,78 @@ pub use generated::*; // Default 100 MB pub const DEFAULT_GRPC_SERVER_MESSAGE_LEN: usize = 100 * 1024 * 1024; -/// Timeout for connection establishment - reduced for faster failure detection -const CONNECT_TIMEOUT_SECS: u64 = 3; - -/// TCP keepalive interval - how often to probe the connection -const TCP_KEEPALIVE_SECS: u64 = 10; - -/// HTTP/2 keepalive interval - application-layer heartbeat -const HTTP2_KEEPALIVE_INTERVAL_SECS: u64 = 5; - -/// HTTP/2 keepalive timeout - how long to wait for PING ACK -const HTTP2_KEEPALIVE_TIMEOUT_SECS: u64 = 3; - -/// Overall RPC timeout - maximum time for any single RPC operation -const RPC_TIMEOUT_SECS: u64 = 30; - /// Default HTTPS prefix for rustfs /// This is the default HTTPS prefix for rustfs. /// It is used to identify HTTPS URLs. /// Default value: https:// const RUSTFS_HTTPS_PREFIX: &str = "https://"; +fn internode_connect_timeout() -> Duration { + Duration::from_secs(rustfs_utils::get_env_u64( + rustfs_config::ENV_INTERNODE_CONNECT_TIMEOUT_SECS, + rustfs_config::DEFAULT_INTERNODE_CONNECT_TIMEOUT_SECS, + )) +} + +fn internode_tcp_keepalive() -> Duration { + Duration::from_secs(rustfs_utils::get_env_u64( + rustfs_config::ENV_INTERNODE_TCP_KEEPALIVE_SECS, + rustfs_config::DEFAULT_INTERNODE_TCP_KEEPALIVE_SECS, + )) +} + +fn internode_http2_keep_alive_interval() -> Duration { + Duration::from_secs(rustfs_utils::get_env_u64( + rustfs_config::ENV_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS, + rustfs_config::DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS, + )) +} + +fn internode_http2_keep_alive_timeout() -> Duration { + Duration::from_secs(rustfs_utils::get_env_u64( + rustfs_config::ENV_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS, + rustfs_config::DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS, + )) +} + +fn internode_rpc_timeout() -> Duration { + Duration::from_secs(rustfs_utils::get_env_u64( + rustfs_config::ENV_INTERNODE_RPC_TIMEOUT_SECS, + rustfs_config::DEFAULT_INTERNODE_RPC_TIMEOUT_SECS, + )) +} + /// Creates a new gRPC channel with optimized keepalive settings for cluster resilience. /// -/// This function is designed to detect dead peers quickly: -/// - Fast connection timeout (3s instead of default 30s+) -/// - Aggressive TCP keepalive (10s) -/// - HTTP/2 PING every 5s, timeout at 3s -/// - Overall RPC timeout of 30s (reduced from 60s) +/// This function is designed to detect dead peers quickly using env-configurable +/// internode transport settings. Defaults come from `rustfs_config` constants: +/// - Connect timeout: `DEFAULT_INTERNODE_CONNECT_TIMEOUT_SECS` (3s) +/// - TCP keepalive: `DEFAULT_INTERNODE_TCP_KEEPALIVE_SECS` (10s) +/// - HTTP/2 keepalive interval: `DEFAULT_INTERNODE_HTTP2_KEEPALIVE_INTERVAL_SECS` (5s) +/// - HTTP/2 keepalive timeout: `DEFAULT_INTERNODE_HTTP2_KEEPALIVE_TIMEOUT_SECS` (3s) +/// - RPC timeout: `DEFAULT_INTERNODE_RPC_TIMEOUT_SECS` (10s) pub async fn create_new_channel(addr: &str) -> Result> { debug!("Creating new gRPC channel to: {}", addr); let dial_started_at = Instant::now(); + let connect_timeout = internode_connect_timeout(); + let tcp_keepalive = internode_tcp_keepalive(); + let http2_keepalive_interval = internode_http2_keep_alive_interval(); + let http2_keepalive_timeout = internode_http2_keep_alive_timeout(); + let rpc_timeout = internode_rpc_timeout(); let mut connector = Endpoint::from_shared(addr.to_string())? // Fast connection timeout for dead peer detection - .connect_timeout(Duration::from_secs(CONNECT_TIMEOUT_SECS)) + .connect_timeout(connect_timeout) // TCP-level keepalive - OS will probe connection - .tcp_keepalive(Some(Duration::from_secs(TCP_KEEPALIVE_SECS))) + .tcp_keepalive(Some(tcp_keepalive)) // HTTP/2 PING frames for application-layer health check - .http2_keep_alive_interval(Duration::from_secs(HTTP2_KEEPALIVE_INTERVAL_SECS)) + .http2_keep_alive_interval(http2_keepalive_interval) // How long to wait for PING ACK before considering connection dead - .keep_alive_timeout(Duration::from_secs(HTTP2_KEEPALIVE_TIMEOUT_SECS)) + .keep_alive_timeout(http2_keepalive_timeout) // Send PINGs even when no active streams (critical for idle connections) .keep_alive_while_idle(true) // Overall timeout for any RPC - fail fast on unresponsive peers - .timeout(Duration::from_secs(RPC_TIMEOUT_SECS)); + .timeout(rpc_timeout); let root_cert = GLOBAL_ROOT_CERT.read().await; if addr.starts_with(RUSTFS_HTTPS_PREFIX) { diff --git a/docker-compose-simple.yml b/docker-compose-simple.yml index d6af57124..96b9dc232 100644 --- a/docker-compose-simple.yml +++ b/docker-compose-simple.yml @@ -35,6 +35,9 @@ services: - RUSTFS_SECRET_KEY=rustfsadmin # CHANGEME - RUSTFS_OBS_LOGGER_LEVEL=info - RUSTFS_TLS_PATH=/opt/tls + # Keep strict disk topology checks enabled by default. + # For local testing only, set `RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true` explicitly. + - RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-false} volumes: - rustfs_data_0:/data/rustfs0 diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index 0a0aa4b3c..786e629e2 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -1444,6 +1444,7 @@ impl DefaultBucketUsecase { return Err(s3_error!(InvalidArgument, "{err}")); } + input_cfg.expiry_updated_at = Some(Timestamp::from(time::OffsetDateTime::now_utc())); let data = serialize_config(&input_cfg)?; metadata_sys::update(&bucket, BUCKET_LIFECYCLE_CONFIG, data) .await diff --git a/scripts/run_four_node_cluster_failover_bench.sh b/scripts/run_four_node_cluster_failover_bench.sh new file mode 100755 index 000000000..f9f0fb673 --- /dev/null +++ b/scripts/run_four_node_cluster_failover_bench.sh @@ -0,0 +1,766 @@ +#!/usr/bin/env bash +set -euo pipefail + +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +CLUSTER_COMPOSE="${CLUSTER_COMPOSE:-${PROJECT_ROOT}/.docker/compose/docker-compose.cluster.local-build.yml}" +OBS_COMPOSE="${OBS_COMPOSE:-${PROJECT_ROOT}/.docker/observability/docker-compose.yml}" +PROJECT_NAME="${PROJECT_NAME:-rustfs-four-node-test}" +IMAGE_TAG="${IMAGE_TAG:-rustfs/rustfs:local-4node}" +WITH_OBSERVABILITY="${WITH_OBSERVABILITY:-true}" +BUILD_LOCAL_IMAGE="${BUILD_LOCAL_IMAGE:-true}" +RUN_FAILOVER="${RUN_FAILOVER:-true}" +RUN_BENCHMARK="${RUN_BENCHMARK:-true}" +KEEP_UP="${KEEP_UP:-false}" +PRECHECK_AUTO_CLEANUP="${PRECHECK_AUTO_CLEANUP:-true}" +WAIT_PROBE_MODE="${WAIT_PROBE_MODE:-service}" + +RUSTFS_ACCESS_KEY="${RUSTFS_ACCESS_KEY:-rustfsadmin}" +RUSTFS_SECRET_KEY="${RUSTFS_SECRET_KEY:-rustfsadmin}" +RUSTFS_OBS_ENDPOINT="${RUSTFS_OBS_ENDPOINT:-}" +RUSTFS_UNSAFE_BYPASS_DISK_CHECK="${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}" + +WAIT_TIMEOUT_SECS="${WAIT_TIMEOUT_SECS:-180}" +BENCH_READY_TIMEOUT_SECS="${BENCH_READY_TIMEOUT_SECS:-180}" +FAILOVER_NODE="${FAILOVER_NODE:-node4}" +FAILOVER_WARMUP_SECS="${FAILOVER_WARMUP_SECS:-5}" +FAILOVER_SAMPLE_SECS="${FAILOVER_SAMPLE_SECS:-60}" +FAILOVER_INTERVAL_SECS="${FAILOVER_INTERVAL_SECS:-1}" +BENCH_WAIT_MODE="${BENCH_WAIT_MODE:-ready}" + +BENCH_ENDPOINT="${BENCH_ENDPOINT:-http://127.0.0.1:9000}" +BENCH_BUCKET="${BENCH_BUCKET:-rustfs-four-node-bench}" +BENCH_CONCURRENCY="${BENCH_CONCURRENCY:-}" +BENCH_CONCURRENCIES="${BENCH_CONCURRENCIES:-}" +BENCH_DURATION="${BENCH_DURATION:-60s}" +BENCH_SIZES="${BENCH_SIZES:-1KiB,4KiB,11Mi}" + +OUT_DIR="${OUT_DIR:-${PROJECT_ROOT}/target/bench/four-node-failover-$(date +%Y%m%d-%H%M%S)}" + +usage() { + cat <<'USAGE' +Usage: + scripts/run_four_node_cluster_failover_bench.sh [options] + +Options: + --cluster-compose 4-node compose file + --obs-compose observability compose file + --project-name docker compose project name + --image-tag image tag to build/use + --with-observability bring up .docker/observability stack together + --without-observability only bring up 4-node cluster + --skip-build skip docker build from Dockerfile.source + --skip-failover skip failover recovery validation + --skip-bench skip benchmark phase + --failover-node node to stop during failover test (default: node4) + --obs-endpoint RUSTFS_OBS_ENDPOINT (default: auto-select by mode) + --bench-endpoint benchmark endpoint (default: http://127.0.0.1:9000) + --bench-sizes comma list (default: 1KiB,4KiB,11Mi) + --bench-concurrency benchmark concurrency + --bench-concurrencies benchmark concurrency list (default: 8,16,32,64,128) + --bench-duration benchmark duration + --out-dir output directory + --keep-up keep compose services running after script exits + -h, --help show help + +Environment: + CLUSTER_COMPOSE OBS_COMPOSE PROJECT_NAME IMAGE_TAG + WITH_OBSERVABILITY BUILD_LOCAL_IMAGE RUN_FAILOVER RUN_BENCHMARK KEEP_UP + RUSTFS_ACCESS_KEY RUSTFS_SECRET_KEY RUSTFS_OBS_ENDPOINT + PRECHECK_AUTO_CLEANUP (true|false, default: true) + WAIT_PROBE_MODE (service|ready, default: service) + WAIT_TIMEOUT_SECS FAILOVER_NODE FAILOVER_WARMUP_SECS FAILOVER_SAMPLE_SECS + FAILOVER_INTERVAL_SECS BENCH_ENDPOINT BENCH_BUCKET BENCH_CONCURRENCY + BENCH_CONCURRENCIES BENCH_DURATION BENCH_SIZES OUT_DIR + BENCH_WAIT_MODE (ready|service, default: ready) + BENCH_READY_TIMEOUT_SECS (default: 180) +USAGE +} + +log_info() { + printf '[INFO] %s\n' "$*" +} + +log_warn() { + printf '[WARN] %s\n' "$*" +} + +log_error() { + printf '[ERROR] %s\n' "$*" >&2 +} + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + log_error "command not found: $1" + exit 1 + fi +} + +compose() { + if [[ "${WITH_OBSERVABILITY}" == "true" ]]; then + docker compose \ + --project-name "${PROJECT_NAME}" \ + -f "${OBS_COMPOSE}" \ + -f "${CLUSTER_COMPOSE}" \ + "$@" + else + docker compose \ + --project-name "${PROJECT_NAME}" \ + -f "${CLUSTER_COMPOSE}" \ + "$@" + fi +} + +resolve_bool() { + local key="$1" + local value="$2" + case "${value}" in + true|false) ;; + *) + log_error "invalid ${key}: ${value} (expected true|false)" + exit 1 + ;; + esac +} + +resolve_probe_mode() { + case "${WAIT_PROBE_MODE}" in + service|ready) ;; + *) + log_error "invalid WAIT_PROBE_MODE: ${WAIT_PROBE_MODE} (expected service|ready)" + exit 1 + ;; + esac +} + +resolve_bench_wait_mode() { + case "${BENCH_WAIT_MODE}" in + ready|service) ;; + *) + log_error "invalid BENCH_WAIT_MODE: ${BENCH_WAIT_MODE} (expected ready|service)" + exit 1 + ;; + esac +} + +resolve_bench_concurrency() { + if [[ -n "${BENCH_CONCURRENCIES}" && -n "${BENCH_CONCURRENCY}" && "${BENCH_CONCURRENCIES}" != "${BENCH_CONCURRENCY}" ]]; then + log_warn "BENCH_CONCURRENCY is ignored because BENCH_CONCURRENCIES is set" + return + fi + + if [[ -n "${BENCH_CONCURRENCIES}" ]]; then + return + fi + + if [[ -n "${BENCH_CONCURRENCY}" ]]; then + BENCH_CONCURRENCIES="${BENCH_CONCURRENCY}" + return + fi + +# BENCH_CONCURRENCIES="8,16,32,64,128" + BENCH_CONCURRENCIES="8,16" +} + +cluster_compose_uses_otel_network() { + # Detect whether any service in cluster compose joins otel-network. + grep -Eq '^[[:space:]]*-[[:space:]]*otel-network([[:space:]]*#.*)?$' "${CLUSTER_COMPOSE}" +} + +obs_compose_has_otel_collector() { + grep -Eq '^[[:space:]]*otel-collector:[[:space:]]*$' "${OBS_COMPOSE}" +} + +resolve_default_obs_endpoint() { + if [[ "${WITH_OBSERVABILITY}" != "true" ]]; then + RUSTFS_OBS_ENDPOINT="http://host.docker.internal:4318" + log_info "Auto-selected RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT} (observability stack disabled)" + return + fi + + if cluster_compose_uses_otel_network && obs_compose_has_otel_collector; then + RUSTFS_OBS_ENDPOINT="http://otel-collector:4318" + log_info "Auto-selected RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT} (shared docker network detected)" + return + fi + + RUSTFS_OBS_ENDPOINT="http://host.docker.internal:4318" + log_info "Auto-selected RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT} (cross-network fallback)" +} + +docker_daemon_ready() { + docker info >/dev/null 2>&1 +} + +port_is_occupied() { + local port="$1" + + if command -v lsof >/dev/null 2>&1; then + lsof -nP -iTCP:"${port}" -sTCP:LISTEN >/dev/null 2>&1 + return $? + fi + + if command -v ss >/dev/null 2>&1; then + ss -ltn "sport = :${port}" 2>/dev/null | awk 'NR>1 {found=1} END{exit found?0:1}' + return $? + fi + + if command -v netstat >/dev/null 2>&1; then + netstat -an 2>/dev/null | grep -E "[\.\:]${port}[[:space:]].*LISTEN" >/dev/null 2>&1 + return $? + fi + + # Fallback: no tool available; treat as unknown (not occupied) and rely on compose failure. + return 1 +} + +print_port_owner() { + local port="$1" + + if command -v lsof >/dev/null 2>&1; then + lsof -nP -iTCP:"${port}" -sTCP:LISTEN 2>/dev/null | awk 'NR==1 || NR==2 {print " " $0}' + return + fi + + if command -v ss >/dev/null 2>&1; then + ss -ltnp "sport = :${port}" 2>/dev/null | awk 'NR==1 || NR==2 {print " " $0}' + fi +} + +cleanup_existing_project_containers() { + local existing_ids + existing_ids="$(docker ps -aq --filter "label=com.docker.compose.project=${PROJECT_NAME}")" + + if [[ -z "${existing_ids}" ]]; then + return 0 + fi + + log_warn "Found existing containers for project ${PROJECT_NAME}." + docker ps -a --filter "label=com.docker.compose.project=${PROJECT_NAME}" --format ' - {{.Names}} ({{.Status}})' + + if [[ "${PRECHECK_AUTO_CLEANUP}" == "true" ]]; then + log_info "PRECHECK_AUTO_CLEANUP=true, removing existing project containers." + # shellcheck disable=SC2086 + docker rm -f ${existing_ids} >/dev/null + else + log_error "existing project containers detected and PRECHECK_AUTO_CLEANUP=false" + log_error "run docker compose down --remove-orphans first, or set PRECHECK_AUTO_CLEANUP=true" + exit 1 + fi +} + +check_required_ports_free() { + local required_ports=( + 9000 9001 9002 9003 + ) + local occupied_ports=() + local port + + if [[ "${WITH_OBSERVABILITY}" == "true" ]]; then + required_ports+=( + 1888 3000 3100 3200 4040 4317 4318 55679 8888 8889 9090 13133 14269 16686 + ) + fi + + for port in "${required_ports[@]}"; do + if port_is_occupied "${port}"; then + occupied_ports+=("${port}") + fi + done + + if [[ "${#occupied_ports[@]}" -gt 0 ]]; then + log_error "required host ports are occupied: ${occupied_ports[*]}" + for port in "${occupied_ports[@]}"; do + print_port_owner "${port}" || true + done + log_error "free these ports or run with a different compose/profile before retrying" + exit 1 + fi +} + +ensure_runtime_image_exists() { + if ! docker image inspect "${IMAGE_TAG}" >/dev/null 2>&1; then + log_error "image not found: ${IMAGE_TAG}" + log_error "build it first or rerun without --skip-build" + exit 1 + fi +} + +check_cluster_volumes_writable() { + local node_idx + local disk_idx + local volume_name + + log_info "Checking cluster data volumes writable" + # Do not pre-create compose-managed volumes here. + # If we create them via plain docker run, compose will warn: + # "already exists but was not created by Docker Compose". + for node_idx in 1 2 3 4; do + for disk_idx in 1 2 3 4; do + volume_name="${PROJECT_NAME}_node${node_idx}_data_${disk_idx}" + if ! docker volume inspect "${volume_name}" >/dev/null 2>&1; then + log_info "volume not present yet (will be created by compose): ${volume_name}" + continue + fi + if ! docker run --rm --entrypoint sh -v "${volume_name}:/probe" "${IMAGE_TAG}" -c \ + 'set -e; touch /probe/.rwtest; rm -f /probe/.rwtest' >/dev/null 2>&1; then + log_error "volume write check failed: ${volume_name}" + exit 1 + fi + done + done +} + +run_precheck_before_build() { + log_info "Running precheck: docker daemon, residue containers, host ports" + + if ! docker_daemon_ready; then + log_error "cannot connect to docker daemon (permission or runtime not ready)" + exit 1 + fi + + cleanup_existing_project_containers + check_required_ports_free +} + +run_precheck_after_build() { + log_info "Running precheck: image exists, cluster volumes writable" + ensure_runtime_image_exists + check_cluster_volumes_writable +} + +node_port() { + case "$1" in + node1) echo "9000" ;; + node2) echo "9001" ;; + node3) echo "9002" ;; + node4) echo "9003" ;; + *) + log_error "unknown node name: $1 (expected node1..node4)" + exit 1 + ;; + esac +} + +wait_http_ok() { + local url="$1" + local start now + start="$(date +%s)" + + while true; do + if curl -fsS --connect-timeout 2 --max-time 3 "${url}" >/dev/null 2>&1; then + return 0 + fi + + now="$(date +%s)" + if (( now - start >= WAIT_TIMEOUT_SECS )); then + log_error "timed out waiting for ${url}" + return 1 + fi + sleep 2 + done +} + +probe_node_service_ok() { + local port="$1" + local health_code root_code + + health_code="$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 2 --max-time 3 "http://127.0.0.1:${port}/health" || true)" + if [[ "${health_code}" != "200" ]]; then + return 1 + fi + + if [[ "${WAIT_PROBE_MODE}" == "ready" ]]; then + local ready_code + ready_code="$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 2 --max-time 3 "http://127.0.0.1:${port}/health/ready" || true)" + [[ "${ready_code}" == "200" ]] + return $? + fi + + # Service mode: keep startup probe permissive to avoid local false negatives. + # Benchmark phase has its own stricter readiness gate via wait_bench_endpoint_ready. + root_code="$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 2 --max-time 3 "http://127.0.0.1:${port}/" || true)" + case "${root_code}" in + [1-5][0-9][0-9]) return 0 ;; + *) return 1 ;; + esac +} + +probe_bench_endpoint_ok() { + local endpoint health_url ready_url root_url + local health_code ready_code root_code + endpoint="${BENCH_ENDPOINT%/}" + health_url="${endpoint}/health" + ready_url="${endpoint}/health/ready" + root_url="${endpoint}/" + + health_code="$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 2 --max-time 3 "${health_url}" || true)" + if [[ "${health_code}" != "200" ]]; then + return 1 + fi + + if [[ "${BENCH_WAIT_MODE}" == "ready" ]]; then + ready_code="$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 2 --max-time 3 "${ready_url}" || true)" + [[ "${ready_code}" == "200" ]] + return $? + fi + + root_code="$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 2 --max-time 3 "${root_url}" || true)" + case "${root_code}" in + 2[0-9][0-9]|3[0-9][0-9]|401|403|404) return 0 ;; + *) return 1 ;; + esac +} + +wait_bench_endpoint_ready() { + local start now + start="$(date +%s)" + + while true; do + if probe_bench_endpoint_ok; then + return 0 + fi + + now="$(date +%s)" + if (( now - start >= BENCH_READY_TIMEOUT_SECS )); then + log_error "timed out waiting for benchmark endpoint ${BENCH_ENDPOINT} (mode=${BENCH_WAIT_MODE})" + return 1 + fi + sleep 2 + done +} + +wait_node_probe_ok() { + local port="$1" + local start now + start="$(date +%s)" + + while true; do + if probe_node_service_ok "${port}"; then + return 0 + fi + + now="$(date +%s)" + if (( now - start >= WAIT_TIMEOUT_SECS )); then + log_error "timed out waiting for node probe on 127.0.0.1:${port} (mode=${WAIT_PROBE_MODE})" + return 1 + fi + sleep 2 + done +} + +wait_cluster_ready() { + local port + for port in 9000 9001 9002 9003; do + wait_node_probe_ok "${port}" + done +} + +probe_survivors_ready() { + local failover_port="$1" + local port + for port in 9000 9001 9002 9003; do + if [[ "${port}" == "${failover_port}" ]]; then + continue + fi + if ! probe_node_service_ok "${port}"; then + return 1 + fi + done + return 0 +} + +run_failover_validation() { + local failover_port + local probe_file + local summary_file + local event_epoch + local end_epoch + local ts + local first_fail + local first_recover + local recovery_secs + + failover_port="$(node_port "${FAILOVER_NODE}")" + probe_file="${OUT_DIR}/failover-probe.csv" + summary_file="${OUT_DIR}/failover-summary.txt" + mkdir -p "$(dirname "${probe_file}")" + + log_info "Running failover validation: stopping ${FAILOVER_NODE}" + sleep "${FAILOVER_WARMUP_SECS}" + + compose stop "${FAILOVER_NODE}" >/dev/null + event_epoch="$(date +%s)" + end_epoch="$((event_epoch + FAILOVER_SAMPLE_SECS))" + + echo "timestamp_epoch,status" > "${probe_file}" + while (( "$(date +%s)" <= end_epoch )); do + ts="$(date +%s)" + if probe_survivors_ready "${failover_port}"; then + echo "${ts},ok" >> "${probe_file}" + else + echo "${ts},fail" >> "${probe_file}" + fi + sleep "${FAILOVER_INTERVAL_SECS}" + done + + first_fail="$(awk -F',' 'NR>1 && $2=="fail" {print $1; exit}' "${probe_file}")" + if [[ -z "${first_fail}" ]]; then + recovery_secs="0" + { + echo "failover_node=${FAILOVER_NODE}" + echo "outage_observed=false" + echo "recovery_seconds=${recovery_secs}" + echo "note=no survivor readiness interruption observed in probe window" + } > "${summary_file}" + else + first_recover="$(awk -F',' -v fail_ts="${first_fail}" 'NR>1 && $1>fail_ts && $2=="ok" {print $1; exit}' "${probe_file}")" + if [[ -z "${first_recover}" ]]; then + { + echo "failover_node=${FAILOVER_NODE}" + echo "outage_observed=true" + echo "recovery_seconds=unrecovered_within_${FAILOVER_SAMPLE_SECS}s" + echo "first_fail_epoch=${first_fail}" + } > "${summary_file}" + else + recovery_secs="$((first_recover - first_fail))" + { + echo "failover_node=${FAILOVER_NODE}" + echo "outage_observed=true" + echo "first_fail_epoch=${first_fail}" + echo "first_recover_epoch=${first_recover}" + echo "recovery_seconds=${recovery_secs}" + } > "${summary_file}" + fi + fi + + log_info "Restarting ${FAILOVER_NODE}" + compose start "${FAILOVER_NODE}" >/dev/null + wait_node_probe_ok "${failover_port}" + wait_cluster_ready +} + +run_benchmark() { + local bench_out_dir + local conc + local conc_dir + bench_out_dir="${OUT_DIR}/benchmark" + mkdir -p "${bench_out_dir}" + + if ! command -v warp >/dev/null 2>&1; then + log_error "warp is required for benchmark phase. Please install warp or run with --skip-bench." + exit 1 + fi + + log_info "Waiting for benchmark endpoint readiness (mode=${BENCH_WAIT_MODE})" + wait_bench_endpoint_ready + + IFS=',' read -r -a conc_list <<< "${BENCH_CONCURRENCIES}" + for conc in "${conc_list[@]}"; do + conc="$(echo "${conc}" | xargs)" + if [[ -z "${conc}" ]]; then + continue + fi + if ! [[ "${conc}" =~ ^[0-9]+$ ]] || [[ "${conc}" -le 0 ]]; then + log_error "invalid concurrency in BENCH_CONCURRENCIES: ${conc}" + exit 1 + fi + + conc_dir="${bench_out_dir}/concurrency-${conc}" + log_info "Running benchmark sequentially with concurrency=${conc}" + ( + cd "${PROJECT_ROOT}" + ./scripts/run_object_batch_bench.sh \ + --tool warp \ + --endpoint "${BENCH_ENDPOINT}" \ + --access-key "${RUSTFS_ACCESS_KEY}" \ + --secret-key "${RUSTFS_SECRET_KEY}" \ + --bucket "${BENCH_BUCKET}" \ + --concurrency "${conc}" \ + --duration "${BENCH_DURATION}" \ + --sizes "${BENCH_SIZES}" \ + --out-dir "${conc_dir}" + ) + done +} + +cleanup() { + if [[ "${KEEP_UP}" == "true" ]]; then + log_info "KEEP_UP=true, leaving containers running" + return + fi + + log_info "Stopping compose services" + compose down --remove-orphans >/dev/null 2>&1 || true +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --cluster-compose) + CLUSTER_COMPOSE="$2" + shift 2 + ;; + --obs-compose) + OBS_COMPOSE="$2" + shift 2 + ;; + --project-name) + PROJECT_NAME="$2" + shift 2 + ;; + --image-tag) + IMAGE_TAG="$2" + shift 2 + ;; + --with-observability) + WITH_OBSERVABILITY=true + shift + ;; + --without-observability) + WITH_OBSERVABILITY=false + shift + ;; + --skip-build) + BUILD_LOCAL_IMAGE=false + shift + ;; + --skip-failover) + RUN_FAILOVER=false + shift + ;; + --skip-bench) + RUN_BENCHMARK=false + shift + ;; + --keep-up) + KEEP_UP=true + shift + ;; + --failover-node) + FAILOVER_NODE="$2" + shift 2 + ;; + --obs-endpoint) + RUSTFS_OBS_ENDPOINT="$2" + shift 2 + ;; + --bench-endpoint) + BENCH_ENDPOINT="$2" + shift 2 + ;; + --bench-sizes) + BENCH_SIZES="$2" + shift 2 + ;; + --bench-concurrency) + BENCH_CONCURRENCY="$2" + BENCH_CONCURRENCIES="$2" + shift 2 + ;; + --bench-concurrencies) + BENCH_CONCURRENCIES="$2" + shift 2 + ;; + --bench-duration) + BENCH_DURATION="$2" + shift 2 + ;; + --out-dir) + OUT_DIR="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + log_error "unknown argument: $1" + usage + exit 1 + ;; + esac + done +} + +main() { + parse_args "$@" + + resolve_bool "WITH_OBSERVABILITY" "${WITH_OBSERVABILITY}" + resolve_bool "BUILD_LOCAL_IMAGE" "${BUILD_LOCAL_IMAGE}" + resolve_bool "RUN_FAILOVER" "${RUN_FAILOVER}" + resolve_bool "RUN_BENCHMARK" "${RUN_BENCHMARK}" + resolve_bool "KEEP_UP" "${KEEP_UP}" + resolve_bool "PRECHECK_AUTO_CLEANUP" "${PRECHECK_AUTO_CLEANUP}" + resolve_probe_mode + resolve_bench_wait_mode + resolve_bench_concurrency + + require_cmd docker + require_cmd curl + require_cmd awk + + if [[ ! -f "${CLUSTER_COMPOSE}" ]]; then + log_error "cluster compose file not found: ${CLUSTER_COMPOSE}" + exit 1 + fi + if [[ "${WITH_OBSERVABILITY}" == "true" && ! -f "${OBS_COMPOSE}" ]]; then + log_error "observability compose file not found: ${OBS_COMPOSE}" + exit 1 + fi + + if [[ -z "${RUSTFS_OBS_ENDPOINT}" ]]; then + resolve_default_obs_endpoint + fi + + if [[ "${RUSTFS_OBS_ENDPOINT}" == "http://127.0.0.1:4318" ]]; then + log_warn "RUSTFS_OBS_ENDPOINT is set to container loopback default (${RUSTFS_OBS_ENDPOINT})." + log_warn "If you need host collector routing, consider: --obs-endpoint http://host.docker.internal:4318" + fi + + mkdir -p "${OUT_DIR}" + + trap cleanup EXIT INT TERM + + export RUSTFS_IMAGE="${IMAGE_TAG}" + export RUSTFS_ACCESS_KEY + export RUSTFS_SECRET_KEY + export RUSTFS_OBS_ENDPOINT + export RUSTFS_UNSAFE_BYPASS_DISK_CHECK + + run_precheck_before_build + + if [[ "${BUILD_LOCAL_IMAGE}" == "true" ]]; then + log_info "Building local image from Dockerfile.source: ${IMAGE_TAG}" + docker build -f "${PROJECT_ROOT}/Dockerfile.source" -t "${IMAGE_TAG}" "${PROJECT_ROOT}" + else + log_info "Skipping image build" + fi + + run_precheck_after_build + + log_info "Starting compose stack" + compose up -d + + log_info "Waiting for 4-node cluster readiness (mode=${WAIT_PROBE_MODE})" + wait_cluster_ready + + if [[ "${RUN_FAILOVER}" == "true" ]]; then + run_failover_validation + else + log_info "Skipping failover validation" + fi + + if [[ "${RUN_BENCHMARK}" == "true" ]]; then + run_benchmark + else + log_info "Skipping benchmark" + fi + + log_info "Validation finished" + log_info "Artifacts directory: ${OUT_DIR}" + log_info "Failover summary: ${OUT_DIR}/failover-summary.txt" + log_info "Failover probe: ${OUT_DIR}/failover-probe.csv" + log_info "Benchmark summary: ${OUT_DIR}/benchmark/summary.csv" +} + +main "$@" diff --git a/scripts/run_object_batch_bench.sh b/scripts/run_object_batch_bench.sh index 7ff320a54..120231a72 100755 --- a/scripts/run_object_batch_bench.sh +++ b/scripts/run_object_batch_bench.sh @@ -74,6 +74,16 @@ require_cmd() { fi } +normalize_warp_host() { + local raw="$1" + raw="${raw#http://}" + raw="${raw#https://}" + raw="${raw%%/*}" + raw="${raw%%\?*}" + raw="${raw%%\#*}" + echo "$raw" +} + parse_args() { while [[ $# -gt 0 ]]; do case "$1" in @@ -127,6 +137,14 @@ validate_args() { exit 1 fi fi + if [[ "$TOOL" == "warp" ]]; then + local warp_host + warp_host="$(normalize_warp_host "$ENDPOINT")" + if [[ -z "$warp_host" ]]; then + echo "ERROR: invalid --endpoint for warp: $ENDPOINT" >&2 + exit 1 + fi + fi } setup_output() { @@ -141,7 +159,7 @@ setup_output() { extract_value() { local pattern="$1" local file="$2" - rg -o "$pattern" "$file" | head -n1 | sed -E "s/$pattern/\\1/" || true + rg -o "$pattern" "$file" | head -n1 || true } collect_metrics() { @@ -161,9 +179,11 @@ run_one() { echo "==== [$TOOL] size=$size concurrency=$CONCURRENCY ====" if [[ "$TOOL" == "warp" ]]; then + local warp_host + warp_host="$(normalize_warp_host "$ENDPOINT")" local cmd=( "$WARP_BIN" "$WARP_MODE" - "--host" "$ENDPOINT" + "--host" "$warp_host" "--access-key" "$ACCESS_KEY" "--secret-key" "$SECRET_KEY" "--bucket" "$BUCKET" @@ -175,7 +195,9 @@ run_one() { if [[ "$INSECURE" == "true" ]]; then cmd+=("--insecure") fi - cmd+=("${EXTRA_ARGS[@]}") + if [[ ${EXTRA_ARGS[@]+_} ]]; then + cmd+=("${EXTRA_ARGS[@]}") + fi if [[ "$DRY_RUN" == "true" ]]; then printf '[DRY-RUN] %q ' "${cmd[@]}" @@ -201,7 +223,9 @@ run_one() { if [[ "$INSECURE" == "true" ]]; then cmd+=("-insecure") fi - cmd+=("${EXTRA_ARGS[@]}") + if [[ ${EXTRA_ARGS[@]+_} ]]; then + cmd+=("${EXTRA_ARGS[@]}") + fi if [[ "$DRY_RUN" == "true" ]]; then printf '[DRY-RUN] %q ' "${cmd[@]}" @@ -255,4 +279,3 @@ main() { } main "$@" - diff --git a/scripts/run_object_batch_bench_enhanced.sh b/scripts/run_object_batch_bench_enhanced.sh index e72b71ba7..57f80f07d 100755 --- a/scripts/run_object_batch_bench_enhanced.sh +++ b/scripts/run_object_batch_bench_enhanced.sh @@ -95,6 +95,16 @@ require_cmd() { fi } +normalize_warp_host() { + local raw="$1" + raw="${raw#http://}" + raw="${raw#https://}" + raw="${raw%%/*}" + raw="${raw%%\?*}" + raw="${raw%%\#*}" + echo "$raw" +} + parse_args() { while [[ $# -gt 0 ]]; do case "$1" in @@ -162,6 +172,14 @@ validate_args() { echo "ERROR: --baseline-csv does not exist: $BASELINE_CSV" >&2 exit 1 fi + if [[ "$TOOL" == "warp" ]]; then + local warp_host + warp_host="$(normalize_warp_host "$ENDPOINT")" + if [[ -z "$warp_host" ]]; then + echo "ERROR: invalid --endpoint for warp: $ENDPOINT" >&2 + exit 1 + fi + fi } setup_output() { @@ -286,9 +304,11 @@ run_one_attempt() { local status="ok" if [[ "$TOOL" == "warp" ]]; then + local warp_host + warp_host="$(normalize_warp_host "$ENDPOINT")" local cmd=( "$WARP_BIN" "$WARP_MODE" - "--host" "$ENDPOINT" + "--host" "$warp_host" "--access-key" "$ACCESS_KEY" "--secret-key" "$SECRET_KEY" "--bucket" "$BUCKET" @@ -300,7 +320,9 @@ run_one_attempt() { if [[ "$INSECURE" == "true" ]]; then cmd+=("--insecure") fi - cmd+=("${EXTRA_ARGS[@]}") + if [[ ${EXTRA_ARGS[@]+_} ]]; then + cmd+=("${EXTRA_ARGS[@]}") + fi if [[ "$DRY_RUN" == "true" ]]; then printf '[DRY-RUN] %q ' "${cmd[@]}" @@ -326,7 +348,9 @@ run_one_attempt() { if [[ "$INSECURE" == "true" ]]; then cmd+=("-insecure") fi - cmd+=("${EXTRA_ARGS[@]}") + if [[ ${EXTRA_ARGS[@]+_} ]]; then + cmd+=("${EXTRA_ARGS[@]}") + fi if [[ "$DRY_RUN" == "true" ]]; then printf '[DRY-RUN] %q ' "${cmd[@]}" @@ -484,4 +508,3 @@ main() { } main "$@" - diff --git a/scripts/validate_issue_1365_docker.sh b/scripts/validate_issue_1365_docker.sh new file mode 100755 index 000000000..f53d313e3 --- /dev/null +++ b/scripts/validate_issue_1365_docker.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +set -euo pipefail + +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +COMPOSE_FILE="${COMPOSE_FILE:-docker-compose-simple.yml}" +WAIT_TIMEOUT_SECS="${WAIT_TIMEOUT_SECS:-120}" +KEEP_UP="${KEEP_UP:-false}" +RUN_S3_TESTS="${RUN_S3_TESTS:-true}" +BUILD_LOCAL_IMAGE="${BUILD_LOCAL_IMAGE:-true}" +S3_HOST="${S3_HOST:-127.0.0.1}" +S3_PORT="${S3_PORT:-9000}" + +usage() { + cat <<'USAGE' +Usage: + scripts/validate_issue_1365_docker.sh [options] + +Options: + --compose-file docker compose file (default: docker-compose-simple.yml) + --wait-timeout health wait timeout (default: 120) + --keep-up keep compose services up after the script exits + --skip-s3-tests skip scripts/s3-tests/run.sh + --skip-build skip local Dockerfile.source image build + -h, --help show help + +Environment: + COMPOSE_FILE + WAIT_TIMEOUT_SECS + KEEP_UP + RUN_S3_TESTS + BUILD_LOCAL_IMAGE + S3_HOST + S3_PORT +USAGE +} + +log_info() { + printf '[INFO] %s\n' "$*" +} + +log_error() { + printf '[ERROR] %s\n' "$*" >&2 +} + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + log_error "command not found: $1" + exit 1 + fi +} + +compose() { + local compose_path + compose_path="$(resolve_compose_file)" + docker compose -f "${compose_path}" "$@" +} + +resolve_compose_file() { + if [[ "${COMPOSE_FILE}" = /* ]]; then + printf '%s\n' "${COMPOSE_FILE}" + else + printf '%s\n' "${PROJECT_ROOT}/${COMPOSE_FILE}" + fi +} + +cleanup() { + if [[ "${KEEP_UP}" == "true" ]]; then + log_info "KEEP_UP=true, leaving compose services running" + return + fi + + log_info "Stopping docker compose services" + compose down -v >/dev/null 2>&1 || true +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --compose-file) + COMPOSE_FILE="$2" + shift 2 + ;; + --wait-timeout) + WAIT_TIMEOUT_SECS="$2" + shift 2 + ;; + --keep-up) + KEEP_UP=true + shift + ;; + --skip-s3-tests) + RUN_S3_TESTS=false + shift + ;; + --skip-build) + BUILD_LOCAL_IMAGE=false + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + log_error "unknown argument: $1" + usage + exit 1 + ;; + esac + done +} + +wait_for_endpoint() { + local url="$1" + local start now + start="$(date +%s)" + + while true; do + if curl -fsS --connect-timeout 2 --max-time 3 "${url}" >/dev/null 2>&1; then + return 0 + fi + + now="$(date +%s)" + if (( now - start >= WAIT_TIMEOUT_SECS )); then + log_error "timed out waiting for ${url}" + compose ps || true + compose logs rustfs --tail 200 || true + return 1 + fi + + sleep 2 + done +} + +main() { + parse_args "$@" + require_cmd docker + require_cmd curl + + trap cleanup EXIT INT TERM + + if [[ "${BUILD_LOCAL_IMAGE}" == "true" ]]; then + log_info "Building rustfs/rustfs:latest from Dockerfile.source" + docker build -f "${PROJECT_ROOT}/Dockerfile.source" -t rustfs/rustfs:latest "${PROJECT_ROOT}" + else + log_info "Skipping local image build" + fi + + if [[ -z "${RUSTFS_UNSAFE_BYPASS_DISK_CHECK+x}" ]]; then + export RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true + log_info "RUSTFS_UNSAFE_BYPASS_DISK_CHECK not set; defaulting to true for local validation" + fi + + log_info "Starting docker compose from $(resolve_compose_file)" + compose up -d + + log_info "Waiting for RustFS health endpoint" + wait_for_endpoint "http://${S3_HOST}:${S3_PORT}/health" + + log_info "Waiting for RustFS readiness endpoint" + wait_for_endpoint "http://${S3_HOST}:${S3_PORT}/health/ready" + + log_info "Docker health checks passed" + + if [[ "${RUN_S3_TESTS}" == "true" ]]; then + log_info "Running S3 compatibility tests against the running dockerized service" + ( + cd "${PROJECT_ROOT}" + DEPLOY_MODE=existing S3_HOST="${S3_HOST}" S3_PORT="${S3_PORT}" ./scripts/s3-tests/run.sh + ) + else + log_info "Skipping S3 compatibility tests" + fi + + log_info "Issue 1365 docker validation completed successfully" +} + +main "$@"