feat(cache): add object data cache engine and app flow (#4187)

* feat(cache): add object data cache engine

* feat(cache): wire app-layer object cache flow

* refactor(cache): streamline app-layer cache flow

* refactor(cache): tighten cache flow internals

* refactor: address final clippy cleanup

* chore(deps): update quick-xml to 0.41.0

* feat(cache): wire object data cache env config

* fix(cache): gate materialize fill by cache plan

* chore(cache): add object data cache benchmark gate

* fix(cache): guard object cache fill size mismatches

* refactor(cache): streamline object cache body planning

* fix(cache): align object cache rollout config

* test(cache): cover buffered object cache benchmark

* test(cache): isolate object cache benchmark metrics

* test(cache): mark materialize rollout experimental

* test(cache): tighten object cache benchmark gate

* fix(cache): address review findings for object data cache

- singleflight: clean up leader entry on cancellation (Drop impl) so a dropped GET future can no longer wedge all subsequent fills for the same key; switch the fill map to a std Mutex and add a regression test

- adapter: honor RUSTFS_OBJECT_DATA_CACHE_ENABLE=true by defaulting to hit_only when no explicit mode is set (explicit mode still wins)

- planner: treat nil version UUIDs as "no value" per repo convention so unversioned objects key under the canonical "null" instead of fragmenting the key space

- multipart: invalidate the object cache on the quota-exceeded rollback delete after complete-multipart, closing a stale-cache window

- layering: move the disabled-cache fallback into app::context and drop the new infra->app layer-dependency baseline entry

* fix(cache): close invalidation races and drop full-cache scan on writes

- index: make identity-index insert/remove/prune atomic via starshard compute_if_present/compute_if_absent so concurrent fills can no longer drop each other's keys (lost keys made entries unreachable to invalidation until TTL); add a concurrency regression test

- fill: register the key in the identity index before the entry becomes visible in the cache and re-check the index afterwards, undoing the fill when an invalidation raced in between (new skipped_invalidation_race fill result)

- invalidate: with the index now authoritative, remove the full-cache iter() fallback that made every PUT/DELETE of a never-cached object O(total cache entries) (two scans per PUT, 2N per batch delete)

- materialize-fill: fail the GET instead of falling back to the partially consumed stream after a mid-read error (the fallback would send a body missing its prefix under a full-length Content-Length), and log the same size-mismatch warning as the sibling buffering paths

Co-Authored-By: heihutu <heihutu@gmail.com>

* test(storage): fix media-dependent buffer clamp expectation

test_concurrency_manager_multi_factor_strategy_buffer_clamp asserted media_cap.min(MI_B), but the implementation's final safety clamp is [32KiB, media_cap.max(MI_B)] — deliberately so a media cap above 1MiB (NVMe's 2MiB default) stays effective. The test only passed on machines detected as SSD/Unknown (cap == 1MiB) and failed on NVMe-backed CI runners with 2MiB != 1MiB. Assert the media cap itself, which is what the strategy actually guarantees on every environment.

Co-Authored-By: heihutu <heihutu@gmail.com>

* test(storage): format buffer clamp assertion

* chore(logging): update tier guardrail path

---------

Signed-off-by: houseme <housemecn@gmail.com>
Co-authored-by: cxymds <cxymds@gmail.com>
Co-authored-by: overtrue <anzhengchao@gmail.com>
Co-authored-by: heihutu <heihutu@gmail.com>

scripts/check_logging_guardrails.sh

@@ -43,7 +43,7 @@ checked_files=(
   "crates/targets/src/target/webhook.rs"
   "crates/ecstore/src/store/peer.rs"
   "crates/ecstore/src/store/init.rs"
-  "crates/ecstore/src/tier/tier.rs"
+  "crates/ecstore/src/services/tier/tier.rs"
   "crates/heal/src/heal/manager.rs"
   "crates/heal/src/heal/storage.rs"
   "crates/heal/src/heal/task.rs"

scripts/run_get_codec_streaming_smoke.sh

@@ -26,6 +26,9 @@ ROUND_COOLDOWN_SECS=20
 WARP_OBJECTS=""
 WARP_OBJECT_LIFECYCLE="per-round"
 WARP_PREPARE_DURATION="1s"
+WARP_MODE="get"
+WARP_EXTRA_ARGS=""
+WARP_WARMUP_GET_BEFORE_BENCH=false
 GET_OBJECT_METADATA_CACHE_MAX_ENTRIES=""
 GET_SMALL_OBJECT_DIRECT_MEMORY=""
 GET_SMALL_OBJECT_DIRECT_MEMORY_THRESHOLD=""
@@ -161,6 +164,13 @@ Core options:
   --duration <duration>          warp duration per round (default: 30s)
   --warp-objects <n>             Number of objects prepared by warp for each size
                                  (default: warp default)
+  --warp-mode <get|mixed>        warp workload mode used by the benchmark
+                                 (default: get)
+  --warp-extra-args <args>       Extra arguments passed to the warp benchmark
+                                 command after lifecycle arguments
+  --warp-warmup-get-before-bench Run a GET warmup with --noclear before the
+                                 measured benchmark. Useful for read/write
+                                 invalidation workloads.
   --warp-object-lifecycle <mode> Object lifecycle mode for warp GET:
                                  per-round|prepare-once|existing-only
                                  (default: per-round)
@@ -319,6 +329,9 @@ parse_args() {
       --concurrency) CONCURRENCY="$2"; shift 2 ;;
       --duration) DURATION="$2"; shift 2 ;;
       --warp-objects) WARP_OBJECTS="$2"; shift 2 ;;
+      --warp-mode) WARP_MODE="$2"; shift 2 ;;
+      --warp-extra-args) WARP_EXTRA_ARGS="$2"; shift 2 ;;
+      --warp-warmup-get-before-bench) WARP_WARMUP_GET_BEFORE_BENCH=true; shift ;;
       --warp-object-lifecycle) WARP_OBJECT_LIFECYCLE="$2"; shift 2 ;;
       --warp-prepare-duration) WARP_PREPARE_DURATION="$2"; shift 2 ;;
       --metadata-cache-max-entries) GET_OBJECT_METADATA_CACHE_MAX_ENTRIES="$2"; shift 2 ;;
@@ -415,6 +428,10 @@ validate_args() {
   if [[ -n "$WARP_OBJECTS" ]]; then
     validate_positive_int "$WARP_OBJECTS" "--warp-objects"
   fi
+  case "$WARP_MODE" in
+    get|mixed) ;;
+    *) die "--warp-mode must be get or mixed" ;;
+  esac
   case "$WARP_OBJECT_LIFECYCLE" in
     per-round|prepare-once|existing-only) ;;
     *) die "--warp-object-lifecycle must be per-round, prepare-once, or existing-only" ;;
@@ -940,6 +957,9 @@ rounds=${ROUNDS}
 retry_per_round=${RETRY_PER_ROUND}
 round_cooldown_secs=${ROUND_COOLDOWN_SECS}
 warp_objects=${WARP_OBJECTS}
+warp_mode=${WARP_MODE}
+warp_extra_args=${WARP_EXTRA_ARGS}
+warp_warmup_get_before_bench=${WARP_WARMUP_GET_BEFORE_BENCH}
 warp_object_lifecycle=${WARP_OBJECT_LIFECYCLE}
 warp_prepare_duration=${WARP_PREPARE_DURATION}
 rustfs_bin=${RUSTFS_BIN}
@@ -1207,6 +1227,63 @@ prepare_warp_existing_objects() {
   done < <(benchmark_sizes)
 }
 
+warmup_warp_get_before_bench() {
+  local profile="$1"
+  local profile_dir="${OUT_DIR}/${profile}"
+  local warmup_dir="${profile_dir}/warp_warmup_get"
+
+  if [[ "$WARP_WARMUP_GET_BEFORE_BENCH" != "true" ]]; then
+    return 0
+  fi
+
+  mkdir -p "$warmup_dir"
+
+  local size bucket size_slug
+  while IFS= read -r size; do
+    [[ -n "$size" ]] || continue
+    bucket="$(bucket_for_size "$size")"
+    size_slug="$(sanitize_bucket_suffix "$size")"
+
+    local cmd=(
+      "$WARP_BIN" get
+      --host "$ADDRESS"
+      --access-key "$ACCESS_KEY"
+      --secret-key "$SECRET_KEY"
+      --bucket "$bucket"
+      --obj.size "$size"
+      --concurrent "$CONCURRENCY"
+      --duration "$WARP_PREPARE_DURATION"
+      --region "$REGION"
+      --noclear
+    )
+    if [[ -n "$WARP_OBJECTS" ]]; then
+      cmd+=(--objects "$WARP_OBJECTS")
+    fi
+
+    {
+      printf 'profile=%s\n' "$profile"
+      printf 'mode=warmup-get-before-bench\n'
+      printf 'size=%s\n' "$size"
+      printf 'duration=%s\n' "$WARP_PREPARE_DURATION"
+      printf 'bucket=%s\n' "$bucket"
+      printf 'command='
+      printf '%q ' "${cmd[@]}"
+      printf '\n'
+    } >"${warmup_dir}/manifest-${size_slug}.env"
+
+    if [[ "$DRY_RUN" == "true" ]]; then
+      log "[DRY-RUN] warmup GET before benchmark profile=${profile} size=${size} bucket=${bucket}"
+      printf '[DRY-RUN] ' >"${warmup_dir}/warmup-${size_slug}.log"
+      printf '%q ' "${cmd[@]}" >>"${warmup_dir}/warmup-${size_slug}.log"
+      printf '\n' >>"${warmup_dir}/warmup-${size_slug}.log"
+      continue
+    fi
+
+    log "Running warmup GET before benchmark profile=${profile} size=${size} bucket=${bucket}..."
+    "${cmd[@]}" >"${warmup_dir}/warmup-${size_slug}.log" 2>&1
+  done < <(benchmark_sizes)
+}
+
 run_bench() {
   local profile="$1"
   local baseline_csv="${2:-}"
@@ -1221,7 +1298,7 @@ run_bench() {
     --bucket "$BUCKET"
     --region "$REGION"
     --warp-bin "$WARP_BIN"
-    --warp-mode get
+    --warp-mode "$WARP_MODE"
     --sizes "$SIZES"
     --concurrency "$CONCURRENCY"
     --duration "$DURATION"
@@ -1234,19 +1311,24 @@ run_bench() {
   if [[ -n "$baseline_csv" ]]; then
     cmd+=(--baseline-csv "$baseline_csv")
   fi
+  local lifecycle_args=""
   if [[ "$WARP_OBJECT_LIFECYCLE" == "per-round" && -n "$WARP_OBJECTS" ]]; then
-    cmd+=(--extra-args "--objects ${WARP_OBJECTS}")
-  fi
-  if [[ "$WARP_OBJECT_LIFECYCLE" != "per-round" ]]; then
-    local lifecycle_args="--list-existing --noclear"
+    lifecycle_args="--objects ${WARP_OBJECTS}"
+  elif [[ "$WARP_OBJECT_LIFECYCLE" != "per-round" ]]; then
+    lifecycle_args="--list-existing --noclear"
     if [[ -n "$WARP_OBJECTS" ]]; then
       lifecycle_args="${lifecycle_args} --objects ${WARP_OBJECTS}"
     fi
-    cmd+=(--extra-args "$lifecycle_args")
     if [[ "$(benchmark_size_count)" -gt 1 ]]; then
       cmd+=(--bucket-size-suffix)
     fi
   fi
+  if [[ -n "$WARP_EXTRA_ARGS" ]]; then
+    lifecycle_args="${lifecycle_args:+${lifecycle_args} }${WARP_EXTRA_ARGS}"
+  fi
+  if [[ -n "$lifecycle_args" ]]; then
+    cmd+=(--extra-args "$lifecycle_args")
+  fi
   if [[ "$DIAGNOSTIC_METRICS" == "true" && -z "$DIAGNOSTIC_PROMETHEUS_QUERY_URL" ]]; then
     cmd+=(
       --service-metrics-url "$DIAGNOSTIC_METRICS_URL"
@@ -3925,6 +4007,7 @@ run_profile() {
   stop_server
   start_server "$profile"
   prepare_warp_existing_objects "$profile"
+  warmup_warp_get_before_bench "$profile"
   capture_service_metrics_snapshot "$profile" before
   if run_bench "$profile" "$baseline_csv"; then
     :