chore: merge main after ECStore compile repair

.config/make/pre-commit.mak

@@ -3,9 +3,10 @@
 .NOTPARALLEL: pre-commit pre-pr dev-check
 
 .PHONY: setup-hooks
-setup-hooks: ## Set up git hooks
+setup-hooks: ## Install the configured pre-commit hooks
 	@echo "🔧 Setting up git hooks..."
-	chmod +x .git/hooks/pre-commit
+	pre-commit validate-config
+	pre-commit install
 	@echo "✅ Git hooks setup complete!"
 
 .PHONY: doc-paths-check

.github/pull_request_template.md

@@ -10,16 +10,16 @@ Use N/A when there is no related issue.
 
 ## Summary of Changes
 <!--
-Describe the concrete problem and resulting behavior. For a behavior change, name the input or state that triggers it and the expected outcome. Explain any new dependency or abstraction that the change needs.
+Briefly explain what changed and why reviewers should accept it.
+Focus on behavior, compatibility, and review-relevant context.
 -->
 
 ## Verification
 <!--
-Give 1–3 concrete pieces of evidence for the changed behavior: the test or command, its observed result, and the regression it catches. For a bug fix, record a failing-before/passing-after check or explain why it was unavailable.
+List the commands or checks you ran, for example:
+- `make pre-commit`
 
-Identify the tested commit and any local changes. When testing a prebuilt binary or external service, include its source/version and artifact identity; a successful run against a different build is not evidence for this change.
-
-List relevant checks not run and the remaining risk. Use the validation tier in AGENTS.md; do not run broader checks solely to fill this section. For documentation-only changes, list the applicable documentation checks. Use N/A only when verification is not applicable.
+Use N/A only when verification is not applicable.
 -->
 
 ## Impact

.github/workflows/rustfs-functional-chain.yml

@@ -14,8 +14,8 @@
 
 # Functional chain driver: runs the ten functional suites in a fixed order
 # (upgrade -> s3 -> kms -> tier -> storage -> heal -> pool -> security ->
-# replication -> performance). Each suite attempts the next handoff even
-# when its tests fail.
+# replication, with performance on its own runner in parallel) and guarantees
+# the chain keeps moving even when individual suites fail.
 #
 # Each suite workflow can still be dispatched standalone (workflow_dispatch);
 # only chain-triggered runs forward to the next suite via repository_dispatch,
@@ -59,3 +59,16 @@ jobs:
           gh api --method POST repos/rustfs/rustfs/dispatches \
             -f event_type='rustfs-chain-upgrade' \
             -F 'client_payload[from_suite]=nightly-build'
+
+      - name: Dispatch performance suite (parallel, own runner)
+        env:
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -euo pipefail
+          if [ -z "${GH_TOKEN:-}" ]; then
+            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch performance" >&2
+            exit 1
+          fi
+          gh api --method POST repos/rustfs/rustfs/dispatches \
+            -f event_type='rustfs-chain-performance' \
+            -F 'client_payload[from_suite]=nightly-build'

.github/workflows/rustfs-performance-test.yml

@@ -49,16 +49,17 @@ on:
         type: boolean
         default: true
   repository_dispatch:
-    # Chain handoff: dispatched when the replication suite finishes.
+    # Chain entry: dispatched by rustfs-functional-chain.yml (runs on its own
+    # pf-testing runner, in parallel with the shared-VM chain).
     types: [rustfs-chain-performance]
 
 permissions:
   contents: read
 
-# The default performance nodes overlap the other suites' remote VMs, even
-# though the runner differs. Hold the shared lock through cleanup as well.
+# Dedicated pf-testing runner/environment: own concurrency group so perf runs
+# never block (or are blocked by) the pool-expansion / heal tests.
 concurrency:
-  group: rustfs-shared-functional-tests
+  group: rustfs-performance-test
   cancel-in-progress: false
 
 defaults:

.github/workflows/rustfs-replication-test.yml

@@ -34,7 +34,8 @@ on:
           - site
         default: all
   repository_dispatch:
-    # Chain handoff: dispatched when the security suite finishes.
+    # Chain handoff: dispatched when the security suite finishes. This is the
+    # last link of the functional chain.
     types: [rustfs-chain-replication]
 
 permissions:
@@ -61,6 +62,9 @@ env:
 jobs:
   replication-test:
     runs-on: smoke-testing
+    # A failed replication run must not break the chain or the workflow: the
+    # failure is reported to rustfs/backlog instead (see the issue step).
+    continue-on-error: true
     timeout-minutes: 360
     if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
     steps:
@@ -345,50 +349,13 @@ jobs:
             '
           done
 
-      - name: "Continue functional chain (next: Performance)"
+      - name: Chain complete
+        # Replication is the last link of the functional chain: nothing to
+        # dispatch after it. This step just records that the chain finished.
         if: ${{ always() && github.event_name == 'repository_dispatch' }}
-        env:
-          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
         run: |
-          set -uo pipefail
-          if [ -z "${GH_TOKEN:-}" ]; then
-            echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
-            exit 1
-          fi
-          DISPATCHED=0
-          for attempt in 1 2 3; do
-            if gh api --method POST repos/rustfs/rustfs/dispatches \
-              -f event_type='rustfs-chain-performance' \
-              -F 'client_payload[from_suite]=replication'; then
-              echo "dispatched next suite Performance (attempt ${attempt})"
-              DISPATCHED=1
-              break
-            fi
-            echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
-            sleep "${attempt}0"
-          done
-          if [ "${DISPATCHED:-0}" -ne 1 ]; then
-            echo "ERROR: functional chain stalled: could not dispatch Performance after 3 attempts" >&2
-            TITLE="[functional][chain] stalled after replication (run ${GITHUB_RUN_ID})"
-            BODY_FILE="$(mktemp)"
-            trap 'rm -f "${BODY_FILE}"' EXIT
-            {
-              echo "The functional chain could not hand off from **replication** to **Performance** after 3 attempts."
-              echo ""
-              echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
-              echo "- Expected next event: 'rustfs-chain-performance'"
-              echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
-              echo "- Recovery: re-dispatch manually with"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-              echo "  gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-performance'"
-              FENCE="$(printf "\x60\x60\x60")"; echo "  ${FENCE}"
-            } > "${BODY_FILE}"
-            gh issue create -R rustfs/backlog --title "${TITLE}" \
-              --body-file "${BODY_FILE}" --label functional-test \
-              || gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
-              || echo "could not file the stall alert issue either; check the token" >&2
-            exit 1
-          fi
+          echo "Functional chain complete: replication (final suite) finished."
+          echo "from_suite=security trigger=${{ github.event_name }} outcome=${{ steps.test.outcome }}"
 
       - name: Notify on failure
         if: failure()

.pre-commit-config.yaml

@@ -3,9 +3,9 @@
 repos:
   - repo: local
     hooks:
-      - id: rustfs-dev-check
-        name: rustfs dev-check
-        entry: make dev-check
+      - id: rustfs-fmt-check
+        name: Rust formatting
+        entry: cargo fmt --all --check
         language: system
         types: [rust]
         pass_filenames: false

CONTRIBUTING.md

@@ -109,24 +109,17 @@ affected boundaries and risks. CI still runs its configured repository gates.
 
 ### 🔒 Git Pre-commit Hooks (optional)
 
-Git hooks are **not** versioned in this repository, so a fresh clone has no
-active pre-commit hook. If you add your own `.git/hooks/pre-commit` (a good
-choice is a one-liner that runs `make pre-commit`), you can mark it executable
-with:
+The optional hook uses the checked-in `.pre-commit-config.yaml`. Install [pre-commit](https://pre-commit.com/#installation), then run this from the checkout or a linked worktree:
 
 ```bash
 make setup-hooks
 ```
 
-Or manually:
+The hook runs `cargo fmt --all --check` when staged files include Rust source. It does not compile the workspace or run tests. Fix formatting with `cargo fmt --all`, inspect and stage the result, then commit again.
 
-```bash
-chmod +x .git/hooks/pre-commit
-```
+`pre-commit install` resolves Git's hook directory for linked worktrees and preserves an existing hook in migration mode. If you use `core.hooksPath`, keep that hook manager and integrate `pre-commit run` there; the installer refuses to silently replace that configuration.
 
-With or without a hook, follow the verification tiers in `AGENTS.md`. Run the
-applicable scoped checks, and reserve `make pre-pr` for broad cross-module
-changes whose impact cannot be bounded by those checks.
+A local hook provides early formatting feedback. With or without it, follow the verification tiers in `AGENTS.md`, run relevant behavioral tests, and satisfy the CI merge gates. `make pre-commit` and `make dev-check` remain explicit broader commands.
 
 ### 📝 Formatting Configuration
 
@@ -138,31 +131,11 @@ fn_call_width = 90
 single_line_let_else_max_width = 100
 ```
 
-### 🚫 Commit Prevention
-
-If you set up a pre-commit hook and your code doesn't meet the formatting requirements, the hook will:
-
-1. **Block the commit** and show clear error messages
-2. **Provide exact commands** to fix the issues
-3. **Guide you through** the resolution process
-
-Example output when formatting fails:
-
-```
-❌ Code formatting check failed!
-💡 Please run 'cargo fmt --all' to format your code before committing.
-
-🔧 Quick fix:
-   cargo fmt --all
-   git add .
-   git commit
-```
-
 ### 🔄 Development Workflow
 
 1. **Make your changes**
 2. **Format your code**: `make fmt` or `cargo fmt --all`
-3. **Run the fast gate**: `make pre-commit` (no clippy, no tests)
+3. **Select relevant checks** using the validation tier in `AGENTS.md`; use `make pre-commit` when its broader fast gate adds useful coverage
 4. **Commit your changes**: `git commit -m "your message"`
 5. **Complete the applicable multi-role adversarial review** for non-exempt changes (see `AGENTS.md`)
 6. **Run applicable scoped checks before opening/updating a PR**; consider
@@ -206,11 +179,12 @@ Configure your IDE to:
 #### Pre-commit hook not running?
 
 ```bash
-# Check if hook is executable
-ls -la .git/hooks/pre-commit
-
-# Make it executable if needed
-chmod +x .git/hooks/pre-commit
+pre-commit validate-config
+pre-commit run --all-files
+# Inspect any configured hook manager; do not overwrite it.
+git config --get core.hooksPath
+# Install if no separate hook manager is configured.
+make setup-hooks
 ```
 
 #### Formatting issues?

scripts/test_security_workflow.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Exercise functional chain dispatch and security evidence without remote VMs."""
+"""Run the security workflow's evidence and result steps without remote VMs."""
 
 from __future__ import annotations
 
@@ -18,20 +18,16 @@ WORKFLOW = ROOT / ".github/workflows/rustfs-security-test.yml"
 CASE_ROW = "| IAM-101 | user CRUD lifecycle | PASS |"
 
 
-def named_steps(job: list[str]) -> dict[str, list[str]]:
-    starts = [i for i, line in enumerate(job) if line.startswith("      - name: ")]
-    return {
-        job[start].split(": ", 1)[1].strip('"'): job[start:end]
-        for start, end in zip(starts, starts[1:] + [len(job)])
-    }
-
-
 class SecurityWorkflowTests(unittest.TestCase):
     def setUp(self) -> None:
         self.source = WORKFLOW.read_text()
         self.job = yaml_block(self.source.splitlines(), "security-test", 2)
         self.assertIsNotNone(self.job)
-        self.steps = named_steps(self.job)
+        starts = [i for i, line in enumerate(self.job) if line.startswith("      - name: ")]
+        self.steps = {
+            self.job[start].split(": ", 1)[1].strip('"'): self.job[start:end]
+            for start, end in zip(starts, starts[1:] + [len(self.job)])
+        }
         self.temp = tempfile.TemporaryDirectory()
         self.addCleanup(self.temp.cleanup)
         self.directory = Path(self.temp.name)
@@ -196,110 +192,6 @@ class SecurityWorkflowTests(unittest.TestCase):
         self.assertNotIn("OLD RUN REPORT", body.read_text())
         self.assertIn("https://github.com/rustfs/rustfs/actions/runs/314159", body.read_text())
 
-    def test_all_ten_suites_hold_the_shared_lock_for_manual_and_chain_runs(self) -> None:
-        for suite in ("upgrade", "s3-compat", "kms", "tier", "storage", "heal", "pool-expand", "security", "replication", "performance"):
-            with self.subTest(suite=suite):
-                source = (ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text().splitlines()
-                # Workflow-level concurrency covers every job, including cleanup,
-                # regardless of trigger or the runner hosting the job.
-                self.assertEqual([
-                    line.strip() for line in yaml_block(source, "concurrency", 0)
-                    if line.strip() and not line.lstrip().startswith("#")
-                ], [
-                    "group: rustfs-shared-functional-tests", "cancel-in-progress: false",
-                ])
-                self.assertIsNotNone(yaml_block(source, "workflow_dispatch", 2))
-                self.assertIsNotNone(yaml_block(source, "repository_dispatch", 2))
-                cleanup_name = "Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"
-                cleanup = named_steps(yaml_block(source, "jobs", 0))[cleanup_name]
-                self.assertTrue(any(line.startswith("        if:") and "always()" in line for line in cleanup))
-
-    def test_root_dispatches_only_upgrade_and_replication_hands_off_after_failure(self) -> None:
-        for failed_attempts, issue_exit, token in ((0, 0, "fixture"), (2, 0, "fixture"), (3, 0, "fixture"), (3, 7, "fixture"), (0, 0, "")):
-            with self.subTest(failed_attempts=failed_attempts, issue_exit=issue_exit, token=bool(token)):
-                self.setUp()
-                fake_bin = self.directory / "bin"
-                fake_bin.mkdir()
-                commands = {
-                    "gh": '''#!/usr/bin/env bash
-set -euo pipefail
-if [ "$1" = api ]; then
-  printf '%s\\n' "$*" >> "$DISPATCHES"
-  attempt=$(wc -l < "$DISPATCHES")
-  [ "$attempt" -gt "$FAILED_ATTEMPTS" ]
-elif [ "$1 $2" = 'issue create' ]; then
-  printf 'issue\\n' >> "$EXECUTED"
-  while [ "$#" -gt 0 ]; do
-    if [ "$1" = --body-file ]; then
-      cat "$2" > "$CAPTURE_BODY"
-      printf '%s\\n' "$2" > "$CAPTURE_BODY_PATH"
-    fi
-    shift
-  done
-  exit "$ISSUE_EXIT"
-else
-  exit 99
-fi
-''',
-                    "sleep": '#!/bin/sh\nprintf "sleep %s\\n" "$1" >> "$EXECUTED"\n',
-                    "ssh": '#!/bin/sh\nprintf "cleanup\\n" >> "$EXECUTED"\n',
-                }
-                for name, contents in commands.items():
-                    command = fake_bin / name
-                    command.write_text(contents)
-                    command.chmod(0o755)
-                dispatches = self.directory / "dispatches"
-                executed = self.directory / "executed"
-                body = self.directory / "issue-body.md"
-                body_path = self.directory / "issue-body-path"
-                self.env.update(
-                    PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", DISPATCHES=str(dispatches),
-                    EXECUTED=str(executed), CAPTURE_BODY=str(body), CAPTURE_BODY_PATH=str(body_path),
-                    FAILED_ATTEMPTS="0", ISSUE_EXIT=str(issue_exit),
-                    RUSTFS_NODES="fixture-node", RUSTFS_SSH_USER="fixture-user",
-                    RUSTFS_NIGHTLY_PACKAGE_URL="https://example.invalid/package.deb",
-                )
-                self.context.update({"secrets.PF_TESTING_GH_TOKEN": "fixture", "inputs.suite": "all"})
-                driver = (ROOT / ".github/workflows/rustfs-functional-chain.yml").read_text()
-                self.steps = named_steps(yaml_block(driver.splitlines(), "start-chain", 2))
-                self.assertEqual(list(self.steps), ["Dispatch first suite (upgrade)"])
-                started = self.run_step("Dispatch first suite (upgrade)")
-                self.assertEqual(started.returncode, 0, started.stderr)
-                self.assertEqual(dispatches.read_text().splitlines(), [
-                    "api --method POST repos/rustfs/rustfs/dispatches -f event_type=rustfs-chain-upgrade -F client_payload[from_suite]=nightly-build",
-                ])
-                dispatches.unlink()
-
-                replication = (ROOT / ".github/workflows/rustfs-replication-test.yml").read_text()
-                job = yaml_block(replication.splitlines(), "replication-test", 2)
-                self.assertFalse(any(line.startswith("    continue-on-error:") for line in job))
-                self.steps = named_steps(job)
-                handoff = "Continue functional chain (next: Performance)"
-                self.assertIn("        if: ${{ always() && github.event_name == 'repository_dispatch' }}", self.steps[handoff])
-                self.assertFalse(any(line.strip().startswith("continue-on-error:") for line in self.steps[handoff]))
-                self.assertIn("        if: always()", self.steps["Cleanup environment (after)"])
-                self.assertLess(list(self.steps).index("Cleanup environment (after)"), list(self.steps).index(handoff))
-                suite = self.directory / "auto-testing/rustfs-replication-test.sh"
-                suite.write_text('#!/bin/sh\nprintf "suite failed\\n" >> "$EXECUTED"\nexit 17\n')
-                failed = self.run_step("Run replication suite")
-                self.assertEqual(failed.returncode, 17, failed.stderr)
-                cleaned = self.run_step("Cleanup environment (after)")
-                self.assertEqual(cleaned.returncode, 0, cleaned.stderr)
-                self.assertEqual(executed.read_text().splitlines(), ["suite failed", "cleanup"])
-                self.env["FAILED_ATTEMPTS"] = str(failed_attempts)
-                self.context["secrets.PF_TESTING_GH_TOKEN"] = token
-                forwarded = self.run_step(handoff)
-                self.assertEqual(forwarded.returncode == 0, bool(token) and failed_attempts < 3, forwarded.stderr)
-                calls = dispatches.read_text().splitlines() if dispatches.exists() else []
-                self.assertEqual(calls, [
-                    "api --method POST repos/rustfs/rustfs/dispatches -f event_type=rustfs-chain-performance -F client_payload[from_suite]=replication",
-                ] * (min(failed_attempts + 1, 3) if token else 0))
-                if failed_attempts == 3:
-                    self.assertIn("could not hand off from **replication** to **Performance**", body.read_text())
-                    self.assertIn("rustfs-chain-performance", body.read_text())
-                    self.assertEqual(executed.read_text().splitlines().count("issue"), 2 if issue_exit else 1)
-                    self.assertFalse(Path(body_path.read_text().strip()).exists())
-
 
 if __name__ == "__main__":
     unittest.main()