test(scanner): parameterize heal evidence oracles

Allow the scanner/heal release evidence e2e helper to bind the active case identity and oracle file through an explicit descriptor instead of hard-coding the background target restart artifact. Extend the checker self-test so a single run can finish multiple registry-declared oracles while release gates remain pending until every required lane is complete.

Co-Authored-By: heihutu <heihutu@gmail.com>

Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

.github/workflows/docker.yml

@@ -121,6 +121,22 @@ jobs:
           create_latest=false
           source_ref="$GITHUB_SHA"
 
+          # Pre-GA policy: until the first stable (vX.Y.Z) tag exists, every
+          # prerelease (alpha/beta/rc) also moves `latest`, so users pulling
+          # `latest` get the newest test build. Once a stable tag is published
+          # this returns false and `latest` follows stable releases only.
+          prerelease_moves_latest() {
+            local stable_tags
+            stable_tags=$(git ls-remote --tags --refs origin 2>/dev/null \
+              | awk '{print $2}' \
+              | grep -E '^refs/tags/v?[0-9]+\.[0-9]+\.[0-9]+$' || true)
+            if [[ -z "$stable_tags" ]]; then
+              return 0
+            fi
+            echo "ℹ️  Stable release tag(s) already exist; prereleases no longer update latest"
+            return 1
+          }
+
           if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
             # Triggered by build workflow completion
             echo "🔗 Triggered by build workflow completion"
@@ -184,8 +200,8 @@ jobs:
                 if [[ "$version" == *"alpha"* ]] || [[ "$version" == *"beta"* ]] || [[ "$version" == *"rc"* ]]; then
                   build_type="prerelease"
                   is_prerelease=true
-                  # Current policy: create latest tags for stable releases and selected prereleases (alpha/beta).
-                  if [[ "$version" == *"alpha"* ]] || [[ "$version" == *"beta"* ]]; then
+                  # Pre-GA policy: prereleases update latest until the first stable tag exists.
+                  if prerelease_moves_latest; then
                     create_latest=true
                     echo "🧪 Building Docker image for prerelease: $version (creating latest tag)"
                   else
@@ -243,8 +259,8 @@ jobs:
               v*alpha*|v*beta*|v*rc*|*alpha*|*beta*|*rc*)
                 build_type="prerelease"
                 is_prerelease=true
-                # Current policy: create latest tags for stable releases and selected prereleases (alpha/beta).
-                if [[ "$version" == *"alpha"* ]] || [[ "$version" == *"beta"* ]]; then
+                # Pre-GA policy: prereleases update latest until the first stable tag exists.
+                if prerelease_moves_latest; then
                   create_latest=true
                   echo "🧪 Building with prerelease version: $input_version (creating latest tag)"
                 else
@@ -394,11 +410,13 @@ jobs:
           TAG_BASE="${VERSION}${VARIANT_SUFFIX}"
           TAGS="${{ env.REGISTRY_DOCKERHUB }}:$TAG_BASE,${{ env.REGISTRY_GHCR }}:$TAG_BASE,${{ env.REGISTRY_QUAY }}:$TAG_BASE"
 
-          # Add channel tags for prereleases and latest for stable
+          # Add latest when requested (stable releases, and prereleases before GA)
           if [[ "$CREATE_LATEST" == "true" ]]; then
-            # Create latest tags for stable releases and selected prereleases when CREATE_LATEST=true.
             TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:latest${VARIANT_SUFFIX},${{ env.REGISTRY_GHCR }}:latest${VARIANT_SUFFIX},${{ env.REGISTRY_QUAY }}:latest${VARIANT_SUFFIX}"
-          elif [[ "$BUILD_TYPE" == "prerelease" ]]; then
+          fi
+
+          # Always add the channel tag for prereleases, independent of latest
+          if [[ "$BUILD_TYPE" == "prerelease" ]]; then
             # Prerelease channel tags (alpha, beta, rc)
             if [[ "$VERSION" == *"alpha"* ]]; then
               CHANNEL="alpha"
@@ -555,7 +573,7 @@ jobs:
             "prerelease")
               echo "🧪 Prerelease Docker image has been built with ${VERSION} tags"
               echo "⚠️  This is a prerelease image - use with caution"
-              # Create latest tags for stable releases and selected prereleases when CREATE_LATEST=true.
+              # Prereleases move latest until the first stable tag exists (pre-GA policy).
               if [[ "$CREATE_LATEST" == "true" ]]; then
                 echo "🏷️  Latest tag has been created for prerelease: $VERSION"
               else

Cargo.lock

@@ -6139,9 +6139,9 @@ checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
 
 [[package]]
 name = "libflate"
-version = "2.3.1"
+version = "2.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4da9b700e758e57152a1fd1c52cbdc5727c1aa6d8743dc1acda917398f1d76c"
+checksum = "561a8da1a50e1428d3c51321dafeca849df992a5bb67720c386131234caba82e"
 dependencies = [
  "adler32",
  "crc32fast",
@@ -10943,9 +10943,9 @@ dependencies = [
 
 [[package]]
 name = "rustfs-uring"
-version = "0.2.1"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0486e62d0efe25db95c00aeacb2da84368adcba299216cda99fcb11328061c84"
+checksum = "b29bc57b4bd62a73f4fae408b536adf578332e50e464797d09dc2382c7cb68c2"
 dependencies = [
  "io-uring",
  "libc",
@@ -12406,7 +12406,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
  "fastrand",
- "getrandom 0.3.4",
+ "getrandom 0.4.3",
  "once_cell",
  "rustix",
  "windows-sys 0.59.0",

crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs

@@ -54,6 +54,23 @@ mod tests {
         test_binary: EvidenceBuild,
     }
 
+    #[derive(Clone, Copy)]
+    struct ScannerHealEvidenceCase {
+        id: &'static str,
+        oracle: &'static str,
+    }
+
+    const BACKGROUND_TARGET_RESTART_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
+        id: "background-target-restart",
+        oracle: "background-target-restart.json",
+    };
+
+    struct RestartEvidenceContext {
+        directory: PathBuf,
+        run: RestartEvidenceRun,
+        case: ScannerHealEvidenceCase,
+    }
+
     fn file_sha256(path: &Path) -> Result<String, Box<dyn Error + Send + Sync>> {
         let mut file = std::fs::File::open(path)?;
         let mut digest = Sha256::new();
@@ -68,10 +85,22 @@ mod tests {
         Ok(digest.finalize().iter().map(|byte| format!("{byte:02x}")).collect())
     }
 
-    fn restart_evidence_run(binary: &Path) -> Result<Option<(PathBuf, RestartEvidenceRun)>, Box<dyn Error + Send + Sync>> {
+    fn restart_evidence_run(
+        binary: &Path,
+        case: ScannerHealEvidenceCase,
+    ) -> Result<Option<RestartEvidenceContext>, Box<dyn Error + Send + Sync>> {
         let Some(directory) = std::env::var_os("RUSTFS_SCANNER_HEAL_RUN_DIR") else {
             return Ok(None);
         };
+        if case.id.is_empty()
+            || case.oracle.is_empty()
+            || !case.oracle.ends_with(".json")
+            || case.oracle.contains('/')
+            || case.oracle.contains('\\')
+            || case.oracle.contains("..")
+        {
+            return Err("invalid scanner/heal evidence case".into());
+        }
         let directory = PathBuf::from(directory);
         let receipt = directory.join("run.json");
         if receipt.metadata()?.len() > 1024 * 1024 {
@@ -91,10 +120,10 @@ mod tests {
             run.test_binary.sha256,
             "test executable must match the run receipt"
         );
-        if directory.join("background-target-restart.json").exists() {
+        if directory.join(case.oracle).exists() {
             return Err("scanner/heal oracle already exists; create a new execution receipt".into());
         }
-        Ok(Some((directory, run)))
+        Ok(Some(RestartEvidenceContext { directory, run, case }))
     }
 
     fn compiled_test_identity() -> serde_json::Value {
@@ -964,7 +993,7 @@ mod tests {
     async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
         let server_binary = rustfs_binary_path();
         let evidence_run = if scenario == InterruptionScenario::BackgroundTargetRestart {
-            restart_evidence_run(&server_binary)?
+            restart_evidence_run(&server_binary, BACKGROUND_TARGET_RESTART_EVIDENCE)?
         } else {
             None
         };
@@ -1613,15 +1642,20 @@ mod tests {
             return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into());
         }
 
-        if let Some((directory, run)) = evidence_run {
+        if let Some(evidence_context) = evidence_run {
             let restarted_pid = cluster.nodes[1].process.as_ref().ok_or("restarted target is absent")?.id();
             assert_ne!(target_pid, restarted_pid, "target must be a new process");
-            assert_eq!(file_sha256(&server_binary)?, run.binary.sha256, "server build changed during restart");
+            assert_eq!(
+                file_sha256(&server_binary)?,
+                evidence_context.run.binary.sha256,
+                "server build changed during restart"
+            );
             let evidence = serde_json::json!({
-                "schema": 1, "case": "background-target-restart", "evidence": "process-restart",
-                "run_id": run.run_id, "source_revision": run.source_revision,
+                "schema": 1, "case": evidence_context.case.id, "evidence": "process-restart",
+                "run_id": evidence_context.run.run_id, "source_revision": evidence_context.run.source_revision,
                 "test_build": compiled_test_identity(),
-                "binary_sha256": run.binary.sha256, "test_binary_sha256": run.test_binary.sha256,
+                "binary_sha256": evidence_context.run.binary.sha256,
+                "test_binary_sha256": evidence_context.run.test_binary.sha256,
                 "topology": {"nodes": cluster.nodes.len(), "drives_per_node": cluster.nodes[0].data_dirs.len()},
                 "pid_before": target_pid, "pid_after": restarted_pid,
                 "objects": evidence_objects, "node_listings": node_listings,
@@ -1633,7 +1667,7 @@ mod tests {
             let mut output = std::fs::OpenOptions::new()
                 .write(true)
                 .create_new(true)
-                .open(directory.join("background-target-restart.json"))?;
+                .open(evidence_context.directory.join(evidence_context.case.oracle))?;
             output.write_all(&data)?;
             output.sync_all()?;
         }

crates/ecstore/Cargo.toml

@@ -226,7 +226,7 @@ metrics = { workspace = true }
 # crates.io. The guard scripts/check_no_tokio_io_uring.sh allows an explicit
 # io-uring integration; only the tokio "io-uring" runtime feature is banned.
 [target.'cfg(target_os = "linux")'.dependencies]
-rustfs-uring = "0.2.1"
+rustfs-uring = "0.2.2"
 
 [target.'cfg(windows)'.dependencies]
 winapi-util.workspace = true

scripts/check_test_wiring.py

@@ -1309,6 +1309,29 @@ class SelfTests(unittest.TestCase):
             self.assertIn("alternate-target-restart.json", read_json(run_dir / "execution.json")["artifacts"])
             self.assertEqual(check_scanner_heal_evidence(root, run_dir, "alternate-target-restart"), [])
 
+    def test_scanner_heal_release_consumes_multiple_registry_oracles(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root, run_dir = self.scanner_heal_fixture(Path(tmp))
+            registry = read_json(root / ".config/scanner-heal-required-tests.json")
+            alternate = dict(registry["cases"]["background-target-restart"])
+            alternate["oracle"] = "alternate-target-restart.json"
+            registry["cases"]["alternate-target-restart"] = alternate
+            write_json(root / ".config/scanner-heal-required-tests.json", registry)
+            oracle = read_json(run_dir / "background-target-restart.json")
+            oracle["case"] = "alternate-target-restart"
+            write_json(run_dir / "alternate-target-restart.json", oracle)
+            (run_dir / "execution.json").unlink()
+
+            finish_scanner_heal_receipt(run_dir, 0, root)
+
+            artifacts = read_json(run_dir / "execution.json")["artifacts"]
+            self.assertIn("background-target-restart.json", artifacts)
+            self.assertIn("alternate-target-restart.json", artifacts)
+            self.assertEqual(check_scanner_heal_evidence(root, run_dir, "alternate-target-restart"), [])
+            errors = check_scanner_heal_evidence(root, run_dir, "release")
+            self.assertEqual(len(errors), 21)
+            self.assertTrue(all(error.startswith("pending ") for error in errors))
+
     def test_scanner_heal_rejects_broken_execution_and_artifacts(self) -> None:
         for fault in ("exit", "missing", "zero", "skipped", "failed", "retry", "filtered", "ignored", "stale",
                       "hash", "binary", "synthetic", "wrong-run", "same-pid", "body", "parts", "listing", "topology"):