test(ilm): re-enable test_transition_and_restore_flows; fix test-util disk-open and restore error-path lock (#4945)

test(ilm): re-enable test_transition_and_restore_flows; fix test-util disk-open and restore error-path lock (rustfs/backlog#1303)

The excluded test's 'missing xl.meta ... on disk2' was NOT an EC
metadata-distribution issue: after a transition all four shard disks hold
a fully consistent xl.meta (verified by decoding each shard). The panic
came from the tier test util's open_disk, which hardcoded disk_index 0
for every disk path; LocalDisk::new validates the endpoint's
(set_idx, disk_idx) against the disk's own format.json and rejected every
non-slot-0 disk with InconsistentDisk, which read_transition_meta
collapsed into 'missing xl.meta'. Derive the real indices from
format.json instead. This also un-breaks free_version_count /
wait_for_free_version_absence for non-first disks (silently 0 before).

With that fixed, the test advanced to the #4877 restore self-deadlock,
whose main paths #4886 already fixed. Complete that fix on the one path
it missed: update_restore_metadata (the restore-failure metadata
rewrite) still rebuilt copy_object options with no_lock=false and would
re-acquire the object write lock the restore handler already holds.
Propagate the caller's no_lock there too.

Remove the test from the serial-lane exclusion list; the four remaining
exclusions are unrelated known issues and stay.

.github/workflows/ci.yml

@@ -221,6 +221,9 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           cache-save-if: ${{ github.ref == 'refs/heads/main' }}
 
+      # test_transition_and_restore_flows was re-enabled by rustfs/backlog#1303:
+      # its "missing xl.meta on disk2" was a test-util bug (open_disk hardcoded
+      # disk_index 0), not an EC metadata-distribution issue.
       # restore_object_usecase_reports_ongoing_conflict_and_completion was
       # re-enabled by backlog#1304 (restore accepts serialize on a short CAS
       # guard; the copy-back no longer holds the #4877 whole-copy-back lock,
@@ -230,9 +233,6 @@ jobs:
       # they keep #[ignore] with a backlog reference):
       #   - test_noncurrent_{expiry,transition}_still_works_after_immediate_compensation_transition:
       #     noncurrent transition/expiry after an immediate compensation transition.
-      #   - test_transition_and_restore_flows: transition metadata is missing on
-      #     one drive (assert_transition_meta_consistent: "missing xl.meta ... on
-      #     disk2") - an EC metadata-distribution issue, not the restore lock.
       #   - test_restore_chain_local_read_expiry_keeps_remote_and_allows_re_restore:
       #     DeleteRestoredAction sets opts.transition.expire_restored, but no
       #     delete path reads that flag, so cleanup deletes the whole object
@@ -242,7 +242,7 @@ jobs:
         run: |
           cargo nextest run -j1 --run-ignored ignored-only \
             -p rustfs-scanner -p rustfs \
-            -E '(binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))) and not (test(test_transition_and_restore_flows) or test(test_noncurrent_expiry_still_works_after_immediate_compensation_transition) or test(test_noncurrent_transition_still_works_after_immediate_compensation_transition) or test(test_restore_chain_local_read_expiry_keeps_remote_and_allows_re_restore))'
+            -E '(binary(lifecycle_integration_test) or (package(rustfs) and test(lifecycle_transition_api_test))) and not (test(test_noncurrent_expiry_still_works_after_immediate_compensation_transition) or test(test_noncurrent_transition_still_works_after_immediate_compensation_transition) or test(test_restore_chain_local_read_expiry_keeps_remote_and_allows_re_restore))'
 
   test-and-lint-rio-v2:
     name: Test and Lint (rio-v2)

crates/ecstore/src/services/tier/test_util.rs

@@ -67,7 +67,8 @@ use uuid::Uuid;
 
 use crate::client::transition_api::{ReadCloser, ReaderImpl};
 use crate::disk::endpoint::Endpoint;
-use crate::disk::{DiskAPI, DiskOption, STORAGE_FORMAT_FILE, new_disk};
+use crate::disk::format::FormatV3;
+use crate::disk::{DiskAPI, DiskOption, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE, new_disk};
 use crate::services::tier::tier::TierConfigMgr;
 use crate::services::tier::tier_config::{TierConfig, TierMinIO, TierType};
 use crate::services::tier::warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options};
@@ -498,10 +499,19 @@ pub struct TransitionMeta {
 }
 
 async fn open_disk(disk_path: &Path) -> Option<crate::disk::DiskStore> {
+    // `LocalDisk::new` rejects an endpoint whose (set_idx, disk_idx) disagrees
+    // with the position recorded in the disk's own format.json, so derive the
+    // real indices instead of assuming slot 0 (rustfs/backlog#1303).
+    let format_data = tokio::fs::read(disk_path.join(RUSTFS_META_BUCKET).join(FORMAT_CONFIG_FILE))
+        .await
+        .ok()?;
+    let format = FormatV3::try_from(format_data.as_slice()).ok()?;
+    let (set_idx, disk_idx) = format.find_disk_index_by_disk_id(format.erasure.this).ok()?;
+
     let mut endpoint = Endpoint::try_from(disk_path.to_str()?).ok()?;
     endpoint.set_pool_index(0);
-    endpoint.set_set_index(0);
-    endpoint.set_disk_index(0);
+    endpoint.set_set_index(set_idx);
+    endpoint.set_disk_index(disk_idx);
     new_disk(
         &endpoint,
         &DiskOption {

crates/scanner/tests/lifecycle_integration_test.rs

@@ -734,7 +734,7 @@ mod serial_tests {
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     #[serial]
-    #[ignore = "FAILING on main: excluded from the serial ILM lane pending a fix, see rustfs/backlog#1148 (ilm-1 partial)"]
+    #[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
     async fn test_transition_and_restore_flows() {
         let (disk_paths, ecstore) = setup_test_env().await;