refactor: establish ecstore layout foundation (#3645)

crates/ecstore/src/api/README.md

@@ -0,0 +1,4 @@
+# ECStore API Layout
+
+Reserved for facade and compatibility re-export ownership during the ECStore
+internal layout migration. No runtime logic lives here yet.

crates/ecstore/src/cluster/README.md

@@ -0,0 +1,4 @@
+# ECStore Cluster Layout
+
+Reserved for remote disk, peer, lock, membership, and health-control-plane
+ownership after their static boundaries are covered by tests.

crates/ecstore/src/core/README.md

@@ -0,0 +1,4 @@
+# ECStore Core Layout
+
+Reserved for the store facade and object, bucket, list, multipart, and heal
+core paths after pure-move compatibility coverage is in place.

crates/ecstore/src/erasure/README.md

@@ -0,0 +1,4 @@
+# ECStore Erasure Layout
+
+Reserved for erasure coding and bitrot ownership once quorum, parity, and read
+integrity checks are pinned by focused tests.

crates/ecstore/src/layout/mod.rs

@@ -0,0 +1,7 @@
+//! Static ECStore layout boundaries.
+//!
+//! This module owns read-only layout descriptors used to keep static set
+//! topology separate from runtime `Sets`/`SetDisks` orchestration before any
+//! file moves happen.
+
+pub(crate) mod set_layout;

crates/ecstore/src/layout/set_layout.rs

@@ -0,0 +1,182 @@
+use crate::disk::format::{DistributionAlgoVersion, FormatV3};
+use std::collections::HashSet;
+use std::io::{Error, Result};
+use uuid::Uuid;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(crate) struct StaticSetLayoutSnapshot {
+    pub(crate) deployment_id: Uuid,
+    pub(crate) set_count: usize,
+    pub(crate) drives_per_set: usize,
+    pub(crate) disk_ids: Vec<Vec<Uuid>>,
+    pub(crate) distribution_algo: DistributionAlgoVersion,
+}
+
+impl StaticSetLayoutSnapshot {
+    pub(crate) fn from_format(format: &FormatV3) -> Self {
+        let disk_ids = format.erasure.sets.clone();
+
+        Self {
+            deployment_id: format.id,
+            set_count: disk_ids.len(),
+            drives_per_set: disk_ids.first().map_or(0, Vec::len),
+            disk_ids,
+            distribution_algo: format.erasure.distribution_algo.clone(),
+        }
+    }
+
+    pub(crate) fn disk_position(&self, disk_id: Uuid) -> Option<SetDiskPosition> {
+        for (set_index, set) in self.disk_ids.iter().enumerate() {
+            for (disk_index, id) in set.iter().enumerate() {
+                if *id == disk_id {
+                    return Some(SetDiskPosition { set_index, disk_index });
+                }
+            }
+        }
+
+        None
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) struct SetDiskPosition {
+    pub(crate) set_index: usize,
+    pub(crate) disk_index: usize,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(crate) struct RuntimeSetLayoutPlan {
+    pub(crate) sets: Vec<Vec<RuntimeSetDrivePlan>>,
+    lock_hosts_by_set: Vec<Vec<String>>,
+}
+
+impl RuntimeSetLayoutPlan {
+    pub(crate) fn from_endpoint_hosts<S>(set_count: usize, drives_per_set: usize, endpoint_hosts: &[S]) -> Result<Self>
+    where
+        S: AsRef<str>,
+    {
+        if set_count == 0 || drives_per_set == 0 {
+            return Err(Error::other("set count and drive count must be non-zero"));
+        }
+
+        let expected = set_count
+            .checked_mul(drives_per_set)
+            .ok_or_else(|| Error::other("set layout size overflow"))?;
+        if endpoint_hosts.len() != expected {
+            return Err(Error::other(format!(
+                "endpoint host count {} does not match set layout {}x{}",
+                endpoint_hosts.len(),
+                set_count,
+                drives_per_set
+            )));
+        }
+
+        let mut sets = Vec::with_capacity(set_count);
+        let mut lock_hosts_by_set = Vec::with_capacity(set_count);
+
+        for set_index in 0..set_count {
+            let mut drives = Vec::with_capacity(drives_per_set);
+            let mut seen_hosts = HashSet::with_capacity(drives_per_set);
+            let mut lock_hosts = Vec::with_capacity(drives_per_set);
+
+            for disk_index in 0..drives_per_set {
+                let flat_disk_index = set_index * drives_per_set + disk_index;
+                let endpoint_host = endpoint_hosts[flat_disk_index].as_ref().to_owned();
+
+                if seen_hosts.insert(endpoint_host.clone()) {
+                    lock_hosts.push(endpoint_host.clone());
+                }
+
+                drives.push(RuntimeSetDrivePlan {
+                    set_index,
+                    disk_index,
+                    flat_disk_index,
+                    endpoint_host,
+                });
+            }
+
+            sets.push(drives);
+            lock_hosts_by_set.push(lock_hosts);
+        }
+
+        Ok(Self { sets, lock_hosts_by_set })
+    }
+
+    pub(crate) fn lock_hosts_for_set(&self, set_index: usize) -> Option<&[String]> {
+        self.lock_hosts_by_set.get(set_index).map(Vec::as_slice)
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(crate) struct RuntimeSetDrivePlan {
+    pub(crate) set_index: usize,
+    pub(crate) disk_index: usize,
+    pub(crate) flat_disk_index: usize,
+    pub(crate) endpoint_host: String,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_eset_001_static_layout_snapshot_preserves_format_distribution() {
+        let format = FormatV3::new(2, 4);
+        let snapshot = StaticSetLayoutSnapshot::from_format(&format);
+
+        assert_eq!(snapshot.deployment_id, format.id);
+        assert_eq!(snapshot.set_count, 2);
+        assert_eq!(snapshot.drives_per_set, 4);
+        assert_eq!(snapshot.disk_ids, format.erasure.sets);
+        assert_eq!(snapshot.distribution_algo, format.erasure.distribution_algo);
+
+        let first_disk_id = format.erasure.sets[0][0];
+        let last_disk_id = format.erasure.sets[1][3];
+        assert_eq!(
+            snapshot.disk_position(first_disk_id),
+            Some(SetDiskPosition {
+                set_index: 0,
+                disk_index: 0,
+            })
+        );
+        assert_eq!(
+            snapshot.disk_position(last_disk_id),
+            Some(SetDiskPosition {
+                set_index: 1,
+                disk_index: 3,
+            })
+        );
+        assert_eq!(snapshot.disk_position(Uuid::new_v4()), None);
+    }
+
+    #[test]
+    fn test_eset_002_runtime_plan_preserves_flat_disk_and_lock_host_mapping() {
+        let endpoint_hosts = [
+            "node-a:9000",
+            "node-a:9000",
+            "node-b:9000",
+            "node-c:9000",
+            "node-d:9000",
+            "node-d:9000",
+        ];
+
+        let plan = RuntimeSetLayoutPlan::from_endpoint_hosts(2, 3, &endpoint_hosts)
+            .expect("valid endpoint host count should build a runtime set layout plan");
+
+        assert_eq!(plan.sets.len(), 2);
+        assert_eq!(plan.sets[0].len(), 3);
+        assert_eq!(plan.sets[0][0].flat_disk_index, 0);
+        assert_eq!(plan.sets[0][2].flat_disk_index, 2);
+        assert_eq!(plan.sets[1][0].flat_disk_index, 3);
+        assert_eq!(plan.sets[1][2].set_index, 1);
+        assert_eq!(plan.sets[1][2].disk_index, 2);
+        assert_eq!(plan.sets[1][2].endpoint_host, "node-d:9000");
+
+        let first_set_hosts = vec!["node-a:9000".to_owned(), "node-b:9000".to_owned()];
+        let second_set_hosts = vec!["node-c:9000".to_owned(), "node-d:9000".to_owned()];
+        assert_eq!(plan.lock_hosts_for_set(0), Some(first_set_hosts.as_slice()));
+        assert_eq!(plan.lock_hosts_for_set(1), Some(second_set_hosts.as_slice()));
+        assert_eq!(plan.lock_hosts_for_set(2), None);
+        assert!(RuntimeSetLayoutPlan::from_endpoint_hosts(2, 3, &endpoint_hosts[..5]).is_err());
+    }
+}

crates/ecstore/src/lib.rs

@@ -30,6 +30,7 @@ pub mod endpoints;
 pub mod erasure_coding;
 pub mod error;
 pub mod global;
+pub(crate) mod layout;
 pub mod metrics_realtime;
 pub mod notification_sys;
 pub mod object_api;

crates/ecstore/src/metadata/README.md

@@ -0,0 +1,4 @@
+# ECStore Metadata Layout
+
+Reserved for bucket metadata, config object-store, and data-usage ownership
+after persisted metadata compatibility is pinned by tests.

crates/ecstore/src/services/README.md

@@ -0,0 +1,4 @@
+# ECStore Services Layout
+
+Reserved for lifecycle, replication, tier, notification, rebalance, and metrics
+service ownership after background side effects are covered.

docs/architecture/ecstore-layout-boundary.md

@@ -0,0 +1,53 @@
+# ECStore Layout Boundary
+
+This document records the `E-001` and `E-SET-001` foundation slice for the
+architecture migration.
+
+## Directory Skeleton
+
+The ECStore migration uses these internal ownership buckets before any pure
+file moves:
+
+- `api`: facade and compatibility re-export ownership.
+- `core`: store facade, object, bucket, list, multipart, and heal paths.
+- `layout`: static endpoint, disk, pool, and set layout descriptions.
+- `disk`: local disk, format, health, and disk error ownership.
+- `erasure`: erasure coding and bitrot ownership.
+- `metadata`: bucket metadata, config object-store, and data-usage ownership.
+- `cluster`: remote disk, peer, lock, membership, and health-control-plane
+  ownership.
+- `services`: lifecycle, replication, tier, notification, rebalance, and
+  metrics service ownership.
+
+## Static Set Layout
+
+Static layout is derived from persisted `FormatV3` data and input endpoint
+expansion. It may describe:
+
+- deployment id;
+- set count and drives per set;
+- disk UUID positions inside `format.erasure.sets`;
+- distribution algorithm;
+- endpoint grouping produced before runtime disk initialization.
+
+Static layout must not own disk handles, lock clients, reconnect loops, repair
+state, or shutdown signaling.
+
+## Runtime Set Orchestration
+
+Runtime orchestration remains owned by `Sets` and `SetDisks` until a later pure
+move. It may describe:
+
+- flat disk index to `(set_index, disk_index)` mapping;
+- per-set local disk replacement after distributed setup detection;
+- per-set lock-client host deduplication;
+- endpoint reconnect monitoring and runtime shutdown signaling;
+- read/write/heal/list orchestration over initialized disks.
+
+## Preservation Rules
+
+- Object-to-set hashing and distribution algorithm selection must not change.
+- Format `sets` ordering and disk UUID position lookup must not change.
+- Local disk replacement and lock-client mapping stay runtime-only.
+- Later file moves must keep old public paths or add explicit compatibility
+  coverage before deleting them.

docs/architecture/migration-progress.md

@@ -5,16 +5,16 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
 ## Current Context
 
 - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
-- Branch: `overtrue/arch-extension-runtime-snapshots`
-- Baseline: `overtrue/arch-ops-profiler-runtime-contract`
-  (`6d4ff92f398a9decdce53a7203744c7c2a70a4e9`).
-- Stacked on: local R-032, which is stacked on rustfs/rustfs#3642.
-- PR type for this branch: `contract`
+- Branch: `overtrue/arch-ecstore-layout-foundation`
+- Baseline: `overtrue/arch-extension-runtime-snapshots`
+  (`6ee557f63c4f8eb45334af23a6b3a32e0ee009e9`).
+- Stacked on: local R-033, which is stacked on local R-032 and rustfs/rustfs#3642.
+- PR type for this branch: `pure-move`
 - Runtime behavior changes: none.
-- Rust code changes: expose builtin diagnostics/profiler runtime capability
-  snapshots through the admin extension catalog response.
+- Rust code changes: add the ECStore internal layout skeleton and read-only set
+  layout boundary tests.
 - CI/script changes: none.
-- Docs changes: record the R-033 extension runtime snapshot slice.
+- Docs changes: record the ECStore layout boundary and E-001/E-SET-001 slice.
 
 ## Phase 0 Tasks
 
@@ -2211,20 +2211,34 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
     check, migration/layer guards, formatting, diff hygiene, Rust risk scan,
     branch freshness check, pre-commit quality gate, and three-expert review.
 
+- [x] `E-001/E-SET-001` Add ECStore layout skeleton and set-layout boundary.
+  - Do: create the ECStore internal layout ownership buckets and pin static set
+    layout versus runtime `Sets`/`SetDisks` orchestration boundaries before any
+    file moves.
+  - Acceptance: the skeleton documents future ownership buckets, static format
+    set distribution is preserved, and runtime flat disk plus per-set lock-host
+    mapping is described by focused tests.
+  - Must preserve: format distribution, object-to-set hashing owner, local disk
+    replacement, lock client mapping, existing public module paths, and runtime
+    `Sets`/`SetDisks` behavior.
+  - Verification: focused ECStore set layout tests, ECStore/RustFS compile
+    checks, migration/layer guards, formatting, diff hygiene, Rust risk scan,
+    branch freshness check, pre-commit quality gate, and three-expert review.
+
 ## Next PRs
 
-1. `pure-move`: continue pruning residual embedded startup-only orchestration
+1. `pure-move`: move ECStore layout files into the new layout bucket with
+   compatibility coverage after E-001/E-SET-001 lands.
+2. `pure-move`: continue pruning residual embedded startup-only orchestration
    once the lifecycle helpers are merged.
-2. `contract`: add the next read-only extension handoff once admin/reporting
-   consumers need additional extension runtime snapshots.
 
 ## Pre-Push Review Log
 
 | Expert | Status | Notes |
 |---|---|---|
-| Quality/architecture | passed | R-033 exposes only read-only diagnostics/profiler capability snapshots through the admin extension catalog response. |
-| Migration preservation | passed | Catalog auth, plugin instance listing, profiler/diagnostics execution paths, and external plugin flow status semantics remain unchanged. |
-| Testing/verification | passed | Focused admin catalog and targets runtime checks, lib check, guards, formatting, diff hygiene, Rust risk scan, and full pre-commit passed. |
+| Quality/architecture | passed | E-001/E-SET-001 adds only ECStore layout skeleton boundaries and read-only set-layout contracts before file moves. |
+| Migration preservation | passed | Format distribution, local disk replacement, lock client mapping, public module paths, and runtime set behavior remain unchanged. |
+| Testing/verification | passed | Focused ECStore set-layout checks, compile checks, guards, formatting, diff hygiene, Rust risk scan, and full pre-commit passed. |
 
 ## Verification Notes
 
@@ -2274,6 +2288,18 @@ Passed before push:
   - `make pre-commit`: passed.
   - Three-expert review: passed.
 
+- Issue #660 E-001/E-SET-001 current slice:
+  - `cargo test -p rustfs-ecstore test_eset -- --nocapture`: passed.
+  - `cargo check -p rustfs-ecstore -p rustfs -p rustfs-heal`: passed.
+  - `./scripts/check_architecture_migration_rules.sh`: passed.
+  - `./scripts/check_layer_dependencies.sh`: passed.
+  - `cargo fmt --all --check`: passed.
+  - `git diff --check`: passed.
+  - Rust risk scan on changed Rust files: passed; only test-only
+    expectation paths were present.
+  - `make pre-commit`: passed.
+  - Three-expert review: passed.
+
 - Issue #660 X-012 current slice:
   - `cargo test -p rustfs-extension-schema`: passed.
   - `cargo check -p rustfs-extension-schema`: passed.