From 88dfdda4a0add02613321605cdfbf77236f2477b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=89=E6=AD=A3=E8=B6=85?= Date: Sat, 20 Jun 2026 09:54:36 +0800 Subject: [PATCH] refactor: establish ecstore layout foundation (#3645) --- crates/ecstore/src/api/README.md | 4 + crates/ecstore/src/cluster/README.md | 4 + crates/ecstore/src/core/README.md | 4 + crates/ecstore/src/erasure/README.md | 4 + crates/ecstore/src/layout/mod.rs | 7 + crates/ecstore/src/layout/set_layout.rs | 182 +++++++++++++++++++ crates/ecstore/src/lib.rs | 1 + crates/ecstore/src/metadata/README.md | 4 + crates/ecstore/src/services/README.md | 4 + docs/architecture/ecstore-layout-boundary.md | 53 ++++++ docs/architecture/migration-progress.md | 54 ++++-- 11 files changed, 307 insertions(+), 14 deletions(-) create mode 100644 crates/ecstore/src/api/README.md create mode 100644 crates/ecstore/src/cluster/README.md create mode 100644 crates/ecstore/src/core/README.md create mode 100644 crates/ecstore/src/erasure/README.md create mode 100644 crates/ecstore/src/layout/mod.rs create mode 100644 crates/ecstore/src/layout/set_layout.rs create mode 100644 crates/ecstore/src/metadata/README.md create mode 100644 crates/ecstore/src/services/README.md create mode 100644 docs/architecture/ecstore-layout-boundary.md diff --git a/crates/ecstore/src/api/README.md b/crates/ecstore/src/api/README.md new file mode 100644 index 000000000..5e1452759 --- /dev/null +++ b/crates/ecstore/src/api/README.md @@ -0,0 +1,4 @@ +# ECStore API Layout + +Reserved for facade and compatibility re-export ownership during the ECStore +internal layout migration. No runtime logic lives here yet. diff --git a/crates/ecstore/src/cluster/README.md b/crates/ecstore/src/cluster/README.md new file mode 100644 index 000000000..639bc56e3 --- /dev/null +++ b/crates/ecstore/src/cluster/README.md @@ -0,0 +1,4 @@ +# ECStore Cluster Layout + +Reserved for remote disk, peer, lock, membership, and health-control-plane +ownership after their static boundaries are covered by tests. diff --git a/crates/ecstore/src/core/README.md b/crates/ecstore/src/core/README.md new file mode 100644 index 000000000..6aa10a2d7 --- /dev/null +++ b/crates/ecstore/src/core/README.md @@ -0,0 +1,4 @@ +# ECStore Core Layout + +Reserved for the store facade and object, bucket, list, multipart, and heal +core paths after pure-move compatibility coverage is in place. diff --git a/crates/ecstore/src/erasure/README.md b/crates/ecstore/src/erasure/README.md new file mode 100644 index 000000000..e8614b782 --- /dev/null +++ b/crates/ecstore/src/erasure/README.md @@ -0,0 +1,4 @@ +# ECStore Erasure Layout + +Reserved for erasure coding and bitrot ownership once quorum, parity, and read +integrity checks are pinned by focused tests. diff --git a/crates/ecstore/src/layout/mod.rs b/crates/ecstore/src/layout/mod.rs new file mode 100644 index 000000000..4b1934524 --- /dev/null +++ b/crates/ecstore/src/layout/mod.rs @@ -0,0 +1,7 @@ +//! Static ECStore layout boundaries. +//! +//! This module owns read-only layout descriptors used to keep static set +//! topology separate from runtime `Sets`/`SetDisks` orchestration before any +//! file moves happen. + +pub(crate) mod set_layout; diff --git a/crates/ecstore/src/layout/set_layout.rs b/crates/ecstore/src/layout/set_layout.rs new file mode 100644 index 000000000..63f8d4a31 --- /dev/null +++ b/crates/ecstore/src/layout/set_layout.rs @@ -0,0 +1,182 @@ +use crate::disk::format::{DistributionAlgoVersion, FormatV3}; +use std::collections::HashSet; +use std::io::{Error, Result}; +use uuid::Uuid; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct StaticSetLayoutSnapshot { + pub(crate) deployment_id: Uuid, + pub(crate) set_count: usize, + pub(crate) drives_per_set: usize, + pub(crate) disk_ids: Vec>, + pub(crate) distribution_algo: DistributionAlgoVersion, +} + +impl StaticSetLayoutSnapshot { + pub(crate) fn from_format(format: &FormatV3) -> Self { + let disk_ids = format.erasure.sets.clone(); + + Self { + deployment_id: format.id, + set_count: disk_ids.len(), + drives_per_set: disk_ids.first().map_or(0, Vec::len), + disk_ids, + distribution_algo: format.erasure.distribution_algo.clone(), + } + } + + pub(crate) fn disk_position(&self, disk_id: Uuid) -> Option { + for (set_index, set) in self.disk_ids.iter().enumerate() { + for (disk_index, id) in set.iter().enumerate() { + if *id == disk_id { + return Some(SetDiskPosition { set_index, disk_index }); + } + } + } + + None + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct SetDiskPosition { + pub(crate) set_index: usize, + pub(crate) disk_index: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct RuntimeSetLayoutPlan { + pub(crate) sets: Vec>, + lock_hosts_by_set: Vec>, +} + +impl RuntimeSetLayoutPlan { + pub(crate) fn from_endpoint_hosts(set_count: usize, drives_per_set: usize, endpoint_hosts: &[S]) -> Result + where + S: AsRef, + { + if set_count == 0 || drives_per_set == 0 { + return Err(Error::other("set count and drive count must be non-zero")); + } + + let expected = set_count + .checked_mul(drives_per_set) + .ok_or_else(|| Error::other("set layout size overflow"))?; + if endpoint_hosts.len() != expected { + return Err(Error::other(format!( + "endpoint host count {} does not match set layout {}x{}", + endpoint_hosts.len(), + set_count, + drives_per_set + ))); + } + + let mut sets = Vec::with_capacity(set_count); + let mut lock_hosts_by_set = Vec::with_capacity(set_count); + + for set_index in 0..set_count { + let mut drives = Vec::with_capacity(drives_per_set); + let mut seen_hosts = HashSet::with_capacity(drives_per_set); + let mut lock_hosts = Vec::with_capacity(drives_per_set); + + for disk_index in 0..drives_per_set { + let flat_disk_index = set_index * drives_per_set + disk_index; + let endpoint_host = endpoint_hosts[flat_disk_index].as_ref().to_owned(); + + if seen_hosts.insert(endpoint_host.clone()) { + lock_hosts.push(endpoint_host.clone()); + } + + drives.push(RuntimeSetDrivePlan { + set_index, + disk_index, + flat_disk_index, + endpoint_host, + }); + } + + sets.push(drives); + lock_hosts_by_set.push(lock_hosts); + } + + Ok(Self { sets, lock_hosts_by_set }) + } + + pub(crate) fn lock_hosts_for_set(&self, set_index: usize) -> Option<&[String]> { + self.lock_hosts_by_set.get(set_index).map(Vec::as_slice) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct RuntimeSetDrivePlan { + pub(crate) set_index: usize, + pub(crate) disk_index: usize, + pub(crate) flat_disk_index: usize, + pub(crate) endpoint_host: String, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_eset_001_static_layout_snapshot_preserves_format_distribution() { + let format = FormatV3::new(2, 4); + let snapshot = StaticSetLayoutSnapshot::from_format(&format); + + assert_eq!(snapshot.deployment_id, format.id); + assert_eq!(snapshot.set_count, 2); + assert_eq!(snapshot.drives_per_set, 4); + assert_eq!(snapshot.disk_ids, format.erasure.sets); + assert_eq!(snapshot.distribution_algo, format.erasure.distribution_algo); + + let first_disk_id = format.erasure.sets[0][0]; + let last_disk_id = format.erasure.sets[1][3]; + assert_eq!( + snapshot.disk_position(first_disk_id), + Some(SetDiskPosition { + set_index: 0, + disk_index: 0, + }) + ); + assert_eq!( + snapshot.disk_position(last_disk_id), + Some(SetDiskPosition { + set_index: 1, + disk_index: 3, + }) + ); + assert_eq!(snapshot.disk_position(Uuid::new_v4()), None); + } + + #[test] + fn test_eset_002_runtime_plan_preserves_flat_disk_and_lock_host_mapping() { + let endpoint_hosts = [ + "node-a:9000", + "node-a:9000", + "node-b:9000", + "node-c:9000", + "node-d:9000", + "node-d:9000", + ]; + + let plan = RuntimeSetLayoutPlan::from_endpoint_hosts(2, 3, &endpoint_hosts) + .expect("valid endpoint host count should build a runtime set layout plan"); + + assert_eq!(plan.sets.len(), 2); + assert_eq!(plan.sets[0].len(), 3); + assert_eq!(plan.sets[0][0].flat_disk_index, 0); + assert_eq!(plan.sets[0][2].flat_disk_index, 2); + assert_eq!(plan.sets[1][0].flat_disk_index, 3); + assert_eq!(plan.sets[1][2].set_index, 1); + assert_eq!(plan.sets[1][2].disk_index, 2); + assert_eq!(plan.sets[1][2].endpoint_host, "node-d:9000"); + + let first_set_hosts = vec!["node-a:9000".to_owned(), "node-b:9000".to_owned()]; + let second_set_hosts = vec!["node-c:9000".to_owned(), "node-d:9000".to_owned()]; + assert_eq!(plan.lock_hosts_for_set(0), Some(first_set_hosts.as_slice())); + assert_eq!(plan.lock_hosts_for_set(1), Some(second_set_hosts.as_slice())); + assert_eq!(plan.lock_hosts_for_set(2), None); + assert!(RuntimeSetLayoutPlan::from_endpoint_hosts(2, 3, &endpoint_hosts[..5]).is_err()); + } +} diff --git a/crates/ecstore/src/lib.rs b/crates/ecstore/src/lib.rs index 4632f56fc..9b2195187 100644 --- a/crates/ecstore/src/lib.rs +++ b/crates/ecstore/src/lib.rs @@ -30,6 +30,7 @@ pub mod endpoints; pub mod erasure_coding; pub mod error; pub mod global; +pub(crate) mod layout; pub mod metrics_realtime; pub mod notification_sys; pub mod object_api; diff --git a/crates/ecstore/src/metadata/README.md b/crates/ecstore/src/metadata/README.md new file mode 100644 index 000000000..6ec195754 --- /dev/null +++ b/crates/ecstore/src/metadata/README.md @@ -0,0 +1,4 @@ +# ECStore Metadata Layout + +Reserved for bucket metadata, config object-store, and data-usage ownership +after persisted metadata compatibility is pinned by tests. diff --git a/crates/ecstore/src/services/README.md b/crates/ecstore/src/services/README.md new file mode 100644 index 000000000..87eecc16f --- /dev/null +++ b/crates/ecstore/src/services/README.md @@ -0,0 +1,4 @@ +# ECStore Services Layout + +Reserved for lifecycle, replication, tier, notification, rebalance, and metrics +service ownership after background side effects are covered. diff --git a/docs/architecture/ecstore-layout-boundary.md b/docs/architecture/ecstore-layout-boundary.md new file mode 100644 index 000000000..d3a769ebf --- /dev/null +++ b/docs/architecture/ecstore-layout-boundary.md @@ -0,0 +1,53 @@ +# ECStore Layout Boundary + +This document records the `E-001` and `E-SET-001` foundation slice for the +architecture migration. + +## Directory Skeleton + +The ECStore migration uses these internal ownership buckets before any pure +file moves: + +- `api`: facade and compatibility re-export ownership. +- `core`: store facade, object, bucket, list, multipart, and heal paths. +- `layout`: static endpoint, disk, pool, and set layout descriptions. +- `disk`: local disk, format, health, and disk error ownership. +- `erasure`: erasure coding and bitrot ownership. +- `metadata`: bucket metadata, config object-store, and data-usage ownership. +- `cluster`: remote disk, peer, lock, membership, and health-control-plane + ownership. +- `services`: lifecycle, replication, tier, notification, rebalance, and + metrics service ownership. + +## Static Set Layout + +Static layout is derived from persisted `FormatV3` data and input endpoint +expansion. It may describe: + +- deployment id; +- set count and drives per set; +- disk UUID positions inside `format.erasure.sets`; +- distribution algorithm; +- endpoint grouping produced before runtime disk initialization. + +Static layout must not own disk handles, lock clients, reconnect loops, repair +state, or shutdown signaling. + +## Runtime Set Orchestration + +Runtime orchestration remains owned by `Sets` and `SetDisks` until a later pure +move. It may describe: + +- flat disk index to `(set_index, disk_index)` mapping; +- per-set local disk replacement after distributed setup detection; +- per-set lock-client host deduplication; +- endpoint reconnect monitoring and runtime shutdown signaling; +- read/write/heal/list orchestration over initialized disks. + +## Preservation Rules + +- Object-to-set hashing and distribution algorithm selection must not change. +- Format `sets` ordering and disk UUID position lookup must not change. +- Local disk replacement and lock-client mapping stay runtime-only. +- Later file moves must keep old public paths or add explicit compatibility + coverage before deleting them. diff --git a/docs/architecture/migration-progress.md b/docs/architecture/migration-progress.md index 635070d51..9fe0c2d77 100644 --- a/docs/architecture/migration-progress.md +++ b/docs/architecture/migration-progress.md @@ -5,16 +5,16 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block ## Current Context - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) -- Branch: `overtrue/arch-extension-runtime-snapshots` -- Baseline: `overtrue/arch-ops-profiler-runtime-contract` - (`6d4ff92f398a9decdce53a7203744c7c2a70a4e9`). -- Stacked on: local R-032, which is stacked on rustfs/rustfs#3642. -- PR type for this branch: `contract` +- Branch: `overtrue/arch-ecstore-layout-foundation` +- Baseline: `overtrue/arch-extension-runtime-snapshots` + (`6ee557f63c4f8eb45334af23a6b3a32e0ee009e9`). +- Stacked on: local R-033, which is stacked on local R-032 and rustfs/rustfs#3642. +- PR type for this branch: `pure-move` - Runtime behavior changes: none. -- Rust code changes: expose builtin diagnostics/profiler runtime capability - snapshots through the admin extension catalog response. +- Rust code changes: add the ECStore internal layout skeleton and read-only set + layout boundary tests. - CI/script changes: none. -- Docs changes: record the R-033 extension runtime snapshot slice. +- Docs changes: record the ECStore layout boundary and E-001/E-SET-001 slice. ## Phase 0 Tasks @@ -2211,20 +2211,34 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block check, migration/layer guards, formatting, diff hygiene, Rust risk scan, branch freshness check, pre-commit quality gate, and three-expert review. +- [x] `E-001/E-SET-001` Add ECStore layout skeleton and set-layout boundary. + - Do: create the ECStore internal layout ownership buckets and pin static set + layout versus runtime `Sets`/`SetDisks` orchestration boundaries before any + file moves. + - Acceptance: the skeleton documents future ownership buckets, static format + set distribution is preserved, and runtime flat disk plus per-set lock-host + mapping is described by focused tests. + - Must preserve: format distribution, object-to-set hashing owner, local disk + replacement, lock client mapping, existing public module paths, and runtime + `Sets`/`SetDisks` behavior. + - Verification: focused ECStore set layout tests, ECStore/RustFS compile + checks, migration/layer guards, formatting, diff hygiene, Rust risk scan, + branch freshness check, pre-commit quality gate, and three-expert review. + ## Next PRs -1. `pure-move`: continue pruning residual embedded startup-only orchestration +1. `pure-move`: move ECStore layout files into the new layout bucket with + compatibility coverage after E-001/E-SET-001 lands. +2. `pure-move`: continue pruning residual embedded startup-only orchestration once the lifecycle helpers are merged. -2. `contract`: add the next read-only extension handoff once admin/reporting - consumers need additional extension runtime snapshots. ## Pre-Push Review Log | Expert | Status | Notes | |---|---|---| -| Quality/architecture | passed | R-033 exposes only read-only diagnostics/profiler capability snapshots through the admin extension catalog response. | -| Migration preservation | passed | Catalog auth, plugin instance listing, profiler/diagnostics execution paths, and external plugin flow status semantics remain unchanged. | -| Testing/verification | passed | Focused admin catalog and targets runtime checks, lib check, guards, formatting, diff hygiene, Rust risk scan, and full pre-commit passed. | +| Quality/architecture | passed | E-001/E-SET-001 adds only ECStore layout skeleton boundaries and read-only set-layout contracts before file moves. | +| Migration preservation | passed | Format distribution, local disk replacement, lock client mapping, public module paths, and runtime set behavior remain unchanged. | +| Testing/verification | passed | Focused ECStore set-layout checks, compile checks, guards, formatting, diff hygiene, Rust risk scan, and full pre-commit passed. | ## Verification Notes @@ -2274,6 +2288,18 @@ Passed before push: - `make pre-commit`: passed. - Three-expert review: passed. +- Issue #660 E-001/E-SET-001 current slice: + - `cargo test -p rustfs-ecstore test_eset -- --nocapture`: passed. + - `cargo check -p rustfs-ecstore -p rustfs -p rustfs-heal`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_layer_dependencies.sh`: passed. + - `cargo fmt --all --check`: passed. + - `git diff --check`: passed. + - Rust risk scan on changed Rust files: passed; only test-only + expectation paths were present. + - `make pre-commit`: passed. + - Three-expert review: passed. + - Issue #660 X-012 current slice: - `cargo test -p rustfs-extension-schema`: passed. - `cargo check -p rustfs-extension-schema`: passed.