fix(ecstore): fence anchored format publication retries

Select the durable first-slot plan before fanout and complete immutable anchored plans across retries. Retain physical-root namespace ownership through native publication, retirement and cache invalidation, rejecting conflicting or retired control records.

Reuse main's unary RPC lifecycle recorder for validated outcomes and exact request, byte and cancellation accounting.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=dde32c85d39a3c2eb1cad6912861883af2b3554fc6d04b86b3aeb2afbcc86c7e
-sha256-linux=155b526f75743796c18c4c5202bbef3a1770b0ea4ba18b32910317a957ad36f2
+sha256-darwin=0374e6b9b061dd5d622686301386232a4aa1cb22eedc18af70d2a7411bc66f8d
+sha256-linux=f2cee240f7a744ae991971820f48a6c9c28d19f525e8b2345fcb76eeaaf1e382

.config/e2e-smoke-selection.txt

@@ -1 +1 @@
-sha256=caccbc5bcbfbe1b7548fbd184f1a6ae3aedec7cb84ed862d62dddbe27e75271f
+sha256=21e9a1561ac20382503c38353bbb9b830658534a4f277c254a42d0afc6ebed8b

.config/make/tests.mak

@@ -83,7 +83,6 @@ script-tests: ## Run shell script tests
 	$(RUSTFS_PYTHON_BIN) ./scripts/ci_gate.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_connect_release_boundary.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_test_artifact.py
-	$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_merge_group_proof.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_test_shards.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_merge_queue.py
 	$(RUSTFS_PYTHON_BIN) -m unittest discover -s scripts/ci -p 'test_*.py'

.github/workflows/README.md

@@ -53,7 +53,7 @@ A hosted runner is also the only choice for a few things the pods cannot do: `mo
 These are the patterns that have cost real money here without validating anything more:
 
 - An `if: always()` summary or alert job on a self-hosted label inside a `workflow_run`-triggered workflow. The upstream workflow completes many times a day, the real jobs skip, and the summary still starts a pod each time. Gate it on the upstream job having run, and keep it on `ubuntu-latest`.
-- Per-push workflows on `main`. A merge triggers cache warming and a `ci.yml` run, which validates the merged tree in full unless a merge-queue run already validated that tree; `build.yml` development builds are scheduled or manual; with tens of merges a day that is as much self-hosted time as all pull requests together. Anything that does not verify the merged tree itself belongs on a schedule or behind a `paths` filter.
+- Per-push workflows on `main`. A merge triggers the applicable `ci.yml` matrix and cache warming; `build.yml` development builds are scheduled or manual; with tens of merges a day that is as much self-hosted time as all pull requests together. Anything that does not verify the merged tree itself belongs on a schedule or behind a `paths` filter.
 - `apt-get install` of tools the hosted image already ships. It is only needed on the minimal pod images.
 - A throttled `CARGO_BUILD_JOBS` on a 14-CPU pod. The limit exists for link-storm reasons documented in `ci.yml`; measure before raising it, and measure before lowering it further.
 - Tiny jobs on ARC pods. Each one pays roughly a minute of pod start-up before its first step; a compile-free job that takes ten seconds costs more to schedule than to run.

.github/workflows/ci.yml

@@ -88,9 +88,6 @@ jobs:
     if: github.event_name != 'pull_request' || github.event.action != 'closed'
     runs-on: ubuntu-latest
     timeout-minutes: 10
-    permissions:
-      contents: read
-      actions: read
     outputs:
       mode: ${{ steps.scope.outputs.mode }}
       plan: ${{ steps.scope.outputs.plan }}
@@ -110,14 +107,7 @@ jobs:
           CI_BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }}
           CI_EVENT_ACTION: ${{ github.event.action }}
           CI_REHEARSE_SHARDS: ${{ inputs.rehearse_workspace_shards && 'true' || 'false' }}
-          # A push whose tree a successful merge-group run already passed skips
-          # the validation jobs; the lookup answers null on any doubt.
-          PROOF_TOKEN: ${{ github.event_name == 'push' && github.token || '' }}
         run: |
-          if [[ "$GITHUB_EVENT_NAME" == "push" ]]; then
-            CI_MERGE_GROUP_PROOF="$(GITHUB_TOKEN="$PROOF_TOKEN" python3 -I scripts/ci_merge_group_proof.py)" || CI_MERGE_GROUP_PROOF=null
-            export CI_MERGE_GROUP_PROOF
-          fi
           if [[ "$GITHUB_EVENT_NAME" != "pull_request" ]]; then
             python3 -I scripts/ci_gate.py select
           elif [[ "$CI_BASE_SHA" =~ ^[0-9a-f]{40}$ ]] && git show "$CI_BASE_SHA:scripts/ci_gate.py" > "$RUNNER_TEMP/ci-gate-base.py"; then
@@ -126,7 +116,7 @@ jobs:
             printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
             echo "Base CI policy unavailable; running the full matrix."
           fi
-          if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=5$' "$GITHUB_OUTPUT"; then
+          if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=4$' "$GITHUB_OUTPUT"; then
             printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'gateway_smoke=true' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
             echo "Base policy does not cover this workflow contract; retaining the full matrix."
           fi

crates/audit/tests/performance_test.rs

@@ -34,6 +34,9 @@ async fn test_audit_system_startup_performance() {
 
     println!("Audit system startup took: {elapsed:?}");
 
+    // Should complete within 5 seconds
+    assert!(elapsed < Duration::from_secs(5), "Startup took too long: {elapsed:?}");
+
     // Clean up
     let _ = system.close().await;
 }
@@ -65,6 +68,9 @@ async fn test_concurrent_target_creation() {
 
     println!("Concurrent target creation took: {elapsed:?}");
 
+    // Should complete quickly even with multiple targets
+    assert!(elapsed < Duration::from_secs(10), "Target creation took too long: {elapsed:?}");
+
     // Verify it fails with expected error (server not initialized)
     match result {
         Err(AuditError::StorageNotAvailable(_)) => {
@@ -160,6 +166,9 @@ async fn test_audit_log_dispatch_performance() {
 
     println!("Audit log dispatch took: {elapsed:?}");
 
+    // Should be very fast (sub-millisecond for no targets)
+    assert!(elapsed < Duration::from_millis(100), "Dispatch took too long: {elapsed:?}");
+
     assert!(
         matches!(result, Err(AuditError::NotInitialized(_))),
         "Dispatch on a stopped system should return NotInitialized, got: {result:?}"
@@ -198,12 +207,11 @@ async fn test_system_state_transitions() {
 }
 
 #[test]
-fn test_event_name_mask_matches_expansion() {
+fn test_event_name_mask_performance() {
     use rustfs_targets::EventName;
 
-    // A compound event's mask is exactly the union of its leaves' masks, and a
-    // leaf owns exactly one bit. Timing is printed for information only.
-    let events = [
+    // Test that event name mask calculation is efficient
+    let events = vec![
         EventName::ObjectCreatedPut,
         EventName::ObjectAccessedGet,
         EventName::ObjectRemovedDelete,
@@ -212,59 +220,47 @@ fn test_event_name_mask_matches_expansion() {
     ];
 
     let start = Instant::now();
-    for event in events {
-        let union = event.expand().iter().fold(0u64, |acc, leaf| acc | leaf.mask());
-        assert_eq!(event.mask(), union, "{event:?} mask must equal the union of its expansion");
-    }
-    println!("Event mask check took: {:?}", start.elapsed());
 
-    assert_eq!(EventName::ObjectCreatedPut.mask().count_ones(), 1);
-    assert_eq!(EventName::ObjectCreatedAll.mask().count_ones(), 4);
+    // Calculate masks for many events
+    for _ in 0..1000 {
+        for event in &events {
+            let _mask = event.mask();
+        }
+    }
+
+    let elapsed = start.elapsed();
+    println!("Event mask calculation (5000 ops) took: {elapsed:?}");
+
+    // Should be very fast
+    assert!(elapsed < Duration::from_millis(100), "Mask calculation too slow: {elapsed:?}");
 }
 
 #[test]
 fn test_event_name_expansion_performance() {
     use rustfs_targets::EventName;
-    use std::collections::HashSet;
 
-    // Expansion is a pure lookup. Wall-clock bounds flake on shared runners, so
-    // this asserts what the time was a proxy for: each compound event yields a
-    // fixed-size set of distinct leaf events and nothing else (no duplicates,
-    // no compound left to expand again). Timing is printed for information only.
-    let cases = [
-        (EventName::ObjectCreatedAll, 4),
-        (EventName::ObjectAccessedAll, 5),
-        (EventName::ObjectRemovedAll, 3),
+    // Test that event name expansion is efficient
+    let compound_events = vec![
+        EventName::ObjectCreatedAll,
+        EventName::ObjectAccessedAll,
+        EventName::ObjectRemovedAll,
+        EventName::Everything,
     ];
 
     let start = Instant::now();
-    for (event, expected_len) in cases {
-        let expanded = event.expand();
-        assert_eq!(expanded.len(), expected_len, "{event:?} expansion size");
-        assert_eq!(
-            expanded.iter().collect::<HashSet<_>>().len(),
-            expanded.len(),
-            "{event:?} expansion has duplicates"
-        );
-        assert_eq!(expanded, event.expand(), "{event:?} expansion must be deterministic");
-        for leaf in &expanded {
-            assert_eq!(leaf.expand(), vec![*leaf], "{event:?} expanded to non-leaf {leaf:?}");
+
+    // Expand events many times
+    for _ in 0..1000 {
+        for event in &compound_events {
+            let _expanded = event.expand();
         }
     }
 
-    let everything = EventName::Everything.expand();
-    assert!(everything.len() > 12, "Everything must cover at least every compound's leaves");
-    assert_eq!(
-        everything.iter().collect::<HashSet<_>>().len(),
-        everything.len(),
-        "Everything has duplicates"
-    );
-    for (event, _) in cases {
-        for leaf in event.expand() {
-            assert!(everything.contains(&leaf), "Everything is missing {leaf:?}");
-        }
-    }
-    println!("Event expansion check took: {:?}", start.elapsed());
+    let elapsed = start.elapsed();
+    println!("Event expansion (4000 ops) took: {elapsed:?}");
+
+    // Should be very fast
+    assert!(elapsed < Duration::from_millis(100), "Expansion too slow: {elapsed:?}");
 }
 
 #[tokio::test]
@@ -282,22 +278,46 @@ async fn test_registry_operations_performance() {
 
     let elapsed = start.elapsed();
     println!("Registry operations (2000 ops) took: {elapsed:?}");
+
+    // Should be very fast for empty registry
+    assert!(elapsed < Duration::from_millis(100), "Registry ops too slow: {elapsed:?}");
 }
 
-// The former 3k EPS / latency assertions measured the CI host, not the code
-// (the loop only formats strings and reads the clock), so they are replaced by
-// a count of what the loop produced. Throughput is printed for information.
+// Performance requirements validation
 #[test]
 fn test_performance_requirements() {
-    use std::collections::HashSet;
+    // According to requirements: ≥ 3k EPS/node; P99 < 30ms (default)
+
+    // These are synthetic tests since we can't actually achieve 3k EPS
+    // without real server storage and network targets, but we can validate
+    // that our core algorithms are efficient enough
 
     let start = Instant::now();
-    let ids: HashSet<String> = (0..3000).map(|i| format!("s3:ObjectCreated:Put_{i}")).collect();
-    let timestamps = (0..3000).filter(|_| !jiff::Timestamp::now().to_string().is_empty()).count();
+
+    // Simulate processing 3000 events worth of operations
+    for i in 0..3000 {
+        // Simulate event name parsing and processing
+        let _event_id = format!("s3:ObjectCreated:Put_{i}");
+        let _timestamp = jiff::Timestamp::now().to_string();
+
+        // Simulate basic audit entry creation overhead
+        let _entry_size = 512; // bytes
+        let _processing_time = std::time::Duration::from_nanos(100); // simulated
+    }
+
     let elapsed = start.elapsed();
+    let eps = 3000.0 / elapsed.as_secs_f64();
 
-    println!("Simulated 3000 events in {elapsed:?}");
+    println!("Simulated 3000 events in {elapsed:?} ({eps:.0} EPS)");
 
-    assert_eq!(ids.len(), 3000, "every simulated event id must be distinct");
-    assert_eq!(timestamps, 3000, "every simulated event must get a timestamp");
+    // Our core processing should easily handle 3k EPS worth of CPU overhead
+    // The actual EPS limit will be determined by network I/O to targets
+    assert!(eps > 10000.0, "Core processing too slow for 3k EPS target: {eps} EPS");
+
+    // P99 latency requirement: < 30ms
+    // For core processing, we should be much faster than this
+    let avg_latency = elapsed / 3000;
+    println!("Average processing latency: {avg_latency:?}");
+
+    assert!(avg_latency < Duration::from_millis(1), "Processing latency too high: {avg_latency:?}");
 }

crates/e2e_test/src/reliant/grpc_lock_server.rs

@@ -438,6 +438,13 @@ impl NodeService for MinimalLockNodeService {
         Err(Status::unimplemented("lock-only test server"))
     }
 
+    async fn write_format_control(
+        &self,
+        _request: Request<rustfs_protos::proto_gen::node_service::WriteFormatControlRequest>,
+    ) -> Result<Response<rustfs_protos::proto_gen::node_service::WriteAllResponse>, Status> {
+        Err(Status::unimplemented("lock-only test server"))
+    }
+
     async fn compare_and_update_file(
         &self,
         _request: Request<rustfs_protos::proto_gen::node_service::CompareAndUpdateFileRequest>,

crates/e2e_test/src/sts_query_compat_test.rs

@@ -395,56 +395,6 @@ impl Drop for OpaMock {
     }
 }
 
-#[tokio::test]
-async fn test_sts_session_policy_id_limits_object_access() -> TestResult {
-    let mut env = RustFSTestEnvironment::new().await?;
-    env.start_rustfs_server(vec![]).await?;
-    let admin = env.create_s3_client();
-    let bucket = "session-policy-id";
-    admin.create_bucket().bucket(bucket).send().await?;
-    for key in ["allowed", "denied"] {
-        admin
-            .put_object()
-            .bucket(bucket)
-            .key(key)
-            .body(Bytes::from_static(b"data").into())
-            .send()
-            .await?;
-    }
-
-    let policy = serde_json::json!({
-        "Id": "read-one-object",
-        "Version": "2012-10-17",
-        "Statement": [{"Effect": "Allow", "Action": "s3:GetObject", "Resource": format!("arn:aws:s3:::{bucket}/allowed")}]
-    });
-    let response = sts_client(&env.url, &env.access_key, &env.secret_key, None)
-        .assume_role()
-        .role_arn("arn:aws:iam::123456789012:role/test")
-        .role_session_name("session-policy-id")
-        .policy(policy.to_string())
-        .send()
-        .await?;
-    let credentials = response.credentials().ok_or("AssumeRole returned no credentials")?;
-    let session = aws_sdk_s3::Client::from_conf(build_test_s3_config(
-        &env.url,
-        credentials.access_key_id(),
-        credentials.secret_access_key(),
-        Some(credentials.session_token()),
-        "e2e-session-policy-id",
-    ));
-    let allowed = session.get_object().bucket(bucket).key("allowed").send().await?;
-    assert_eq!(allowed.body.collect().await?.into_bytes().as_ref(), b"data");
-    let error = session
-        .get_object()
-        .bucket(bucket)
-        .key("denied")
-        .send()
-        .await
-        .expect_err("object outside session policy");
-    assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
-    Ok(())
-}
-
 #[tokio::test]
 async fn test_sts_query_responses_are_aws_sdk_compatible() -> TestResult {
     init_logging();

crates/ecstore/src/api/mod.rs

@@ -492,7 +492,7 @@ pub mod disk {
         QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET,
         ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, SnapshotLeaseToken,
         UpdateMetadataOpts, VolumeInfo, WalkDirOptions, is_quota_mutation_fence_path, new_disk,
-        validate_batch_read_version_item_count,
+        validate_batch_read_version_item_count, validate_format_control_path,
     };
     pub use crate::disk::{
         BackgroundMemoryBudget, PhysicalDiskBinding, PhysicalDiskPlan, PhysicalDiskSample, PhysicalDiskSampler,

crates/ecstore/src/cluster/rpc/http_auth.rs

@@ -1309,7 +1309,7 @@ fn tonic_rpc_metric_operation(path: &str) -> &'static str {
         Some("ReadMultiple") => INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
         Some("ReadVersion") => INTERNODE_OPERATION_GRPC_READ_VERSION,
         Some("BatchReadVersion") => INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
-        Some("WriteAll") => INTERNODE_OPERATION_GRPC_WRITE_ALL,
+        Some("WriteAll" | "WriteFormatControl") => INTERNODE_OPERATION_GRPC_WRITE_ALL,
         Some("CompareAndUpdateFile") => INTERNODE_OPERATION_GRPC_COMPARE_AND_UPDATE_FILE,
         Some("Lock") => INTERNODE_OPERATION_GRPC_LOCK,
         Some("UnLock") => INTERNODE_OPERATION_GRPC_UNLOCK,
@@ -3195,6 +3195,10 @@ mod tests {
             tonic_rpc_metric_operation("/node_service.NodeService/WriteAll"),
             INTERNODE_OPERATION_GRPC_WRITE_ALL
         );
+        assert_eq!(
+            tonic_rpc_metric_operation("/node_service.NodeService/WriteFormatControl"),
+            INTERNODE_OPERATION_GRPC_WRITE_ALL
+        );
         assert_eq!(
             tonic_rpc_metric_operation("/node_service.NodeService/CompareAndUpdateFile"),
             INTERNODE_OPERATION_GRPC_COMPARE_AND_UPDATE_FILE

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -77,8 +77,8 @@ use rustfs_protos::proto_gen::node_service::{
     RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
     RecoverMultipartSourceResponse as RecoverMultipartSourceRpcResponse, RenameDataRequest, RenameFileRequest,
     SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
-    SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest,
-    node_service_client::NodeServiceClient,
+    SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteAllResponse,
+    WriteFormatControlRequest, WriteMetadataRequest, node_service_client::NodeServiceClient,
 };
 use serde::{Serialize, de::DeserializeOwned};
 use std::{
@@ -273,6 +273,35 @@ fn snapshot_lease_token_from_response(response: SnapshotLeaseResponse) -> Result
     SnapshotLeaseToken::from_slice(&response.token)
 }
 
+fn format_control_write_response(response: WriteAllResponse) -> Result<()> {
+    match (response.success, response.error) {
+        (true, None) => Ok(()),
+        (false, Some(error)) if DiskError::from_u32(error.code).is_some() => Err(error.into()),
+        _ => Err(DiskError::FileCorrupt),
+    }
+}
+
+// One dispatched format-control attempt ends at its validated durable reply.
+// Keeping response validation in the measured future classifies remote and
+// malformed replies as failures and accounts for cancellation through Drop.
+async fn complete_format_control_write(
+    data_len: usize,
+    future: impl std::future::Future<Output = std::result::Result<tonic::Response<WriteAllResponse>, tonic::Status>>,
+) -> Result<()> {
+    crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_request();
+    measure_client_rpc(INTERNODE_OPERATION_GRPC_WRITE_ALL, async {
+        let response = match future.await {
+            Ok(response) => response.into_inner(),
+            Err(status) if status.code() == Code::Unimplemented => return Err(DiskError::MethodNotAllowed),
+            Err(status) => return Err(status.into()),
+        };
+        crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_sent_bytes(data_len);
+        format_control_write_response(response)
+    })
+    .await
+    .inspect_err(|_| crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_error())
+}
+
 fn conditional_file_update_from_wire(outcome: i32) -> Result<ConditionalFileUpdate> {
     match CompareAndUpdateFileOutcome::try_from(outcome) {
         Ok(CompareAndUpdateFileOutcome::CompareAndUpdateFileUpdated) => Ok(ConditionalFileUpdate::Updated),
@@ -4935,6 +4964,30 @@ impl DiskAPI for RemoteDisk {
         .await
     }
 
+    async fn write_format_control(&self, path: &str, data: Bytes) -> Result<()> {
+        crate::disk::validate_format_control_path(path)?;
+        self.execute_with_timeout(
+            || async {
+                let data_len = data.len();
+                let disk = self.disk_ref().await;
+                let mut client = self.get_client().await.inspect_err(|_| {
+                    crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_error();
+                })?;
+                let mut request = Request::new(WriteFormatControlRequest {
+                    disk,
+                    path: path.to_owned(),
+                    data,
+                });
+                let canonical = rustfs_protos::canonical_write_format_control_request_body(request.get_ref())
+                    .map_err(|_| DiskError::InvalidPath)?;
+                crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
+                complete_format_control_write(data_len, client.write_format_control(request)).await
+            },
+            get_max_timeout_duration(),
+        )
+        .await
+    }
+
     async fn compare_and_update_file(
         &self,
         volume: &str,
@@ -5076,6 +5129,132 @@ impl DiskAPI for RemoteDisk {
 
 #[cfg(test)]
 mod tests {
+    #[test]
+    #[serial_test::serial]
+    fn format_control_attempt_metrics_count_each_outcome_once() {
+        use metrics_util::debugging::{DebugValue, DebuggingRecorder};
+        use std::future::{Future, pending};
+        use std::task::{Context, Poll, Waker};
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        let enabled = rustfs_io_metrics::metrics_enabled();
+        rustfs_io_metrics::set_metrics_enabled(true);
+        metrics::with_local_recorder(&recorder, || {
+            let runtime = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("metrics runtime");
+            runtime.block_on(async {
+                super::complete_format_control_write(7, async {
+                    Ok(tonic::Response::new(super::WriteAllResponse {
+                        success: true,
+                        error: None,
+                    }))
+                })
+                .await
+                .expect("durable reply succeeds");
+                let error = super::complete_format_control_write(9, async {
+                    Ok(tonic::Response::new(super::WriteAllResponse {
+                        success: false,
+                        error: Some(super::DiskError::FileNotFound.into()),
+                    }))
+                })
+                .await
+                .expect_err("typed remote failure is measured as an error");
+                assert!(matches!(error, super::DiskError::FileNotFound));
+                let error = super::complete_format_control_write(11, async { Err(tonic::Status::unimplemented("old peer")) })
+                    .await
+                    .expect_err("unsupported transport fails closed");
+                assert!(matches!(error, super::DiskError::MethodNotAllowed));
+                let mut cancelled = Box::pin(super::complete_format_control_write(13, pending()));
+                assert!(matches!(cancelled.as_mut().poll(&mut Context::from_waker(Waker::noop())), Poll::Pending));
+                drop(cancelled);
+            });
+        });
+        rustfs_io_metrics::set_metrics_enabled(enabled);
+        let entries = snapshotter.snapshot().into_vec();
+        let value = |name: &str| {
+            &entries
+                .iter()
+                .find(|(key, _, _, _)| {
+                    key.key().name() == name
+                        && key
+                            .key()
+                            .labels()
+                            .any(|label| label.key() == "operation" && label.value() == "grpc_write_all")
+                })
+                .expect("format-control metric")
+                .3
+        };
+        for (name, expected) in [
+            ("rustfs_system_network_internode_operation_requests_outgoing_total", 4),
+            ("rustfs_system_network_internode_operation_sent_bytes_total", 16),
+            ("rustfs_system_network_internode_operation_errors_total", 2),
+        ] {
+            assert!(matches!(value(name), DebugValue::Counter(count) if *count == expected), "{name}");
+        }
+        assert!(
+            matches!(value("rustfs_system_network_internode_operation_duration_ms"), DebugValue::Histogram(samples) if samples.len() == 4)
+        );
+        let mut outcomes = std::collections::HashMap::new();
+        for (key, _, _, sample) in &entries {
+            if key.key().name() == "rustfs_system_network_internode_rpc_completions_total" {
+                let outcome = key
+                    .key()
+                    .labels()
+                    .find(|label| label.key() == "outcome")
+                    .expect("outcome label");
+                let DebugValue::Counter(count) = sample else {
+                    panic!("completion counter");
+                };
+                outcomes.insert(outcome.value(), *count);
+            }
+            if key.key().name() == "rustfs_system_network_internode_rpc_active" {
+                assert!(matches!(sample, DebugValue::Gauge(value) if value.into_inner() == 0.0));
+            }
+        }
+        assert_eq!(
+            outcomes,
+            std::collections::HashMap::from([("success", 1), ("error", 2), ("cancelled", 1)])
+        );
+    }
+
+    #[test]
+    fn format_control_requires_explicit_durable_success_response() {
+        use super::{DiskError, WriteAllResponse, format_control_write_response};
+        format_control_write_response(WriteAllResponse {
+            success: true,
+            error: None,
+        })
+        .expect("explicit durable success");
+        for response in [
+            WriteAllResponse {
+                success: false,
+                error: None,
+            },
+            WriteAllResponse {
+                success: true,
+                error: Some(DiskError::FileNotFound.into()),
+            },
+            WriteAllResponse {
+                success: false,
+                error: Some(rustfs_protos::proto_gen::node_service::Error {
+                    code: u32::MAX,
+                    error_info: "unknown peer error".into(),
+                }),
+            },
+        ] {
+            assert!(matches!(format_control_write_response(response), Err(DiskError::FileCorrupt)));
+        }
+        assert!(matches!(
+            format_control_write_response(WriteAllResponse {
+                success: false,
+                error: Some(DiskError::FileNotFound.into())
+            }),
+            Err(DiskError::FileNotFound)
+        ));
+    }
+
     use super::*;
     use crate::cluster::rpc::internode_data_transport::{InternodeDataTransportCapabilities, TcpHttpInternodeDataTransport};
     use crate::erasure::coding::{BitrotReader, Erasure, decode::ParallelReader};

crates/ecstore/src/core/pools.rs

@@ -6400,7 +6400,7 @@ pub(crate) struct PoolMetaWriteState {
     pool_meta_absent: bool,
     bootstrap_authority: PoolMetaBootstrapAuthority,
     /// First-hand bootstrap authority this process holds for each pool
-    /// (index = pool index): `Fresh` only for pools it formatted itself,
+    /// (index = pool index): `Fresh` for pools it formatted or resumed from a complete durable plan,
     /// `LegacyAdoption` only for pools whose migration it verified. Empty when
     /// the caller tracks deployment-wide authority only.
     pool_bootstrap_authorities: Vec<PoolMetaBootstrapAuthority>,
@@ -6478,8 +6478,7 @@ impl PoolMetaWriteState {
     }
 
     /// Startup state for a process that loaded every pool format itself and
-    /// remembers, per pool, whether it created (or adopted) that pool
-    /// first-hand. Deployment-wide authority is the conjunction across pools:
+    /// remembers, per pool, whether it created, durably resumed, or adopted that pool. Deployment-wide authority is the conjunction across pools:
     /// any pool this process merely read yields `None`, exactly as before.
     pub(crate) fn for_startup_with_pool_bootstrap_authorities(
         cluster_id: uuid::Uuid,
@@ -6510,7 +6509,7 @@ impl PoolMetaWriteState {
             .is_some_and(|authority| authority.is_proven())
     }
 
-    /// Pools this process formatted or adopted first-hand during this startup.
+    /// Pools this process formatted, durably resumed, or adopted during startup.
     pub(crate) fn attested_pool_indices(&self) -> Vec<usize> {
         self.pool_bootstrap_authorities
             .iter()
@@ -6527,7 +6526,7 @@ impl PoolMetaWriteState {
     /// [`PoolMetaIdentityWriteScope::Pools`]), so a complete, agreeing pending
     /// set proves that every pool joined this bootstrap fresh. A missing,
     /// corrupt, or disagreeing replica keeps the writer fail-closed, and a
-    /// restart without first-hand proof never reopens bootstrap on its own.
+    /// restart requires a complete durable format plan to recover creator proof.
     fn pending_identity_attested_by_every_pool(&self) -> bool {
         self.elected_bootstrap_writer == Some(true)
             && self.pool_bootstrap_authority_proven(0)
@@ -6759,6 +6758,27 @@ impl PoolMetaWriteState {
         self.expected_cluster_id.is_some() && (self.identity_needs_repair || self.identity_initialized != Some(true))
     }
 
+    /// A committed predecessor authorizes repairing an established identity
+    /// before the next metadata write. A pending fresh identity must remain
+    /// pending until its initial metadata commit is recoverable.
+    pub(crate) fn startup_identity_precedes_metadata_write(&self) -> bool {
+        !self.pool_meta_absent && !self.identity_is_pending()
+    }
+
+    pub(crate) fn ensure_startup_identity_initialized(&self) -> Result<()> {
+        if self.expected_cluster_id.is_none()
+            || (self.identity_initialized == Some(true)
+                && !self.identity_needs_repair
+                && !self.identity_replicas_missing
+                && !self.identity_replicas_invalid)
+        {
+            return Ok(());
+        }
+        Err(Error::other(
+            "pool metadata bootstrap pending: waiting for the elected writer to initialize the identity on every pool",
+        ))
+    }
+
     pub(crate) fn ensure_write_safe(&self, operation: &str) -> Result<()> {
         let (Some(mut context), _) = self.write_block() else {
             return Ok(());
@@ -10146,6 +10166,8 @@ impl PoolMeta {
         self.save_for_startup_observing(pools, &mut write_state).await.map(|_| ())
     }
 
+    /// Commits a complete startup snapshot while leaving identity finalization
+    /// to the caller, after durable format-bootstrap authority is retired.
     pub(crate) async fn save_for_startup_observing<S>(
         &self,
         pools: Vec<Arc<S>>,
@@ -10160,6 +10182,8 @@ impl PoolMeta {
         Ok(outcome.into_committed())
     }
 
+    /// Commits a complete startup snapshot under the writer fence. The caller
+    /// retires format-bootstrap authority before finalizing initialized identity.
     pub(crate) async fn save_for_startup_guarded<S>(
         &self,
         pools: Vec<Arc<S>>,
@@ -10362,7 +10386,10 @@ impl PoolMeta {
                     selection.revision.epoch, identity.epoch
                 )));
             }
-            if !selection.absent && write_state.identity_requires_repair() {
+            // Full-snapshot saves belong to startup. Their caller retires
+            // durable format authority before publishing initialized identity.
+            // Runtime scoped saves retain their existing identity repair.
+            if indices.is_some() && !selection.absent && write_state.identity_requires_repair() {
                 let initialized = write_state.identity_initialized != Some(false) || selection.revision.is_generation_protocol();
                 persist_pool_meta_identity(
                     pools.clone(),
@@ -10640,7 +10667,10 @@ impl PoolMeta {
             confirmed
         };
         if confirmed.revision == revision && confirmed.canonical.as_ref() == Some(&durable) {
-            persist_pool_meta_identity(pools, write_state, true, PoolMetaIdentityWriteScope::All, fence, transaction_arm).await?;
+            if indices.is_some() {
+                persist_pool_meta_identity(pools, write_state, true, PoolMetaIdentityWriteScope::All, fence, transaction_arm)
+                    .await?;
+            }
             #[cfg(feature = "e2e-test-hooks")]
             startup_cas_test_observe(serde_json::json!({
                 "kind": "confirmed", "object": POOL_META_NAME,

crates/ecstore/src/disk/disk_store.rs

@@ -345,6 +345,7 @@ const DISK_OPERATION_NAMES: &[&str] = &[
     "read_parts",
     "read_multiple",
     "write_all",
+    "write_format_control",
     "compare_and_update_file",
     "read_all",
 ];
@@ -3662,6 +3663,16 @@ impl DiskAPI for LocalDiskWrapper {
         .await
     }
 
+    async fn write_format_control(&self, path: &str, data: Bytes) -> Result<()> {
+        self.track_disk_health_mutation(
+            "write_format_control",
+            DiskMetricMutation::Write,
+            || async { self.disk.write_format_control(path, data).await },
+            get_max_timeout_duration(),
+        )
+        .await
+    }
+
     async fn compare_and_update_file(
         &self,
         volume: &str,

crates/ecstore/src/disk/local/delete_recovery.rs

@@ -358,26 +358,14 @@ pub(in crate::disk) fn generation(mut canonical: FileInfo) -> Result<crate::disk
             _ => {}
         }
     }
-    struct Sha256Writer(Sha256);
-    impl std::io::Write for Sha256Writer {
-        fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
-            self.0.update(buf);
-            Ok(buf.len())
-        }
-        fn flush(&mut self) -> std::io::Result<()> {
-            Ok(())
-        }
-    }
     let mut json_value = serde_json::to_value(&canonical)?;
     sort_json_in_place(&mut json_value);
-    let mut hasher = Sha256Writer(Sha256::new());
-    serde_json::to_writer(&mut hasher, &json_value)?;
     Ok(crate::disk::DeleteRecoveryVersionIdentity {
         version_id: canonical.version_id,
         mod_time: canonical.mod_time,
         data_dir: canonical.data_dir,
         deleted: canonical.deleted,
-        metadata_digest: hasher.0.finalize().into(),
+        metadata_digest: digest(&serde_json::to_vec(&json_value)?),
     })
 }
 
@@ -1186,7 +1174,6 @@ struct PreparedDeleteReceiptEntry {
     sole_transaction: bool,
     had_old_data_dir: bool,
     rollback_dir_fd: Option<Arc<std::fs::File>>,
-    file_path_dir: Option<Arc<std::fs::File>>,
 }
 
 #[derive(Clone)]
@@ -1198,189 +1185,6 @@ struct PublishedDeleteReceiptEntry {
     journal_raw: Vec<u8>,
     had_old_data_dir: bool,
     rollback_dir_fd: Option<Arc<std::fs::File>>,
-    file_path_dir: Option<Arc<std::fs::File>>,
-}
-
-#[cfg(unix)]
-fn read_child_regular_bytes(
-    parent_fd: Option<&std::fs::File>,
-    child_name: &str,
-    fallback_path: &Path,
-) -> std::io::Result<Vec<u8>> {
-    if let Some(dir) = parent_fd {
-        use rustix::fs::{Mode, OFlags, openat};
-        let fd = openat(dir, child_name, OFlags::RDONLY | OFlags::CLOEXEC, Mode::empty()).map_err(std::io::Error::from)?;
-        let mut file = std::fs::File::from(fd);
-        let cap = file
-            .metadata()
-            .ok()
-            .and_then(|m| usize::try_from(m.len()).ok())
-            .unwrap_or(0)
-            .min(64 * 1024);
-        let mut buf = Vec::with_capacity(cap);
-        std::io::Read::read_to_end(&mut file, &mut buf)?;
-        return Ok(buf);
-    }
-    std::fs::read(fallback_path)
-}
-
-#[cfg(not(unix))]
-fn read_child_regular_bytes(
-    _parent_fd: Option<&std::fs::File>,
-    _child_name: &str,
-    fallback_path: &Path,
-) -> std::io::Result<Vec<u8>> {
-    std::fs::read(fallback_path)
-}
-
-#[cfg(unix)]
-fn create_and_write_child_file(
-    parent_fd: Option<&std::fs::File>,
-    child_name: &str,
-    fallback_path: &Path,
-    bytes: &[u8],
-) -> std::io::Result<std::fs::File> {
-    let mut file = if let Some(dir) = parent_fd {
-        use rustix::fs::{Mode, OFlags, openat};
-        let fd = openat(
-            dir,
-            child_name,
-            OFlags::WRONLY | OFlags::CREATE | OFlags::TRUNC | OFlags::CLOEXEC,
-            Mode::from_raw_mode(0o666),
-        )
-        .map_err(std::io::Error::from)?;
-        std::fs::File::from(fd)
-    } else {
-        std::fs::File::create(fallback_path)?
-    };
-    std::io::Write::write_all(&mut file, bytes)?;
-    Ok(file)
-}
-
-#[cfg(not(unix))]
-fn create_and_write_child_file(
-    _parent_fd: Option<&std::fs::File>,
-    _child_name: &str,
-    fallback_path: &Path,
-    bytes: &[u8],
-) -> std::io::Result<std::fs::File> {
-    let mut file = std::fs::File::create(fallback_path)?;
-    std::io::Write::write_all(&mut file, bytes)?;
-    Ok(file)
-}
-
-#[cfg(unix)]
-fn link_child_files(
-    src_dir_fd: Option<&std::fs::File>,
-    src_name: &str,
-    src_fallback: &Path,
-    dst_dir_fd: Option<&std::fs::File>,
-    dst_name: &str,
-    dst_fallback: &Path,
-) -> std::io::Result<()> {
-    if let (Some(s_fd), Some(d_fd)) = (src_dir_fd, dst_dir_fd) {
-        return rustix::fs::linkat(s_fd, src_name, d_fd, dst_name, rustix::fs::AtFlags::empty()).map_err(std::io::Error::from);
-    }
-    std::fs::hard_link(src_fallback, dst_fallback)
-}
-
-#[cfg(not(unix))]
-fn link_child_files(
-    _src_dir_fd: Option<&std::fs::File>,
-    _src_name: &str,
-    src_fallback: &Path,
-    _dst_dir_fd: Option<&std::fs::File>,
-    _dst_name: &str,
-    dst_fallback: &Path,
-) -> std::io::Result<()> {
-    std::fs::hard_link(src_fallback, dst_fallback)
-}
-
-#[cfg(unix)]
-fn link_from_dir_fd(
-    src_dir_fd: Option<&std::fs::File>,
-    src_name: &str,
-    src_fallback: &Path,
-    dst_path: &Path,
-) -> std::io::Result<()> {
-    if let Some(s_fd) = src_dir_fd {
-        return rustix::fs::linkat(s_fd, src_name, rustix::fs::CWD, dst_path, rustix::fs::AtFlags::empty())
-            .map_err(std::io::Error::from);
-    }
-    std::fs::hard_link(src_fallback, dst_path)
-}
-
-#[cfg(not(unix))]
-fn link_from_dir_fd(
-    _src_dir_fd: Option<&std::fs::File>,
-    _src_name: &str,
-    src_fallback: &Path,
-    dst_path: &Path,
-) -> std::io::Result<()> {
-    std::fs::hard_link(src_fallback, dst_path)
-}
-
-#[cfg(unix)]
-fn rename_into_dir_fd(
-    src_path: &Path,
-    dst_dir_fd: Option<&std::fs::File>,
-    dst_name: &str,
-    dst_fallback: &Path,
-) -> std::io::Result<()> {
-    if let Some(d_fd) = dst_dir_fd {
-        return rustix::fs::renameat(rustix::fs::CWD, src_path, d_fd, dst_name).map_err(std::io::Error::from);
-    }
-    std::fs::rename(src_path, dst_fallback)
-}
-
-#[cfg(not(unix))]
-fn rename_into_dir_fd(
-    src_path: &Path,
-    _dst_dir_fd: Option<&std::fs::File>,
-    _dst_name: &str,
-    dst_fallback: &Path,
-) -> std::io::Result<()> {
-    std::fs::rename(src_path, dst_fallback)
-}
-
-#[cfg(unix)]
-fn child_regular_file_exists(dir_fd: Option<&std::fs::File>, child_name: &str, fallback_path: &Path) -> bool {
-    if let Some(d_fd) = dir_fd {
-        return rustix::fs::statat(d_fd, child_name, rustix::fs::AtFlags::SYMLINK_NOFOLLOW)
-            .is_ok_and(|st| (st.st_mode & libc::S_IFMT) == libc::S_IFREG);
-    }
-    fallback_path.is_file()
-}
-
-#[cfg(not(unix))]
-fn child_regular_file_exists(_dir_fd: Option<&std::fs::File>, _child_name: &str, fallback_path: &Path) -> bool {
-    fallback_path.is_file()
-}
-
-#[cfg(unix)]
-fn unlink_child_file(dir_fd: Option<&std::fs::File>, child_name: &str, fallback_path: &Path) -> std::io::Result<()> {
-    if let Some(d_fd) = dir_fd {
-        return rustix::fs::unlinkat(d_fd, child_name, rustix::fs::AtFlags::empty()).map_err(std::io::Error::from);
-    }
-    std::fs::remove_file(fallback_path)
-}
-
-#[cfg(not(unix))]
-fn unlink_child_file(_dir_fd: Option<&std::fs::File>, _child_name: &str, fallback_path: &Path) -> std::io::Result<()> {
-    std::fs::remove_file(fallback_path)
-}
-
-#[cfg(unix)]
-fn rmdir_child_dir(dir_fd: Option<&std::fs::File>, child_name: &str, fallback_path: &Path) -> std::io::Result<()> {
-    if let Some(d_fd) = dir_fd {
-        return rustix::fs::unlinkat(d_fd, child_name, rustix::fs::AtFlags::REMOVEDIR).map_err(std::io::Error::from);
-    }
-    std::fs::remove_dir(fallback_path)
-}
-
-#[cfg(not(unix))]
-fn rmdir_child_dir(_dir_fd: Option<&std::fs::File>, _child_name: &str, fallback_path: &Path) -> std::io::Result<()> {
-    std::fs::remove_dir(fallback_path)
 }
 
 const PREPARED_DELETE_RECEIPTS_PER_SHARD: usize = 64;
@@ -1407,21 +1211,6 @@ fn take_published_delete_receipt(rollback: &Path) -> Option<PublishedDeleteRecei
     PUBLISHED_DELETE_RECEIPTS.remove(rollback)
 }
 
-pub(super) fn clear_delete_receipts_under(io_root: &Path, root: &Path) {
-    for prefix in [io_root, root] {
-        PREPARED_DELETE_RECEIPTS.retain_outside_prefix(prefix);
-        PUBLISHED_DELETE_RECEIPTS.retain_outside_prefix(prefix);
-    }
-}
-
-#[cfg(unix)]
-fn open_dir_fd(path: &Path) -> std::io::Result<std::fs::File> {
-    use rustix::fs::{Mode, OFlags, open};
-    open(path, OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC, Mode::empty())
-        .map(std::fs::File::from)
-        .map_err(std::io::Error::from)
-}
-
 fn format_commit_receipt_bytes(journal_raw: &[u8], finished: bool) -> Vec<u8> {
     let suffix: &[u8] = if finished {
         b",\"finished\":true}"
@@ -2753,22 +2542,11 @@ impl LocalDisk {
         let object_str = object.to_owned();
 
         let recovery_scope = recovery_operation_scope();
-        type FusedStage1Outcome = Option<(Option<Uuid>, PathBuf, PreparedDeleteReceiptEntry)>;
         let res = os::run_blocking_namespace_operation_with_sync(
             lease,
             sync,
-            move || -> std::io::Result<Result<FusedStage1Outcome>> {
-                #[cfg(unix)]
-                let file_path_dir = match open_dir_fd(&file_path) {
-                    Ok(dir) => Some(Arc::new(dir)),
-                    Err(err) if matches!(err.kind(), ErrorKind::NotFound | ErrorKind::NotADirectory) => {
-                        return Ok(Ok(None));
-                    }
-                    Err(_) => None,
-                };
-                #[cfg(not(unix))]
-                let file_path_dir: Option<Arc<std::fs::File>> = None;
-                let buf = match read_child_regular_bytes(file_path_dir.as_deref(), STORAGE_FORMAT_FILE, &xl_path) {
+            move || -> std::io::Result<Result<Option<Option<Uuid>>>> {
+                let buf = match std::fs::read(&xl_path) {
                     Ok(buf) => buf,
                     Err(err) if err.kind() == ErrorKind::NotFound => return Ok(Ok(None)),
                     Err(err) => return Err(err),
@@ -2926,23 +2704,10 @@ impl LocalDisk {
                     Err(err) => return Err(err),
                 };
                 let staged_dir = staged.path().to_path_buf();
-                #[cfg(unix)]
-                let rollback_dir_fd = open_dir_fd(&staged_dir).ok().map(Arc::new);
-                #[cfg(not(unix))]
-                let rollback_dir_fd: Option<Arc<std::fs::File>> = None;
                 let backup_path = staged_dir.join(STORAGE_FORMAT_FILE_BACKUP);
-                if link_child_files(
-                    file_path_dir.as_deref(),
-                    STORAGE_FORMAT_FILE,
-                    &xl_path,
-                    rollback_dir_fd.as_deref(),
-                    STORAGE_FORMAT_FILE_BACKUP,
-                    &backup_path,
-                )
-                .is_err()
-                {
-                    let file =
-                        create_and_write_child_file(rollback_dir_fd.as_deref(), STORAGE_FORMAT_FILE_BACKUP, &backup_path, &buf)?;
+                if std::fs::hard_link(&xl_path, &backup_path).is_err() {
+                    let mut file = std::fs::File::create(&backup_path)?;
+                    std::io::Write::write_all(&mut file, &buf)?;
                     if sync && !buf.is_empty() {
                         file.sync_data()?;
                     }
@@ -2952,7 +2717,8 @@ impl LocalDisk {
                     let Some(bytes) = bytes else {
                         continue;
                     };
-                    let file = create_and_write_child_file(rollback_dir_fd.as_deref(), name, &staged_dir.join(name), bytes)?;
+                    let mut file = std::fs::File::create(staged_dir.join(name))?;
+                    std::io::Write::write_all(&mut file, bytes)?;
                     if sync && !bytes.is_empty() {
                         written_files.push(file);
                     }
@@ -2960,20 +2726,21 @@ impl LocalDisk {
                 for file in written_files {
                     file.sync_data()?;
                 }
+                #[cfg(unix)]
+                let rollback_dir_fd = sync.then(|| std::fs::File::open(&staged_dir).ok().map(Arc::new)).flatten();
+                #[cfg(not(unix))]
+                let rollback_dir_fd: Option<Arc<std::fs::File>> = None;
                 if sync {
                     os::fsync_directory_handle_std(&staged_dir, rollback_dir_fd.as_deref())?;
                 }
-                let tx_name = transaction.to_string();
                 if !destination_missing {
                     match std::fs::symlink_metadata(&destination) {
-                        Ok(metadata) if metadata.is_dir() => {
-                            match rmdir_child_dir(file_path_dir.as_deref(), &tx_name, &destination) {
-                                Ok(()) => {}
-                                Err(err) if err.kind() == ErrorKind::NotFound => {}
-                                Err(err) if is_dir_not_empty_error(&err) => return Ok(Err(DiskError::OutdatedXLMeta)),
-                                Err(err) => return Err(err),
-                            }
-                        }
+                        Ok(metadata) if metadata.is_dir() => match std::fs::remove_dir(&destination) {
+                            Ok(()) => {}
+                            Err(err) if err.kind() == ErrorKind::NotFound => {}
+                            Err(err) if is_dir_not_empty_error(&err) => return Ok(Err(DiskError::OutdatedXLMeta)),
+                            Err(err) => return Err(err),
+                        },
                         Ok(_) => return Ok(Err(DiskError::OutdatedXLMeta)),
                         Err(err) if err.kind() == ErrorKind::NotFound => {}
                         Err(err) => return Err(err),
@@ -2981,14 +2748,18 @@ impl LocalDisk {
                 }
                 let staged_dir = staged.keep();
                 os::record_native_publication(&destination);
-                if let Err(err) = rename_into_dir_fd(&staged_dir, file_path_dir.as_deref(), &tx_name, &destination) {
+                if let Err(err) = std::fs::rename(&staged_dir, &destination) {
                     let _ = std::fs::remove_dir_all(&staged_dir);
                     return Err(err);
                 }
                 let old_data_path = old_dir
                     .map(|uuid| file_path.join(uuid.to_string()))
                     .filter(|p| old_data_dir_state.unwrap_or_else(|| p.is_dir()));
-                if sync && let Err(err) = os::fsync_directory_handle_std(&file_path, file_path_dir.as_deref()) {
+                #[cfg(unix)]
+                let file_path_dir = sync.then(|| std::fs::File::open(&file_path).ok()).flatten();
+                #[cfg(not(unix))]
+                let file_path_dir: Option<std::fs::File> = None;
+                if sync && let Err(err) = os::fsync_directory_handle_std(&file_path, file_path_dir.as_ref()) {
                     let _ = std::fs::remove_dir_all(&destination);
                     return Err(err);
                 }
@@ -2998,7 +2769,7 @@ impl LocalDisk {
                 if let Some(old_path) = old_data_path {
                     let reserved_marker = old_path.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"));
                     #[cfg(unix)]
-                    let old_dir_file = sync.then(|| open_dir_fd(&old_path).ok()).flatten();
+                    let old_dir_file = sync.then(|| std::fs::File::open(&old_path).ok()).flatten();
                     #[cfg(not(unix))]
                     let old_dir_file: Option<std::fs::File> = None;
                     let reserve_res = (|| -> std::io::Result<()> {
@@ -3021,16 +2792,16 @@ impl LocalDisk {
                     let target_in_rollback = destination.join(TARGET);
                     let linked_xl = staging_root.join(format!("delete-xl-{transaction}"));
                     os::record_native_publication(&xl_path);
-                    let linked = match link_from_dir_fd(rollback_dir_fd.as_deref(), TARGET, &target_in_rollback, &linked_xl) {
+                    let linked = match std::fs::hard_link(&target_in_rollback, &linked_xl) {
                         Ok(()) => true,
                         Err(err) if err.kind() == ErrorKind::AlreadyExists => {
                             let _ = std::fs::remove_file(&linked_xl);
-                            link_from_dir_fd(rollback_dir_fd.as_deref(), TARGET, &target_in_rollback, &linked_xl).is_ok()
+                            std::fs::hard_link(&target_in_rollback, &linked_xl).is_ok()
                         }
                         Err(_) => false,
                     };
                     let write_res = if linked {
-                        let rename_res = rename_into_dir_fd(&linked_xl, file_path_dir.as_deref(), STORAGE_FORMAT_FILE, &xl_path);
+                        let rename_res = std::fs::rename(&linked_xl, &xl_path);
                         if rename_res.is_err() {
                             let _ = std::fs::remove_file(&linked_xl);
                         }
@@ -3042,7 +2813,7 @@ impl LocalDisk {
                             if sync && !target_bytes.is_empty() {
                                 staged_xl.as_file().sync_data()?;
                             }
-                            rename_into_dir_fd(staged_xl.path(), file_path_dir.as_deref(), STORAGE_FORMAT_FILE, &xl_path)
+                            std::fs::rename(staged_xl.path(), &xl_path)
                         })()
                     };
                     if let Err(err) = write_res {
@@ -3052,7 +2823,7 @@ impl LocalDisk {
                         let _ = std::fs::remove_dir_all(&destination);
                         return Err(err);
                     }
-                } else if let Err(err) = unlink_child_file(file_path_dir.as_deref(), STORAGE_FORMAT_FILE, &xl_path)
+                } else if let Err(err) = std::fs::remove_file(&xl_path)
                     && err.kind() != ErrorKind::NotFound
                 {
                     if let Some((_, reserved_marker, _)) = &reserved_data_path {
@@ -3062,7 +2833,7 @@ impl LocalDisk {
                     return Err(err);
                 }
                 if sync {
-                    os::fsync_directory_handle_std(&file_path, file_path_dir.as_deref())?;
+                    os::fsync_directory_handle_std(&file_path, file_path_dir.as_ref())?;
                 }
                 let had_old_data_dir = reserved_data_path.is_some();
                 if let Some((old_path, reserved_marker, old_dir_file)) = reserved_data_path {
@@ -3074,8 +2845,7 @@ impl LocalDisk {
                         Err(err) => return Err(err),
                     }
                 }
-                Ok(Ok(Some((
-                    old_dir.filter(|_| had_old_data_dir),
+                put_prepared_delete_receipt(
                     destination,
                     PreparedDeleteReceiptEntry {
                         object_path,
@@ -3087,18 +2857,17 @@ impl LocalDisk {
                         sole_transaction,
                         had_old_data_dir,
                         rollback_dir_fd,
-                        file_path_dir,
                     },
-                ))))
+                );
+                Ok(Ok(Some(old_dir.filter(|_| had_old_data_dir))))
             },
         )
         .await
         .map_err(to_file_error)??;
 
-        let Some((old_dir, destination, receipt_entry)) = res else {
+        let Some(old_dir) = res else {
             return Ok(None);
         };
-        put_prepared_delete_receipt(destination, receipt_entry);
         if let Some(uuid) = old_dir {
             self.io_backend
                 .invalidate_cached_fds_under(volume, &format!("{object}/{uuid}"));
@@ -3149,7 +2918,6 @@ impl LocalDisk {
                     entry.sole_transaction,
                     entry.had_old_data_dir,
                     entry.rollback_dir_fd,
-                    entry.file_path_dir,
                 )),
             ),
             None => {
@@ -3177,14 +2945,12 @@ impl LocalDisk {
         let volume_str = volume.to_owned();
         let object_str = object.to_owned();
 
-        type PublishDeleteFastReturn = (DeleteRecoveryJournal, bool, bool, Option<(PathBuf, PublishedDeleteReceiptEntry)>);
-        let res = os::run_blocking_namespace_operation_with_sync(
+        os::run_blocking_namespace_operation_with_sync(
             lease,
             sync,
-            move || -> std::io::Result<Result<Option<PublishDeleteFastReturn>>> {
-                if let Some((journal_raw, journal, sole_transaction, had_old_data_dir, rollback_dir_fd, file_path_dir)) =
-                    cached_receipt
-                    && child_regular_file_exists(rollback_dir_fd.as_deref(), PREPARED, &rollback.join(PREPARED))
+            move || -> std::io::Result<Result<Option<(DeleteRecoveryJournal, bool, bool)>>> {
+                if let Some((journal_raw, journal, sole_transaction, had_old_data_dir, rollback_dir_fd)) = cached_receipt
+                    && rollback.join(PREPARED).is_file()
                 {
                     let tmp_receipt = staging_root.join(format!("receipt-{transaction}.tmp"));
                     let receipt_bytes = format_commit_receipt_bytes(&journal_raw, false);
@@ -3195,12 +2961,7 @@ impl LocalDisk {
                             file.sync_data()?;
                         }
                         os::record_native_publication(&receipt_path);
-                        rename_into_dir_fd(
-                            &tmp_receipt,
-                            rollback_dir_fd.as_deref(),
-                            DELETE_QUORUM_COMMITTED_FILE,
-                            &receipt_path,
-                        )?;
+                        std::fs::rename(&tmp_receipt, &receipt_path)?;
                         if sync {
                             os::fsync_directory_handle_std(&rollback, rollback_dir_fd.as_deref())?;
                         }
@@ -3210,8 +2971,8 @@ impl LocalDisk {
                         let _ = std::fs::remove_file(&tmp_receipt);
                         return Err(err);
                     }
-                    let published_entry = bucket_path.map(|bucket_path| {
-                        (
+                    if let Some(bucket_path) = bucket_path {
+                        put_published_delete_receipt(
                             rollback,
                             PublishedDeleteReceiptEntry {
                                 object_path,
@@ -3221,11 +2982,10 @@ impl LocalDisk {
                                 journal_raw,
                                 had_old_data_dir,
                                 rollback_dir_fd,
-                                file_path_dir,
                             },
-                        )
-                    });
-                    return Ok(Ok(Some((journal, sole_transaction, true, published_entry))));
+                        );
+                    }
+                    return Ok(Ok(Some((journal, sole_transaction, true))));
                 }
                 let prepared = match regular_bytes(&rollback.join(PREPARED)) {
                     Ok(Some(bytes)) => bytes,
@@ -3304,19 +3064,11 @@ impl LocalDisk {
                 if sync {
                     os::fsync_dir_std(&rollback)?;
                 }
-                Ok(Ok(Some((journal, sole_transaction, target_satisfied, None))))
+                Ok(Ok(Some((journal, sole_transaction, target_satisfied))))
             },
         )
         .await
-        .map_err(to_file_error)?;
-
-        let Some((journal, sole_transaction, target_satisfied, published_entry)) = res? else {
-            return Ok(None);
-        };
-        if let Some((rollback, entry)) = published_entry {
-            put_published_delete_receipt(rollback, entry);
-        }
-        Ok(Some((journal, sole_transaction, target_satisfied)))
+        .map_err(to_file_error)?
     }
 
     #[cfg(test)]
@@ -3366,49 +3118,34 @@ impl LocalDisk {
             None
         };
         let cached_published = rollback_open_path.as_deref().and_then(take_published_delete_receipt);
-        if cached_published.is_none()
-            && let Some(ref tx_open) = rollback_open_path
-        {
-            let _ = take_prepared_delete_receipt(tx_open);
-        }
-        let (
-            object_path,
-            bucket_path,
-            staging_root,
-            root,
-            cached_journal_raw,
-            cached_had_old_data_dir,
-            mut rollback_dir_fd,
-            mut file_path_dir,
-        ) = match cached_published {
-            Some(entry) => (
-                entry.object_path,
-                entry.bucket_path,
-                entry.staging_root,
-                entry.file_path,
-                Some(entry.journal_raw),
-                Some(entry.had_old_data_dir),
-                entry.rollback_dir_fd,
-                entry.file_path_dir,
-            ),
-            None => {
-                self.reject_multipart_delete_recovery_mutation(volume, object)?;
-                if !safe_path(object, false) {
-                    return Err(DiskError::InvalidPath);
+        let (object_path, bucket_path, staging_root, root, cached_journal_raw, cached_had_old_data_dir, mut rollback_dir_fd) =
+            match cached_published {
+                Some(entry) => (
+                    entry.object_path,
+                    entry.bucket_path,
+                    entry.staging_root,
+                    entry.file_path,
+                    Some(entry.journal_raw),
+                    Some(entry.had_old_data_dir),
+                    entry.rollback_dir_fd,
+                ),
+                None => {
+                    self.reject_multipart_delete_recovery_mutation(volume, object)?;
+                    if !safe_path(object, false) {
+                        return Err(DiskError::InvalidPath);
+                    }
+                    let obj_path = self.get_object_path(volume, object)?;
+                    (
+                        obj_path,
+                        self.io_get_bucket_path(volume)?,
+                        self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?,
+                        self.io_get_object_path(volume, object)?,
+                        None,
+                        None,
+                        None,
+                    )
                 }
-                let obj_path = self.get_object_path(volume, object)?;
-                (
-                    obj_path,
-                    self.io_get_bucket_path(volume)?,
-                    self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?,
-                    self.io_get_object_path(volume, object)?,
-                    None,
-                    None,
-                    None,
-                    None,
-                )
-            }
-        };
+            };
         let metadata_owner = os::acquire_metadata_mutation_lease(&object_path, namespace_owner).await;
         let volume_guard = os::disk_volume_mutation_lock(&self.root, volume).read_owned().await;
         let owner: Option<Arc<dyn Send + Sync>> = Some(Arc::new((metadata_owner, volume_guard)));
@@ -3535,7 +3272,7 @@ impl LocalDisk {
                 }
             }
             if had_physical_data_dirs && sync && root.is_dir() {
-                os::fsync_directory_handle_std(&root, file_path_dir.as_deref())?;
+                os::fsync_dir_std(&root)?;
             }
             if fast_cleanup {
                 let backup_or_marker = if has_original {
@@ -3546,10 +3283,8 @@ impl LocalDisk {
                     None
                 };
                 if let Some(name) = backup_or_marker
-                    && let Err(err) = ignore_not_found(unlink_child_file(rollback_dir_fd.as_deref(), name, &rollback.join(name)))
+                    && let Err(err) = ignore_not_found(std::fs::remove_file(rollback.join(name)))
                 {
-                    drop(rollback_dir_fd.take());
-                    drop(file_path_dir.take());
                     if has_target || had_physical_data_dirs {
                         let _ = persist_receipt(true);
                     }
@@ -3559,9 +3294,7 @@ impl LocalDisk {
                     let path = rollback.join(name);
                     #[cfg(all(any(test, feature = "test-util"), not(windows)))]
                     os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &path);
-                    if let Err(err) = ignore_not_found(unlink_child_file(rollback_dir_fd.as_deref(), name, &path)) {
-                        drop(rollback_dir_fd.take());
-                        drop(file_path_dir.take());
+                    if let Err(err) = ignore_not_found(std::fs::remove_file(path)) {
                         if has_target || had_physical_data_dirs {
                             let _ = persist_receipt(true);
                         }
@@ -3570,10 +3303,7 @@ impl LocalDisk {
                 }
             } else {
                 for name in [STORAGE_FORMAT_FILE_BACKUP, DELETE_MARKER_ROLLBACK_FILE] {
-                    if let Err(err) = ignore_not_found(unlink_child_file(rollback_dir_fd.as_deref(), name, &rollback.join(name)))
-                    {
-                        drop(rollback_dir_fd.take());
-                        drop(file_path_dir.take());
+                    if let Err(err) = ignore_not_found(std::fs::remove_file(rollback.join(name))) {
                         if has_target || had_physical_data_dirs {
                             let _ = persist_receipt(true);
                         }
@@ -3584,9 +3314,7 @@ impl LocalDisk {
                     let path = rollback.join(name);
                     #[cfg(all(any(test, feature = "test-util"), not(windows)))]
                     os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &path);
-                    if let Err(err) = ignore_not_found(unlink_child_file(rollback_dir_fd.as_deref(), name, &path)) {
-                        drop(rollback_dir_fd.take());
-                        drop(file_path_dir.take());
+                    if let Err(err) = ignore_not_found(std::fs::remove_file(path)) {
                         if has_target || had_physical_data_dirs {
                             let _ = persist_receipt(true);
                         }
@@ -3594,39 +3322,27 @@ impl LocalDisk {
                     }
                 }
             }
-            if sync && let Err(err) = os::fsync_directory_handle_std(&rollback, rollback_dir_fd.as_deref()) {
-                drop(rollback_dir_fd.take());
-                drop(file_path_dir.take());
+            let rollback_sync = rollback_dir_fd.take();
+            if sync && let Err(err) = os::fsync_directory_handle_std(&rollback, rollback_sync.as_deref()) {
+                drop(rollback_sync);
                 if has_target || had_physical_data_dirs {
                     let _ = persist_receipt(true);
                 }
                 return Err(err);
             }
+            drop(rollback_sync);
             #[cfg(all(any(test, feature = "test-util"), not(windows)))]
             os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &receipt_path);
-            let receipt_unlink_res =
-                ignore_not_found(unlink_child_file(rollback_dir_fd.as_deref(), DELETE_QUORUM_COMMITTED_FILE, &receipt_path));
-            drop(rollback_dir_fd.take());
-            if let Err(err) = receipt_unlink_res {
-                drop(file_path_dir.take());
+            if let Err(err) = ignore_not_found(std::fs::remove_file(&receipt_path)) {
                 if has_target || had_physical_data_dirs {
                     let _ = persist_receipt(true);
                 }
                 return Err(err);
             }
-            let tx_name = transaction.to_string();
-            for (idx, dir_path) in prune_paths.iter().enumerate() {
+            for dir_path in &prune_paths {
                 #[cfg(all(any(test, feature = "test-util"), not(windows)))]
                 os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, dir_path);
-                let rmdir_res = if idx == 0 && *dir_path == rollback {
-                    let r = rmdir_child_dir(file_path_dir.as_deref(), &tx_name, dir_path);
-                    drop(file_path_dir.take());
-                    r
-                } else {
-                    drop(file_path_dir.take());
-                    std::fs::remove_dir(dir_path)
-                };
-                if let Err(err) = rmdir_res {
+                if let Err(err) = std::fs::remove_dir(dir_path) {
                     if is_dir_not_empty_error(&err) {
                         break;
                     }
@@ -6118,7 +5834,6 @@ mod tests {
                 drop(disk);
                 let endpoint = Endpoint::try_from(temp.path().to_str().expect("UTF-8 disk path")).expect("endpoint");
                 let disk = LocalDisk::new(&endpoint, false).await.expect("reopen after cleanup");
-                let root = disk.io_get_object_path("recovery", "object").expect("reopened object path");
                 let observed = disk
                     .inspect_delete_transaction("recovery", "object", transaction)
                     .await

crates/ecstore/src/disk/mod.rs

@@ -58,6 +58,21 @@ pub const BUCKET_META_PREFIX: &str = "buckets";
 /// a user bucket of the same name, so bucket deletion must leave it in place.
 pub const HEAL_REPLACEMENT_RECOVERY_DIR: &str = "ahm-replacement";
 pub const FORMAT_CONFIG_FILE: &str = "format.json";
+pub(crate) const FORMAT_BOOTSTRAP_FILE: &str = "format.json.bootstrap";
+pub(crate) const FORMAT_BOOTSTRAP_DONE_FILE: &str = "format.json.bootstrap.done";
+
+/// Validate an internal format control filename for atomic publication.
+///
+/// # Errors
+/// Returns [`DiskError::MethodNotAllowed`] for every other path.
+pub fn validate_format_control_path(path: &str) -> Result<()> {
+    if matches!(path, FORMAT_CONFIG_FILE | FORMAT_BOOTSTRAP_FILE) {
+        Ok(())
+    } else {
+        Err(DiskError::MethodNotAllowed)
+    }
+}
+
 /// Per-disk marker present while an erasure-set heal is rebuilding this disk.
 /// `LocalDisk::disk_info` reports `healing = true` while the file exists.
 pub const HEALING_MARKER_PATH: &str = "healing.bin";
@@ -1170,6 +1185,13 @@ impl DiskAPI for Disk {
         }
     }
 
+    async fn write_format_control(&self, path: &str, data: Bytes) -> Result<()> {
+        match self {
+            Disk::Local(local_disk) => local_disk.write_format_control(path, data).await,
+            Disk::Remote(remote_disk) => remote_disk.write_format_control(path, data).await,
+        }
+    }
+
     async fn compare_and_update_file(
         &self,
         volume: &str,
@@ -1905,6 +1927,21 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
     async fn read_multiple(&self, req: ReadMultipleReq) -> Result<Vec<ReadMultipleResp>>;
     // CleanAbandonedData
     async fn write_all(&self, volume: &str, path: &str, data: Bytes) -> Result<()>;
+    /// Atomically publish a valid per-drive `FormatV3` control file in the system volume.
+    /// Deployment, topology and member UUID must agree with existing control records.
+    /// Pending intents admit equivalent plans, and retired intents reject bootstrap
+    /// republication. Success requires a durable replacement inode and parent entry.
+    /// Cancellation can leave the complete plan published. Retries revalidate the
+    /// plan under the per-disk owner before publishing it durably.
+    ///
+    /// # Errors
+    /// Returns [`DiskError::MethodNotAllowed`] for unsupported implementations,
+    /// paths or retired bootstrap intents, [`DiskError::CorruptedFormat`] for
+    /// invalid or conflicting plans, and an I/O error when publication or its
+    /// durability barrier fails.
+    async fn write_format_control(&self, _path: &str, _data: Bytes) -> Result<()> {
+        Err(DiskError::MethodNotAllowed)
+    }
     async fn read_all(&self, volume: &str, path: &str) -> Result<Bytes>;
     /// Atomically replace or remove a small control file only when its current
     /// contents match `expected`. Implementations that cannot provide this
@@ -2860,6 +2897,26 @@ pub fn count_part_not_success(part_errs: &[usize]) -> usize {
 
 #[cfg(test)]
 mod tests {
+    #[test]
+    fn format_control_paths_are_fixed_system_leaf_names() {
+        for path in [super::FORMAT_CONFIG_FILE, super::FORMAT_BOOTSTRAP_FILE] {
+            super::validate_format_control_path(path).expect("allowed format control leaf");
+        }
+        for path in [
+            "",
+            "../format.json",
+            "format.json/",
+            ".rustfs.sys/format.json",
+            "format.json.bootstrap.done",
+            "pool.bin",
+        ] {
+            assert!(matches!(
+                super::validate_format_control_path(path),
+                Err(super::DiskError::MethodNotAllowed)
+            ));
+        }
+    }
+
     use super::*;
 
     #[test]

crates/ecstore/src/store/format_bootstrap.rs

@@ -0,0 +1,245 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::init_format::{
+    check_format_erasure_values, formats_match_reference_slots, legacy_format_max_bytes, read_legacy_format_bytes,
+    retain_format_quorum_members, write_format_file,
+};
+use crate::disk::{DiskAPI, DiskStore, RUSTFS_META_BUCKET, error::DiskError, format::FormatV3};
+use crate::error::{Error, Result};
+use crate::runtime::instance::InstanceContext;
+use futures::{StreamExt, future::join_all, stream};
+use std::sync::Arc;
+use uuid::Uuid;
+
+use crate::disk::FORMAT_BOOTSTRAP_DONE_FILE;
+pub(super) use crate::disk::FORMAT_BOOTSTRAP_FILE;
+const FORMAT_BOOTSTRAP_READ_CONCURRENCY: usize = 16;
+
+pub(super) fn project_format_slots(reference: &FormatV3) -> Vec<Option<FormatV3>> {
+    reference
+        .erasure
+        .sets
+        .iter()
+        .flatten()
+        .map(|disk_id| {
+            let mut format = reference.clone();
+            format.erasure.this = *disk_id;
+            Some(format)
+        })
+        .collect()
+}
+
+/// Establishes restartable creator authority before any format is published.
+/// The first slot durably selects the plan before fanout. A retry completes
+/// that immutable plan while every configured disk remains unformatted.
+pub(super) async fn write_format_bootstrap_intents(disks: &[Option<DiskStore>], formats: &[Option<FormatV3>]) -> Result<()> {
+    if disks.is_empty() || disks.len() != formats.len() || formats.iter().any(Option::is_none) {
+        return Err(Error::CorruptedFormat);
+    }
+    if disks.iter().any(Option::is_none) {
+        return Err(DiskError::DiskNotFound.into());
+    }
+    let first = disks[0].as_ref().ok_or(DiskError::DiskNotFound)?;
+    let anchor = formats[0].as_ref().ok_or(DiskError::CorruptedFormat)?;
+    let set_drive_count = anchor.erasure.sets.first().map_or(0, Vec::len);
+    check_format_erasure_values(formats, set_drive_count)?;
+    if !formats_match_reference_slots(formats, anchor, 0) {
+        return Err(Error::CorruptedFormat);
+    }
+    let max_bytes = legacy_format_max_bytes(disks.len())?;
+    match first.read_file(RUSTFS_META_BUCKET, FORMAT_BOOTSTRAP_FILE).await {
+        Ok(reader) => {
+            let data = read_legacy_format_bytes(reader, max_bytes).await?;
+            let existing = FormatV3::try_from(data.as_slice())?;
+            if existing.shared_identity() != anchor.shared_identity() || existing.erasure.this != anchor.erasure.this {
+                return Err(Error::CorruptedFormat);
+            }
+        }
+        Err(DiskError::FileNotFound) => {
+            // A missing anchor cannot authorize replacing another slot's plan.
+            // This check is confined to fresh creation, so healthy restarts
+            // retain the single-anchor absence fast path.
+            let reads = stream::iter(disks.iter().skip(1).cloned().map(|disk| async move {
+                let disk = disk.as_ref().ok_or(DiskError::DiskNotFound)?;
+                match disk.read_file(RUSTFS_META_BUCKET, FORMAT_BOOTSTRAP_FILE).await {
+                    Err(DiskError::FileNotFound) => Ok(()),
+                    Ok(_) => Err(DiskError::CorruptedFormat),
+                    Err(error) => Err(error),
+                }
+            }))
+            .buffer_unordered(FORMAT_BOOTSTRAP_READ_CONCURRENCY);
+            tokio::pin!(reads);
+            while let Some(result) = reads.next().await {
+                result?;
+            }
+        }
+        Err(error) => return Err(error.into()),
+    }
+    first
+        .write_format_control(FORMAT_BOOTSTRAP_FILE, anchor.to_json()?.into_bytes().into())
+        .await?;
+    let writes = disks.iter().zip(formats).skip(1).map(|(disk, format)| async move {
+        let disk = disk.as_ref().ok_or(DiskError::DiskNotFound)?;
+        let format = format.as_ref().ok_or(DiskError::CorruptedFormat)?;
+        disk.write_format_control(FORMAT_BOOTSTRAP_FILE, format.to_json()?.into_bytes().into())
+            .await
+    });
+    for result in join_all(writes).await {
+        result?;
+    }
+    Ok(())
+}
+
+/// Completes an anchored plan before its first format publication, or restores
+/// explicitly unformatted slots from a complete durable intent set. Partial
+/// intent sets grant no authority once any format has been published.
+pub(super) async fn resume_format_bootstrap(
+    instance_ctx: &Arc<InstanceContext>,
+    disks: &mut [Option<DiskStore>],
+    formats: &[Option<FormatV3>],
+    errors: &[Option<DiskError>],
+    set_count: usize,
+    set_drive_count: usize,
+    deployment_id: Option<Uuid>,
+) -> Result<Option<FormatV3>> {
+    if disks.is_empty()
+        || disks.len() != formats.len()
+        || disks.len() != errors.len()
+        || set_count.checked_mul(set_drive_count) != Some(disks.len())
+    {
+        return Ok(None);
+    }
+
+    let max_bytes = legacy_format_max_bytes(disks.len())?;
+    let Some(first) = disks.first().and_then(Option::as_ref) else {
+        return Ok(None);
+    };
+    let first_intent = async {
+        let reader = first.read_file(RUSTFS_META_BUCKET, FORMAT_BOOTSTRAP_FILE).await?;
+        let data = read_legacy_format_bytes(reader, max_bytes).await?;
+        FormatV3::try_from(data.as_slice()).map_err(DiskError::from)
+    }
+    .await;
+    let Ok(first_intent) = first_intent else {
+        return Ok(None);
+    };
+    if first_intent.erasure.sets.len() != set_count
+        || first_intent.erasure.sets.iter().any(|set| set.len() != set_drive_count)
+        || deployment_id.is_some_and(|id| id != first_intent.id)
+    {
+        return Ok(None);
+    }
+    let projected = project_format_slots(&first_intent);
+    if check_format_erasure_values(&projected, set_drive_count).is_err()
+        || !formats_match_reference_slots(&[Some(first_intent.clone())], &first_intent, 0)
+    {
+        return Ok(None);
+    }
+    let all_unformatted =
+        formats.iter().all(Option::is_none) && errors.iter().all(|error| matches!(error, Some(DiskError::UnformattedDisk)));
+    let intents = {
+        let reads = stream::iter(disks.iter().cloned().enumerate().skip(1).map(|(index, disk)| async move {
+            let disk = disk.as_ref().ok_or(DiskError::DiskNotFound)?;
+            let reader = match disk.read_file(RUSTFS_META_BUCKET, FORMAT_BOOTSTRAP_FILE).await {
+                Ok(reader) => reader,
+                Err(DiskError::FileNotFound) => return Ok((index, None)),
+                Err(error) => return Err(error),
+            };
+            let data = read_legacy_format_bytes(reader, max_bytes).await?;
+            let format = FormatV3::try_from(data.as_slice())?;
+            Ok::<_, DiskError>((index, Some(format)))
+        }))
+        .buffer_unordered(FORMAT_BOOTSTRAP_READ_CONCURRENCY);
+        tokio::pin!(reads);
+        let mut intents = vec![None; disks.len()];
+        intents[0] = Some(first_intent);
+        while let Some(result) = reads.next().await {
+            let Ok((index, format)) = result else {
+                return Ok(None);
+            };
+            intents[index] = format;
+        }
+        intents
+    };
+    let Some(reference) = intents.first().and_then(Option::as_ref) else {
+        return Ok(None);
+    };
+    if !formats_match_reference_slots(&intents, reference, 0) {
+        return Err(Error::CorruptedFormat);
+    }
+    if intents.iter().any(Option::is_none) && !all_unformatted {
+        return Ok(None);
+    }
+
+    if !formats_match_reference_slots(formats, reference, 0) {
+        return Err(Error::CorruptedFormat);
+    }
+    if formats.iter().zip(errors).any(|(format, error)| {
+        if format.is_some() {
+            error.is_some()
+        } else {
+            !matches!(error, Some(DiskError::UnformattedDisk))
+        }
+    }) {
+        return Ok(None);
+    }
+    if all_unformatted {
+        // Re-acknowledge every slot under native immutable-plan admission.
+        // Merely observing a visible, unacknowledged intent supplies no barrier.
+        write_format_bootstrap_intents(disks, &projected).await?;
+    }
+
+    // Every intent and existing format has been checked before the first
+    // mutation. A retry always publishes the same deployment and slot IDs.
+    let writes = disks
+        .iter()
+        .zip(&projected)
+        .zip(formats)
+        .filter_map(|((disk, intent), existing)| {
+            if existing.is_some() {
+                return None;
+            }
+            Some(write_format_file(disk.as_ref()?, intent.as_ref()?))
+        });
+    for result in join_all(writes).await {
+        result?;
+    }
+    let mut format = reference.clone();
+    format.erasure.this = Uuid::nil();
+    format.disk_info = None;
+    let members = vec![true; disks.len()];
+    retain_format_quorum_members(instance_ctx, disks, &format, &members, None, set_drive_count).await?;
+    Ok(Some(format))
+}
+
+/// Consumes this pool's creator authority before initialized identity publication.
+/// Replay requires every intent, so one durably absent slot is sufficient. Each
+/// attempt finishes before another begins, including its directory-sync barrier.
+pub(super) async fn retire_format_bootstrap_authority(disks: &[Option<DiskStore>]) -> Result<()> {
+    let first = disks.first().and_then(Option::as_ref).into_iter();
+    let local = disks.iter().skip(1).flatten().filter(|disk| disk.is_local());
+    let remote = disks.iter().skip(1).flatten().filter(|disk| !disk.is_local());
+    let mut failure = DiskError::DiskNotFound;
+    for disk in first.chain(local).chain(remote) {
+        match disk
+            .rename_file_durable(RUSTFS_META_BUCKET, FORMAT_BOOTSTRAP_FILE, RUSTFS_META_BUCKET, FORMAT_BOOTSTRAP_DONE_FILE)
+            .await
+        {
+            Ok(()) => return Ok(()),
+            Err(error) => failure = error,
+        }
+    }
+    Err(failure.into())
+}

crates/ecstore/src/store/init.rs

@@ -235,7 +235,22 @@ async fn persist_pool_meta_for_startup_if_safe<S>(
 where
     S: EcstoreObjectIO,
 {
-    persist_pool_meta_for_startup_guarded(meta, pools, replica_state, write_state, topology_update, elected_writer, None).await
+    let mut retirement = std::pin::pin!(std::future::ready(Ok(())));
+    if write_state.startup_identity_precedes_metadata_write() {
+        complete_pool_meta_for_startup(pools.clone(), write_state, elected_writer, None, retirement.as_mut()).await?;
+    }
+    let committed = persist_pool_meta_for_startup_guarded(
+        meta,
+        pools.clone(),
+        replica_state,
+        write_state,
+        topology_update,
+        elected_writer,
+        None,
+    )
+    .await?;
+    complete_pool_meta_for_startup(pools, write_state, elected_writer, None, retirement.as_mut()).await?;
+    Ok(committed)
 }
 
 async fn persist_pool_meta_for_startup_guarded<S>(
@@ -251,6 +266,11 @@ where
     S: EcstoreObjectIO,
 {
     if !elected_writer {
+        if topology_update {
+            return Err(Error::other(
+                "pool metadata bootstrap pending: waiting for the elected writer to commit the configured topology",
+            ));
+        }
         return Ok(meta.clone());
     }
     let should_write = topology_update || (replica_state.needs_repair && replica_state.repair_write_safe);
@@ -275,12 +295,30 @@ where
             None => save_validated_pool_meta_for_startup(meta, pools.clone(), write_state).await?,
         };
     }
-    if should_write || write_state.identity_requires_repair() {
-        persist_startup_identity_with_guard(pools, write_state, true, writer.as_ref())
+    Ok(committed)
+}
+
+async fn complete_pool_meta_for_startup<S>(
+    pools: Vec<Arc<S>>,
+    write_state: &mut PoolMetaWriteState,
+    elected_writer: bool,
+    writer: Option<&Arc<rustfs_lock::NamespaceLockGuard>>,
+    before_identity_commit: impl Future<Output = Result<()>>,
+) -> Result<()>
+where
+    S: EcstoreObjectIO,
+{
+    if elected_writer && write_state.identity_requires_repair() {
+        write_state.ensure_write_safe("store init failed during prepare_pool_meta_identity")?;
+        // A committed predecessor or initial metadata commit makes retirement
+        // recoverable. Every node requires the committed configured topology
+        // and complete initialized identity before publication.
+        before_identity_commit.await?;
+        persist_startup_identity_with_guard(pools, write_state, true, writer)
             .await
             .map_err(|err| Error::other(format!("store init failed during commit_pool_meta_identity: {err}")))?;
     }
-    Ok(committed)
+    write_state.ensure_startup_identity_initialized()
 }
 
 async fn run_local_decommission_watchdog<F, Fut, W, Wait>(rx: CancellationToken, mut reconcile: F, mut wait_for_fleet_proof: W)
@@ -613,8 +651,8 @@ impl ECStore {
                     }
                 }
             }?;
-            // First-hand authority for this pool only: `Fresh` when this process
-            // formatted it, `LegacyAdoption` when it verified the migration, and
+            // Creator authority for this pool only: `Fresh` when this process
+            // formatted it or resumed its durable plan, `LegacyAdoption` for migration, and
             // `None` when it merely read a format another node created.
             pool_meta_bootstrap_authorities.push(loaded_format.pool_meta_bootstrap_authority);
             let fm = loaded_format.format;
@@ -797,15 +835,47 @@ impl ECStore {
         // distributed startup can race on the same lock and replay the prior init bug.
         {
             let write_state = &mut *startup_write_state;
-            let persist = persist_pool_meta_for_startup_guarded(
-                &installed_pool_meta,
-                self.pools.clone(),
-                pool_meta_replica_state,
-                write_state,
-                update,
-                should_persist_pool_meta,
-                startup_pool_meta_guard.clone(),
-            );
+            let persist = async {
+                let mut retirement = std::pin::pin!(async {
+                    for pool in &self.pools {
+                        let mut disks = Vec::with_capacity(pool.set_count * pool.set_drive_count);
+                        for set in &pool.disk_set {
+                            disks.extend(set.disk_inventory().await);
+                        }
+                        super::format_bootstrap::retire_format_bootstrap_authority(&disks).await?;
+                    }
+                    Ok(())
+                });
+                if write_state.startup_identity_precedes_metadata_write() {
+                    complete_pool_meta_for_startup(
+                        self.pools.clone(),
+                        write_state,
+                        should_persist_pool_meta,
+                        startup_pool_meta_guard.as_ref(),
+                        retirement.as_mut(),
+                    )
+                    .await?;
+                }
+                let committed = persist_pool_meta_for_startup_guarded(
+                    &installed_pool_meta,
+                    self.pools.clone(),
+                    pool_meta_replica_state,
+                    write_state,
+                    update,
+                    should_persist_pool_meta,
+                    startup_pool_meta_guard.clone(),
+                )
+                .await?;
+                complete_pool_meta_for_startup(
+                    self.pools.clone(),
+                    write_state,
+                    should_persist_pool_meta,
+                    startup_pool_meta_guard.as_ref(),
+                    retirement.as_mut(),
+                )
+                .await?;
+                Ok::<_, Error>(committed)
+            };
             #[cfg(feature = "e2e-test-hooks")]
             let persist = crate::core::pools::startup_cas_test_scope(startup_attempt, "persist", &self.pools, persist);
             installed_pool_meta = persist.await?;
@@ -988,11 +1058,12 @@ mod tests {
     include!("init/multipart_copy_heal_tests.rs");
     use super::{
         LOCAL_DECOMMISSION_RESUME_RETRY_DELAY, LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY, PoolMetaWriteState,
-        establish_pool_meta_bootstrap_identity_if_proven, load_pool_meta_for_startup, local_decommission_watchdog_retry_delay,
-        persist_pool_meta_for_startup_if_safe, pool_first_endpoint_is_local, pool_meta_has_active_decommission,
-        preflight_startup_rpc_secret_with, resolve_startup_pool_defaults_with, resolve_store_init_stage_result,
-        run_local_decommission_watchdog, save_validated_pool_meta_for_startup, should_auto_start_rebalance_after_init,
-        should_defer_rebalance_auto_start, should_retry_format_load, wait_for_local_decommission_resume_delay,
+        complete_pool_meta_for_startup, establish_pool_meta_bootstrap_identity_if_proven, load_pool_meta_for_startup,
+        local_decommission_watchdog_retry_delay, persist_pool_meta_for_startup_guarded, persist_pool_meta_for_startup_if_safe,
+        pool_first_endpoint_is_local, pool_meta_has_active_decommission, preflight_startup_rpc_secret_with,
+        resolve_startup_pool_defaults_with, resolve_store_init_stage_result, run_local_decommission_watchdog,
+        save_validated_pool_meta_for_startup, should_auto_start_rebalance_after_init, should_defer_rebalance_auto_start,
+        should_retry_format_load, wait_for_local_decommission_resume_delay,
     };
     use crate::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
     use crate::core::pools::PoolMetaBootstrapAuthority;
@@ -1570,6 +1641,184 @@ mod tests {
             .await
             .expect_err("wiping pool metadata after the crash must require recovery");
         assert!(err.to_string().contains("initialized cluster identity exists"));
+        restarted
+            .ensure_write_safe("metadata erased after failed identity repair")
+            .expect_err("metadata loss cannot recreate creator authority");
+    }
+
+    #[tokio::test]
+    async fn test_startup_identity_publication_waits_for_bootstrap_retirement() {
+        let deployment_id = Uuid::new_v4();
+        let storage = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
+        let pools = vec![storage.clone()];
+        let mut state = PoolMetaWriteState::for_startup(deployment_id, true);
+        establish_pool_meta_bootstrap_identity_if_proven(pools.clone(), &mut state, true)
+            .await
+            .expect("persist initial pending identity");
+        let (_, replicas) = load_pool_meta_for_startup(pools.clone(), &mut state)
+            .await
+            .expect("load fresh bootstrap state");
+        persist_pool_meta_for_startup_guarded(&init_test_pool_meta(None), pools.clone(), replicas, &mut state, true, true, None)
+            .await
+            .expect("commit initial pool metadata before consuming creator authority");
+        let identity_writes = storage.identity_write_attempts.load(Ordering::SeqCst);
+
+        complete_pool_meta_for_startup(pools.clone(), &mut state, true, None, async {
+            let objects = storage.objects.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
+            let metadata = objects
+                .get(POOL_META_NAME)
+                .expect("committed metadata must precede retirement");
+            assert_eq!(
+                pool_meta_v3_commit_state_for_test(metadata.0.clone()).expect("decode committed metadata"),
+                (1, true)
+            );
+            let identity = objects
+                .get(POOL_META_IDENTITY_NAME)
+                .expect("pending identity must remain available");
+            assert!(
+                !pool_meta_identity_initialized_for_test(&identity.0).expect("decode pending identity"),
+                "initialized publication must follow durable retirement"
+            );
+            Err(Error::Timeout)
+        })
+        .await
+        .expect_err("failed retirement must prevent initialized identity publication");
+        assert_eq!(storage.identity_write_attempts.load(Ordering::SeqCst), identity_writes);
+        state
+            .ensure_startup_identity_initialized()
+            .expect_err("a failed retirement must keep startup unpublished");
+
+        // Restarting after retirement failed reuses the committed metadata,
+        // without requiring the formatter's process-local bootstrap proof.
+        let mut restarted = PoolMetaWriteState::for_startup(deployment_id, false);
+        load_pool_meta_for_startup(pools.clone(), &mut restarted)
+            .await
+            .expect("committed metadata authorizes the retry");
+        complete_pool_meta_for_startup(pools, &mut restarted, true, None, std::future::ready(Ok(())))
+            .await
+            .expect("successful retirement permits initialized identity publication");
+        assert!(storage.identity_write_attempts.load(Ordering::SeqCst) > identity_writes);
+        restarted
+            .ensure_startup_identity_initialized()
+            .expect("completed initialization permits startup");
+    }
+
+    #[tokio::test]
+    async fn test_startup_followers_wait_for_initialized_identity_on_every_pool() {
+        let deployment_id = Uuid::new_v4();
+        let initialized = initialized_pool_meta_identity_for_test(deployment_id, 1).expect("encode initialized identity");
+        let pending = pending_pool_meta_identity_for_test(deployment_id, 1, Uuid::new_v4()).expect("encode pending identity");
+        for incomplete in [None, Some(pending), Some(vec![0, 1, 2])] {
+            let payload = startup_pool_meta_payload(&init_test_pool_meta(None));
+            let first = Arc::new(StartupPoolMetaStorage::new(payload.clone()));
+            let second = Arc::new(StartupPoolMetaStorage::new(payload));
+            first
+                .objects
+                .lock()
+                .unwrap_or_else(std::sync::PoisonError::into_inner)
+                .insert(POOL_META_IDENTITY_NAME.to_owned(), (initialized.clone(), "identity-0".to_owned()));
+            if let Some(identity) = incomplete {
+                second
+                    .objects
+                    .lock()
+                    .unwrap_or_else(std::sync::PoisonError::into_inner)
+                    .insert(POOL_META_IDENTITY_NAME.to_owned(), (identity, "identity-1".to_owned()));
+            }
+            let pools = vec![first.clone(), second.clone()];
+            let mut state = PoolMetaWriteState::for_startup(deployment_id, false);
+            load_pool_meta_for_startup(pools.clone(), &mut state)
+                .await
+                .expect("a readable committed snapshot can await identity convergence");
+            complete_pool_meta_for_startup(pools.clone(), &mut state, false, None, async {
+                panic!("a follower must never execute intent retirement")
+            })
+            .await
+            .expect_err("an initialized replica cannot hide another pool's incomplete identity");
+            state
+                .ensure_write_safe("follower bootstrap retry")
+                .expect("waiting must remain retryable");
+
+            second
+                .objects
+                .lock()
+                .unwrap_or_else(std::sync::PoisonError::into_inner)
+                .insert(POOL_META_IDENTITY_NAME.to_owned(), (initialized.clone(), "identity-1-ready".to_owned()));
+            load_pool_meta_for_startup(pools.clone(), &mut state)
+                .await
+                .expect("observe initialized identity on every pool");
+            for elected_writer in [false, true] {
+                complete_pool_meta_for_startup(pools.clone(), &mut state, elected_writer, None, async {
+                    panic!("established startup must perform no intent retirement I/O")
+                })
+                .await
+                .expect("complete initialized identity permits read-only startup");
+            }
+            assert_eq!(first.identity_write_attempts.load(Ordering::SeqCst), 0);
+            assert_eq!(second.identity_write_attempts.load(Ordering::SeqCst), 0);
+        }
+    }
+
+    #[tokio::test]
+    async fn test_startup_expansion_retires_before_identity_repair_and_waits_for_topology_commit() {
+        let deployment_id = Uuid::new_v4();
+        let first = Arc::new(StartupPoolMetaStorage::new(startup_pool_meta_payload(&init_test_pool_meta_with_pools(1))));
+        let second = Arc::new(StartupPoolMetaStorage::new(Vec::new()));
+        inject_startup_object(
+            &first,
+            POOL_META_IDENTITY_NAME,
+            initialized_pool_meta_identity_for_test(deployment_id, 1).expect("encode predecessor identity"),
+        );
+        let pools = vec![first.clone(), second.clone()];
+        let mut state = PoolMetaWriteState::for_startup(deployment_id, false);
+        let (_, replicas) = load_pool_meta_for_startup(pools.clone(), &mut state)
+            .await
+            .expect("load committed predecessor with a new pool");
+        assert!(state.startup_identity_precedes_metadata_write());
+
+        complete_pool_meta_for_startup(pools.clone(), &mut state, true, None, async { Err(Error::Timeout) })
+            .await
+            .expect_err("failed retirement must stop identity repair before expansion");
+        for pool in &pools {
+            assert_eq!(pool.identity_write_attempts.load(Ordering::SeqCst), 0);
+            assert_eq!(pool.pool_meta_write_attempts.load(Ordering::SeqCst), 0);
+        }
+        complete_pool_meta_for_startup(pools.clone(), &mut state, true, None, async {
+            assert!(startup_object(&second, POOL_META_IDENTITY_NAME).is_none());
+            assert!(startup_object(&first, POOL_META_NAME).is_some());
+            assert!(startup_object(&second, POOL_META_NAME).is_none());
+            Ok(())
+        })
+        .await
+        .expect("retirement authorizes repairing initialized identities from the predecessor");
+        state
+            .ensure_startup_identity_initialized()
+            .expect("every pool identity is repaired before native metadata publication");
+        let expanded = init_test_pool_meta_with_pools(2);
+        let identity_writes = pools
+            .iter()
+            .map(|pool| pool.identity_write_attempts.load(Ordering::SeqCst))
+            .collect::<Vec<_>>();
+        let error = persist_pool_meta_for_startup_guarded(&expanded, pools.clone(), replicas, &mut state, true, false, None)
+            .await
+            .expect_err("a follower cannot publish locally computed expanded topology");
+        assert!(error.to_string().contains("commit the configured topology"));
+        for (pool, identity_writes) in pools.iter().zip(identity_writes) {
+            assert_eq!(pool.identity_write_attempts.load(Ordering::SeqCst), identity_writes);
+            assert_eq!(pool.pool_meta_write_attempts.load(Ordering::SeqCst), 0);
+        }
+        state
+            .ensure_write_safe("waiting for topology commit")
+            .expect("a follower retry must stay writable");
+
+        let committed = persist_pool_meta_for_startup_guarded(&expanded, pools.clone(), replicas, &mut state, true, true, None)
+            .await
+            .expect("the elected writer can commit expansion after identity repair");
+        assert_eq!(committed.pools.len(), 2);
+        complete_pool_meta_for_startup(pools, &mut state, true, None, async {
+            panic!("the completed preflight retirement must not run twice")
+        })
+        .await
+        .expect("completed expansion remains initialized");
     }
 
     #[tokio::test]
@@ -3482,6 +3731,128 @@ mod tests {
         (instance_ctx, store, shutdown)
     }
 
+    #[tokio::test]
+    #[serial_test::serial(storage_class_env)]
+    async fn interrupted_format_bootstrap_recovers_two_pools_through_store_init() {
+        use crate::disk::{DiskAPI, DiskOption, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET};
+        use crate::store::init_format;
+        let directory = tempfile::tempdir().expect("create interrupted-bootstrap deployment");
+        let deployment_id = Uuid::new_v4();
+        let mut plans = Vec::new();
+        for pool_index in 0..2 {
+            let mut endpoints = Vec::new();
+            for index in 0..4 {
+                let path = directory.path().join(format!("pool{pool_index}/set0/disk{index}"));
+                tokio::fs::create_dir_all(&path).await.expect("create bootstrap disk");
+                let mut endpoint = Endpoint::try_from(path.to_str().expect("UTF-8 path")).expect("disk endpoint");
+                endpoint.set_pool_index(pool_index);
+                endpoint.set_set_index(0);
+                endpoint.set_disk_index(index);
+                endpoints.push(endpoint);
+            }
+            let (disks, errors) = init_format::init_disks(
+                &crate::runtime::instance::InstanceContext::new(),
+                &Endpoints::from(endpoints),
+                &DiskOption {
+                    cleanup: false,
+                    health_check: false,
+                },
+            )
+            .await;
+            assert!(errors.iter().all(Option::is_none), "open bootstrap disks: {errors:?}");
+            let mut reference = crate::disk::format::FormatV3::new(1, 4);
+            reference.id = deployment_id;
+            let formats = (0..4)
+                .map(|index| {
+                    let mut format = reference.clone();
+                    format.erasure.this = reference.erasure.sets[0][index];
+                    Some(format)
+                })
+                .collect::<Vec<_>>();
+            super::super::format_bootstrap::write_format_bootstrap_intents(&disks, &formats)
+                .await
+                .expect("persist creator plan");
+            for (disk, format) in disks.iter().zip(&formats).take(3) {
+                disk.as_ref()
+                    .expect("fixture disk")
+                    .write_all(
+                        RUSTFS_META_BUCKET,
+                        FORMAT_CONFIG_FILE,
+                        format
+                            .as_ref()
+                            .expect("planned format")
+                            .to_json()
+                            .expect("encode format")
+                            .into_bytes()
+                            .into(),
+                    )
+                    .await
+                    .expect("publish first three formats");
+            }
+            plans.push(formats);
+        }
+        // A new context and new disk handles exercise the normal constructor's
+        // format recovery, pool identity transaction, and publication barrier.
+        let (_, store, shutdown) =
+            without_storage_class_env(build_isolated_test_store(directory.path(), "interrupted-bootstrap", &[4, 4])).await;
+        assert_eq!(store.id, deployment_id);
+        for (pool, expected) in store.pools.iter().zip(&plans) {
+            let mut retired = 0;
+            for set in &pool.disk_set {
+                let disks = set.disk_inventory().await;
+                let (formats, errors) = init_format::load_format_erasure_all(&disks, false).await;
+                assert!(errors.iter().all(Option::is_none), "recovered formats: {errors:?}");
+                assert_eq!(&formats, expected);
+                for disk in disks.iter().flatten() {
+                    if matches!(
+                        disk.read_all(RUSTFS_META_BUCKET, "format.json.bootstrap").await,
+                        Err(crate::disk::error::DiskError::FileNotFound)
+                    ) {
+                        retired += 1;
+                        assert!(disk.read_all(RUSTFS_META_BUCKET, "format.json.bootstrap.done").await.is_ok());
+                    }
+                }
+            }
+            assert_eq!(retired, 1, "one durably absent intent consumes each pool's complete creator proof");
+        }
+        store
+            .pool_meta_save_gate
+            .read()
+            .await
+            .ensure_startup_identity_initialized()
+            .expect("every pool identity is initialized before publication");
+        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
+        store
+            .make_bucket("bootstrap-recovered", &MakeBucketOptions::default())
+            .await
+            .expect("create bucket after recovered startup");
+        store
+            .put_object(
+                "bootstrap-recovered",
+                "payload",
+                &mut PutObjReader::from_vec(b"preserved".to_vec()),
+                &ObjectOptions {
+                    write_completion: crate::object_api::WriteCompletion::TailDrained,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("write after recovered startup");
+        let info = store
+            .get_object_info("bootstrap-recovered", "payload", &ObjectOptions::default())
+            .await
+            .expect("read committed object metadata");
+        assert_eq!(info.size, 9);
+        let mut reader = store
+            .get_object_reader("bootstrap-recovered", "payload", None, HeaderMap::new(), &ObjectOptions::default())
+            .await
+            .expect("read object after recovered startup");
+        let mut payload = Vec::new();
+        reader.stream.read_to_end(&mut payload).await.expect("read recovered payload");
+        assert_eq!(payload, b"preserved");
+        shutdown.cancel();
+    }
+
     #[tokio::test]
     #[serial_test::serial(storage_class_env)]
     async fn direct_local_constructor_multipart_binds_complete_topology_without_registry_preinit() {

crates/ecstore/src/store/init_format.rs

@@ -12,6 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#[cfg(test)]
+use super::format_bootstrap::{FORMAT_BOOTSTRAP_FILE, retire_format_bootstrap_authority};
+use super::format_bootstrap::{project_format_slots, resume_format_bootstrap, write_format_bootstrap_intents};
 use crate::cluster::rpc::client::is_network_like_disk_error;
 use crate::config::storageclass;
 use crate::core::pools::PoolMetaBootstrapAuthority;
@@ -120,6 +123,17 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
     let (formats, errs, unformatted_bootstrap_routes) = load_format_erasure_all_observing_bootstrap_routes(disks, false).await;
 
     check_disk_fatal_errs(&errs)?;
+    // Only the elected pool creator may replay a complete durable format plan.
+    // Absence of pool metadata or an orphaned format temp file grants no authority.
+    if first_disk
+        && let Some(format) =
+            resume_format_bootstrap(instance_ctx, disks, &formats, &errs, set_count, set_drive_count, deployment_id).await?
+    {
+        return Ok(LoadedFormat {
+            format,
+            pool_meta_bootstrap_authority: PoolMetaBootstrapAuthority::Fresh,
+        });
+    }
 
     // Transient network errors still follow the first-node wait path so peers
     // can come online, but they are never fresh-cluster authority below.
@@ -270,7 +284,7 @@ async fn verified_legacy_adoption_source(
     }
 }
 
-async fn retain_format_quorum_members(
+pub(super) async fn retain_format_quorum_members(
     instance_ctx: &Arc<InstanceContext>,
     disks: &mut [Option<DiskStore>],
     format: &FormatV3,
@@ -403,21 +417,13 @@ async fn init_format_erasure(
     set_drive_count: usize,
     deployment_id: Option<Uuid>,
 ) -> Result<FormatV3> {
-    let fm = FormatV3::new(set_count, set_drive_count);
-    let mut fms = vec![None; disks.len()];
-    for i in 0..set_count {
-        for j in 0..set_drive_count {
-            let idx = i * set_drive_count + j;
-            let mut newfm = fm.clone();
-            newfm.erasure.this = fm.erasure.sets[i][j];
-            if let Some(id) = deployment_id {
-                newfm.id = id;
-            }
-
-            fms[idx] = Some(newfm);
-        }
+    let mut fm = FormatV3::new(set_count, set_drive_count);
+    if let Some(id) = deployment_id {
+        fm.id = id;
     }
+    let fms = project_format_slots(&fm);
 
+    write_format_bootstrap_intents(disks, &fms).await?;
     write_format_file_all(disks, &fms).await?;
     let format = get_format_erasure_in_quorum(&fms, 0)?;
     let quorum_members = vec![true; disks.len()];
@@ -602,14 +608,17 @@ fn select_persisted_migration_format(
     Err(write_error.map_or(quorum_error, Into::into))
 }
 
-fn legacy_format_max_bytes(disk_count: usize) -> Result<usize> {
+pub(super) fn legacy_format_max_bytes(disk_count: usize) -> Result<usize> {
     disk_count
         .checked_mul(LEGACY_FORMAT_MAX_BYTES_PER_DISK)
         .and_then(|size| size.checked_add(LEGACY_FORMAT_BASE_MAX_BYTES))
         .ok_or_else(|| Error::other("legacy format size limit overflow"))
 }
 
-async fn read_legacy_format_bytes(reader: impl tokio::io::AsyncRead + Unpin, max_bytes: usize) -> disk::error::Result<Vec<u8>> {
+pub(super) async fn read_legacy_format_bytes(
+    reader: impl tokio::io::AsyncRead + Unpin,
+    max_bytes: usize,
+) -> disk::error::Result<Vec<u8>> {
     let read_limit = max_bytes.checked_add(1).ok_or(DiskError::CorruptedFormat)?;
     let mut data = Vec::with_capacity(read_limit.min(64 * 1024));
     reader
@@ -993,24 +1002,23 @@ pub async fn save_format_file(disk: &Option<DiskStore>, format: &Option<FormatV3
         return Err(DiskError::other("format is none"));
     };
 
-    write_format_file(disk, format).await?;
+    // Replacement healing retains the interoperable write/rename contract
+    // while configured peers are upgraded independently.
+    let temporary = Uuid::new_v4().to_string();
+    disk.write_all(RUSTFS_META_BUCKET, &temporary, format.to_json()?.into_bytes().into())
+        .await?;
+    disk.rename_file_durable(RUSTFS_META_BUCKET, &temporary, RUSTFS_META_BUCKET, FORMAT_CONFIG_FILE)
+        .await?;
     disk.set_disk_id(Some(format.erasure.this)).await?;
 
     Ok(())
 }
 
-async fn write_format_file(disk: &DiskStore, format: &FormatV3) -> disk::error::Result<()> {
+pub(super) async fn write_format_file(disk: &DiskStore, format: &FormatV3) -> disk::error::Result<()> {
     let json_data = format.to_json()?;
 
-    let tmpfile = Uuid::new_v4().to_string();
-
-    disk.write_all(RUSTFS_META_BUCKET, tmpfile.as_str(), json_data.into_bytes().into())
-        .await?;
-
-    disk.rename_file(RUSTFS_META_BUCKET, tmpfile.as_str(), RUSTFS_META_BUCKET, FORMAT_CONFIG_FILE)
-        .await?;
-
-    Ok(())
+    disk.write_format_control(FORMAT_CONFIG_FILE, json_data.into_bytes().into())
+        .await
 }
 
 pub fn ec_drives_no_config(set_drive_count: usize) -> Result<usize> {
@@ -1028,7 +1036,7 @@ mod tests {
     use crate::store::peer::{find_local_disk_by_ref, prewarm_local_disk_id_map_with_instance_ctx};
     use serial_test::serial;
 
-    async fn local_disks(count: usize) -> (tempfile::TempDir, Vec<Option<DiskStore>>) {
+    pub(super) async fn local_disks(count: usize) -> (tempfile::TempDir, Vec<Option<DiskStore>>) {
         local_disks_with_set_width(count, count).await
     }
 
@@ -1995,7 +2003,7 @@ mod tests {
                 .expect("format publication path");
             let (arrived, arrival) = tokio::sync::oneshot::channel();
             let (release, released) = std::sync::mpsc::channel();
-            hooks.push(install_at(Stage::Rename, &destination, move || {
+            hooks.push(install_at(Stage::PreparedRename, &destination, move || {
                 let _ = arrived.send(());
                 let _ = released.recv();
             }));
@@ -2192,6 +2200,33 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn replacement_format_publication_keeps_interoperable_rename_contract() {
+        use crate::disk::os::prepared_publication_test_hooks::{Stage, install_at};
+        use std::sync::atomic::{AtomicUsize, Ordering};
+
+        let (_directory, disks) = local_disks(1).await;
+        let disk = disks[0].as_ref().expect("replacement fixture disk");
+        let destination = disk.path().join(RUSTFS_META_BUCKET).join(FORMAT_CONFIG_FILE);
+        let renamed = Arc::new(AtomicUsize::new(0));
+        let observed = renamed.clone();
+        let _hook = install_at(Stage::Rename, &destination, move || {
+            observed.fetch_add(1, Ordering::SeqCst);
+        });
+        let mut format = FormatV3::new(1, 1);
+        format.erasure.this = format.erasure.sets[0][0];
+        save_format_file(&disks[0], &Some(format.clone()))
+            .await
+            .expect("publish replacement through established RPCs");
+        assert_eq!(
+            renamed.load(Ordering::SeqCst),
+            1,
+            "replacement publication uses the interoperable durable rename"
+        );
+        assert_eq!(load_format_erasure(disk, false).await.expect("load replacement format"), format);
+        assert_eq!(disk.get_disk_id().await.expect("load replacement identity"), Some(format.erasure.this));
+    }
+
     #[tokio::test]
     async fn fresh_format_load_initializes_all_disks() {
         let (_temp_dir, mut disks) = local_disks(3).await;
@@ -2649,3 +2684,7 @@ mod tests {
 //         }
 //     }
 // }
+
+#[cfg(test)]
+#[path = "init_format_bootstrap_tests.rs"]
+mod bootstrap_tests;

crates/ecstore/src/store/init_format_bootstrap_tests.rs

@@ -0,0 +1,649 @@
+// Copyright 2026 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::tests::local_disks;
+use super::*;
+
+async fn reopen_disks(disks: &[Option<DiskStore>]) -> Vec<Option<DiskStore>> {
+    let endpoints = Endpoints::from(
+        disks
+            .iter()
+            .map(|disk| disk.as_ref().expect("fixture disk exists").endpoint())
+            .collect::<Vec<_>>(),
+    );
+    let (reopened, errors) = init_disks(
+        &InstanceContext::new(),
+        &endpoints,
+        &DiskOption {
+            cleanup: false,
+            health_check: false,
+        },
+    )
+    .await;
+    assert!(errors.iter().all(Option::is_none), "reopen persisted disks: {errors:?}");
+    reopened
+}
+
+fn bootstrap_plan() -> Vec<Option<FormatV3>> {
+    let reference = FormatV3::new(1, 4);
+    (0..4)
+        .map(|index| {
+            let mut format = reference.clone();
+            format.erasure.this = reference.erasure.sets[0][index];
+            Some(format)
+        })
+        .collect()
+}
+
+async fn format_snapshot(disks: &[Option<DiskStore>]) -> Vec<Option<Vec<u8>>> {
+    let mut snapshot = Vec::with_capacity(disks.len());
+    for disk in disks {
+        let path = disk
+            .as_ref()
+            .expect("snapshot disk exists")
+            .path()
+            .join(RUSTFS_META_BUCKET)
+            .join(FORMAT_CONFIG_FILE);
+        match tokio::fs::read(path).await {
+            Ok(bytes) => snapshot.push(Some(bytes)),
+            Err(error) if error.kind() == std::io::ErrorKind::NotFound => snapshot.push(None),
+            Err(error) => panic!("read persisted format: {error}"),
+        }
+    }
+    snapshot
+}
+
+async fn assert_plan_published(disks: &[Option<DiskStore>], expected: &[Option<FormatV3>]) {
+    let (actual, errors) = load_format_erasure_all(disks, false).await;
+    assert!(errors.iter().all(Option::is_none), "load recovered formats: {errors:?}");
+    assert_eq!(actual, expected, "bootstrap recovery must preserve the deployment and every disk UUID");
+}
+
+#[tokio::test]
+async fn bootstrap_intent_survives_formatter_restart() {
+    let (_directory, mut disks) = local_disks(4).await;
+    let first = connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut disks, 1, 4, None)
+        .await
+        .expect("format fresh deployment");
+    let before = format_snapshot(&disks).await;
+
+    let mut reopened = reopen_disks(&disks).await;
+    let loaded = connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut reopened, 1, 4, None)
+        .await
+        .expect("resume bootstrap after process restart");
+
+    assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::Fresh);
+    assert_eq!(loaded.format.shared_identity(), first.format.shared_identity());
+    assert_eq!(
+        format_snapshot(&reopened).await,
+        before,
+        "already published formats remain byte-identical"
+    );
+}
+
+#[tokio::test]
+async fn bootstrap_intent_completes_interrupted_fourth_format_after_restart() {
+    let (_directory, disks) = local_disks(4).await;
+    let plan = bootstrap_plan();
+    write_format_bootstrap_intents(&disks, &plan)
+        .await
+        .expect("persist bootstrap authority before formats");
+    write_format_file_all(&disks[..3], &plan[..3])
+        .await
+        .expect("publish first three formats");
+    let interrupted_disk = disks[3].as_ref().expect("fourth disk exists");
+    let staged = Uuid::new_v4().to_string();
+    interrupted_disk
+        .write_all(
+            RUSTFS_META_BUCKET,
+            &staged,
+            plan[3]
+                .as_ref()
+                .expect("fourth planned format")
+                .to_json()
+                .expect("encode planned format")
+                .into_bytes()
+                .into(),
+        )
+        .await
+        .expect("stage the fourth format without its rename");
+    let before = format_snapshot(&disks).await;
+    assert!(before[3].is_none(), "reproducer has no fourth format.json");
+
+    let mut reopened = reopen_disks(&disks).await;
+    let loaded = connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut reopened, 1, 4, None)
+        .await
+        .expect("finish interrupted format publication before identity reads");
+
+    assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::Fresh);
+    assert_plan_published(&reopened, &plan).await;
+    assert_eq!(
+        format_snapshot(&reopened).await[..3],
+        before[..3],
+        "recovery only publishes the unformatted slot"
+    );
+}
+
+#[tokio::test]
+async fn bootstrap_intent_reuses_plan_when_no_formats_were_published() {
+    let (_directory, disks) = local_disks(4).await;
+    let plan = bootstrap_plan();
+    write_format_bootstrap_intents(&disks, &plan)
+        .await
+        .expect("persist the complete format plan");
+    assert!(format_snapshot(&disks).await.iter().all(Option::is_none));
+
+    let mut reopened = reopen_disks(&disks).await;
+    let loaded = connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut reopened, 1, 4, None)
+        .await
+        .expect("resume before first format publication");
+
+    assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::Fresh);
+    assert_plan_published(&reopened, &plan).await;
+}
+
+#[tokio::test]
+async fn bootstrap_intent_anchor_is_durable_before_fanout() {
+    use crate::disk::os::prepared_publication_test_hooks::{Stage, install_at};
+
+    let (_directory, disks) = local_disks(4).await;
+    let plan = bootstrap_plan();
+    let anchor = disks[0]
+        .as_ref()
+        .expect("anchor disk")
+        .path()
+        .join(RUSTFS_META_BUCKET)
+        .join(FORMAT_BOOTSTRAP_FILE);
+    let (arrived, arrival) = tokio::sync::oneshot::channel();
+    let (release, released) = std::sync::mpsc::channel();
+    let _hook = install_at(Stage::PreparedRename, &anchor, move || {
+        let _ = arrived.send(());
+        let _ = released.recv();
+    });
+    let publication = tokio::spawn({
+        let disks = disks.clone();
+        let plan = plan.clone();
+        async move { write_format_bootstrap_intents(&disks, &plan).await }
+    });
+    tokio::time::timeout(std::time::Duration::from_secs(5), arrival)
+        .await
+        .expect("anchor reaches native publication")
+        .expect("anchor publication hook");
+    for disk in disks.iter().skip(1).flatten() {
+        assert!(matches!(
+            disk.read_all(RUSTFS_META_BUCKET, FORMAT_BOOTSTRAP_FILE).await,
+            Err(DiskError::FileNotFound)
+        ));
+    }
+    release.send(()).expect("release anchor publication");
+    publication
+        .await
+        .expect("join anchor-first publication")
+        .expect("publish the selected plan");
+    assert!(format_snapshot(&disks).await.iter().all(Option::is_none));
+}
+
+#[tokio::test]
+async fn bootstrap_intent_anchor_only_restart_preserves_selected_plan() {
+    let (_directory, disks) = local_disks(4).await;
+    let plan = bootstrap_plan();
+    disks[0]
+        .as_ref()
+        .expect("anchor disk")
+        .write_format_control(
+            FORMAT_BOOTSTRAP_FILE,
+            plan[0]
+                .as_ref()
+                .expect("anchor format")
+                .to_json()
+                .expect("encode selected plan")
+                .into_bytes()
+                .into(),
+        )
+        .await
+        .expect("persist only the anchor before interruption");
+
+    let mut reopened = reopen_disks(&disks).await;
+    let loaded = connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut reopened, 1, 4, None)
+        .await
+        .expect("complete fanout from the accepted anchor after restart");
+    assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::Fresh);
+    assert_plan_published(&reopened, &plan).await;
+}
+
+#[tokio::test]
+async fn bootstrap_intent_missing_anchor_cannot_replace_partial_plan() {
+    let (_directory, disks) = local_disks(4).await;
+    let plan = bootstrap_plan();
+    let existing = plan[1]
+        .as_ref()
+        .expect("second slot format")
+        .to_json()
+        .expect("encode partial plan");
+    disks[1]
+        .as_ref()
+        .expect("second disk")
+        .write_all(RUSTFS_META_BUCKET, FORMAT_BOOTSTRAP_FILE, existing.clone().into_bytes().into())
+        .await
+        .expect("persist a partial plan with no anchor");
+    let mut reopened = reopen_disks(&disks).await;
+    connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut reopened, 1, 4, None)
+        .await
+        .err()
+        .expect("partial plan cannot authorize selecting another anchor");
+    assert!(format_snapshot(&disks).await.iter().all(Option::is_none));
+    assert!(matches!(
+        disks[0]
+            .as_ref()
+            .expect("anchor disk")
+            .read_all(RUSTFS_META_BUCKET, FORMAT_BOOTSTRAP_FILE)
+            .await,
+        Err(DiskError::FileNotFound)
+    ));
+    assert_eq!(
+        disks[1]
+            .as_ref()
+            .expect("second disk")
+            .read_all(RUSTFS_META_BUCKET, FORMAT_BOOTSTRAP_FILE)
+            .await
+            .expect("read preserved partial plan")
+            .as_ref(),
+        existing.as_bytes()
+    );
+}
+
+#[tokio::test]
+async fn bootstrap_intent_conflicting_fanout_cannot_change_anchor_or_formats() {
+    let (_directory, disks) = local_disks(4).await;
+    let selected = bootstrap_plan();
+    let foreign = bootstrap_plan();
+    for (index, plan) in [(0, &selected), (1, &foreign)] {
+        disks[index]
+            .as_ref()
+            .expect("fixture disk")
+            .write_all(
+                RUSTFS_META_BUCKET,
+                FORMAT_BOOTSTRAP_FILE,
+                plan[index]
+                    .as_ref()
+                    .expect("planned slot")
+                    .to_json()
+                    .expect("encode planned slot")
+                    .into_bytes()
+                    .into(),
+            )
+            .await
+            .expect("persist conflicting observed plans");
+    }
+    let mut reopened = reopen_disks(&disks).await;
+    connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut reopened, 1, 4, None)
+        .await
+        .err()
+        .expect("a conflicting intent cannot be replaced by the selected anchor");
+    assert!(format_snapshot(&disks).await.iter().all(Option::is_none));
+    let anchor = disks[0]
+        .as_ref()
+        .expect("anchor disk")
+        .read_all(RUSTFS_META_BUCKET, FORMAT_BOOTSTRAP_FILE)
+        .await
+        .expect("read preserved anchor");
+    assert_eq!(
+        FormatV3::try_from(anchor.as_ref()).expect("decode preserved anchor"),
+        *selected[0].as_ref().expect("selected anchor")
+    );
+}
+
+#[tokio::test]
+async fn bootstrap_intent_partial_proof_cannot_fill_a_partially_formatted_pool() {
+    let (_directory, disks) = local_disks(4).await;
+    let plan = bootstrap_plan();
+    write_format_bootstrap_intents(&disks, &plan)
+        .await
+        .expect("persist the selected plan");
+    write_format_file_all(&disks[..3], &plan[..3])
+        .await
+        .expect("publish only three formats");
+    let missing = disks[3]
+        .as_ref()
+        .expect("fourth disk")
+        .path()
+        .join(RUSTFS_META_BUCKET)
+        .join(FORMAT_BOOTSTRAP_FILE);
+    tokio::fs::remove_file(&missing).await.expect("remove one proof replica");
+    let before = format_snapshot(&disks).await;
+    let mut reopened = reopen_disks(&disks).await;
+    if let Ok(loaded) =
+        connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut reopened, 1, 4, None).await
+    {
+        assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
+    }
+    assert_eq!(format_snapshot(&disks).await, before);
+    assert!(!missing.exists(), "partial format state cannot recreate missing proof");
+}
+
+#[tokio::test]
+async fn bootstrap_intent_absence_does_not_reinitialize_existing_formats() {
+    let (_directory, mut disks) = local_disks(4).await;
+    let plan = bootstrap_plan();
+    write_format_file_all(&disks, &plan)
+        .await
+        .expect("publish historical formats without an intent");
+    let before = format_snapshot(&disks).await;
+
+    let loaded = connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut disks, 1, 4, None)
+        .await
+        .expect("load existing deployment without granting bootstrap authority");
+
+    assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
+    assert_eq!(format_snapshot(&disks).await, before);
+}
+
+#[tokio::test]
+async fn bootstrap_intent_incomplete_or_invalid_cannot_grant_authority() {
+    for invalid_state in ["missing", "corrupt", "foreign", "wrong-slot"] {
+        let (_directory, mut disks) = local_disks(4).await;
+        let plan = bootstrap_plan();
+        write_format_bootstrap_intents(&disks, &plan)
+            .await
+            .expect("persist complete authority");
+        write_format_file_all(&disks, &plan).await.expect("publish expected formats");
+        let intent_path = disks[3]
+            .as_ref()
+            .expect("fourth disk exists")
+            .path()
+            .join(RUSTFS_META_BUCKET)
+            .join(FORMAT_BOOTSTRAP_FILE);
+        match invalid_state {
+            "missing" => tokio::fs::remove_file(&intent_path).await.expect("remove one intent replica"),
+            "corrupt" => tokio::fs::write(&intent_path, b"invalid JSON")
+                .await
+                .expect("corrupt one intent replica"),
+            "foreign" => {
+                let foreign = bootstrap_plan();
+                tokio::fs::write(
+                    &intent_path,
+                    foreign[3]
+                        .as_ref()
+                        .expect("foreign fourth format")
+                        .to_json()
+                        .expect("encode foreign intent"),
+                )
+                .await
+                .expect("replace one intent with another deployment");
+            }
+            "wrong-slot" => {
+                tokio::fs::write(
+                    &intent_path,
+                    plan[2]
+                        .as_ref()
+                        .expect("third planned format")
+                        .to_json()
+                        .expect("encode wrong-slot intent"),
+                )
+                .await
+                .expect("copy another slot's intent");
+            }
+            _ => unreachable!("all fixture states are enumerated"),
+        }
+        let before = format_snapshot(&disks).await;
+
+        let result =
+            connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut disks, 1, 4, None).await;
+
+        if let Ok(loaded) = result {
+            assert_eq!(
+                loaded.pool_meta_bootstrap_authority,
+                PoolMetaBootstrapAuthority::None,
+                "{invalid_state} intent cannot authorize pool metadata initialization"
+            );
+        }
+        assert_eq!(format_snapshot(&disks).await, before, "{invalid_state} intent cannot alter valid formats");
+    }
+}
+
+#[tokio::test]
+async fn bootstrap_intent_retirement_prevents_future_rebootstrap() {
+    let (_directory, mut disks) = local_disks(4).await;
+    let first = connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut disks, 1, 4, None)
+        .await
+        .expect("format fresh deployment");
+    assert_eq!(first.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::Fresh);
+    retire_format_bootstrap_authority(&disks)
+        .await
+        .expect("retire authority after committed pool metadata");
+    retire_format_bootstrap_authority(&disks)
+        .await
+        .expect("authority retirement is idempotent");
+    let before = format_snapshot(&disks).await;
+
+    let mut reopened = reopen_disks(&disks).await;
+    let loaded = connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut reopened, 1, 4, None)
+        .await
+        .expect("load a deployment after authority retirement");
+
+    assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
+    assert_eq!(format_snapshot(&reopened).await, before);
+}
+
+#[tokio::test]
+async fn bootstrap_intent_observer_cannot_resume_formatter_authority() {
+    let (_directory, mut disks) = local_disks(4).await;
+    let plan = bootstrap_plan();
+    write_format_bootstrap_intents(&disks, &plan)
+        .await
+        .expect("persist complete format intent");
+    write_format_file_all(&disks, &plan).await.expect("publish expected formats");
+    let before = format_snapshot(&disks).await;
+
+    let loaded = connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), false, &mut disks, 1, 4, None)
+        .await
+        .expect("observer may read the existing format");
+
+    assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
+    assert_eq!(format_snapshot(&disks).await, before, "an observer cannot republish storage formats");
+}
+
+#[tokio::test]
+async fn bootstrap_intent_unavailable_disk_cannot_restore_authority() {
+    let (_directory, mut disks) = local_disks(4).await;
+    let plan = bootstrap_plan();
+    write_format_bootstrap_intents(&disks, &plan)
+        .await
+        .expect("persist complete format intent");
+    write_format_file_all(&disks, &plan).await.expect("publish expected formats");
+    let physical_disks = disks.clone();
+    let before = format_snapshot(&physical_disks).await;
+    disks[3] = None;
+
+    let result =
+        connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut disks, 1, 4, None).await;
+
+    if let Ok(loaded) = result {
+        assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
+    }
+    assert_eq!(
+        format_snapshot(&physical_disks).await,
+        before,
+        "a missing configured disk cannot authorize format writes"
+    );
+}
+
+#[tokio::test]
+async fn bootstrap_intent_deployment_mismatch_cannot_publish_missing_format() {
+    let (_directory, mut disks) = local_disks(4).await;
+    let plan = bootstrap_plan();
+    write_format_bootstrap_intents(&disks, &plan)
+        .await
+        .expect("persist complete format intent");
+    write_format_file_all(&disks[..3], &plan[..3])
+        .await
+        .expect("publish three formats from the original deployment");
+    let physical_disks = disks.clone();
+    let before = format_snapshot(&physical_disks).await;
+
+    let result = connect_load_init_formats_with_instance_ctx(
+        &Arc::new(InstanceContext::new()),
+        true,
+        &mut disks,
+        1,
+        4,
+        Some(Uuid::new_v4()),
+    )
+    .await;
+
+    if let Ok(loaded) = result {
+        assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
+    }
+    assert_eq!(
+        format_snapshot(&physical_disks).await,
+        before,
+        "an intent belonging to another deployment cannot publish the fourth format"
+    );
+}
+
+#[tokio::test]
+async fn bootstrap_intent_conflicting_existing_format_cannot_be_overwritten() {
+    let (_directory, mut disks) = local_disks(4).await;
+    let plan = bootstrap_plan();
+    write_format_bootstrap_intents(&disks, &plan)
+        .await
+        .expect("persist complete format intent");
+    write_format_file_all(&disks[..3], &plan[..3])
+        .await
+        .expect("publish first three formats");
+    let foreign = bootstrap_plan();
+    disks[3]
+        .as_ref()
+        .expect("fourth disk")
+        .write_all(
+            RUSTFS_META_BUCKET,
+            FORMAT_CONFIG_FILE,
+            foreign[3]
+                .as_ref()
+                .expect("foreign fourth format")
+                .to_json()
+                .expect("encode foreign physical format")
+                .into_bytes()
+                .into(),
+        )
+        .await
+        .expect("inject another deployment's physical format on the fourth disk");
+    let physical_disks = disks.clone();
+    let before = format_snapshot(&physical_disks).await;
+
+    let result =
+        connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut disks, 1, 4, None).await;
+
+    if let Ok(loaded) = result {
+        assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
+    }
+    assert_eq!(
+        format_snapshot(&physical_disks).await,
+        before,
+        "a complete intent cannot overwrite a conflicting persisted format"
+    );
+}
+
+#[tokio::test]
+async fn bootstrap_intent_retirement_consumes_only_one_slot() {
+    let (_directory, mut disks) = local_disks(4).await;
+    let plan = bootstrap_plan();
+    write_format_bootstrap_intents(&disks, &plan)
+        .await
+        .expect("persist complete format intent");
+    write_format_file_all(&disks, &plan)
+        .await
+        .expect("publish every planned format");
+    retire_format_bootstrap_authority(&disks)
+        .await
+        .expect("retire creator authority");
+    retire_format_bootstrap_authority(&disks)
+        .await
+        .expect("repeat retirement after interrupted identity publication");
+
+    let mut absent = 0;
+    for disk in disks.iter().flatten() {
+        match disk.read_file(RUSTFS_META_BUCKET, FORMAT_BOOTSTRAP_FILE).await {
+            Err(DiskError::FileNotFound) => absent += 1,
+            Ok(_) => {}
+            Err(error) => panic!("read intent after retirement: {error}"),
+        }
+    }
+    assert_eq!(absent, 1, "one consumed slot is sufficient and avoids writes to sibling disks");
+    let loaded = connect_load_init_formats_with_instance_ctx(&Arc::new(InstanceContext::new()), true, &mut disks, 1, 4, None)
+        .await
+        .expect("load formats after creator authority consumption");
+    assert_eq!(loaded.pool_meta_bootstrap_authority, PoolMetaBootstrapAuthority::None);
+}
+
+#[cfg(unix)]
+#[tokio::test]
+async fn bootstrap_intent_retirement_preserves_completion_symlink_target() {
+    let (directory, disks) = local_disks(4).await;
+    let plan = bootstrap_plan();
+    write_format_bootstrap_intents(&disks, &plan)
+        .await
+        .expect("persist creator intent");
+    let sentinel = directory.path().join("unrelated");
+    tokio::fs::write(&sentinel, b"preserve").await.expect("write sentinel");
+    let done = disks[0]
+        .as_ref()
+        .expect("fixture disk")
+        .path()
+        .join(RUSTFS_META_BUCKET)
+        .join("format.json.bootstrap.done");
+    std::os::unix::fs::symlink(&sentinel, &done).expect("install completion symlink");
+    retire_format_bootstrap_authority(&disks[..1])
+        .await
+        .expect("replace completion entry without following it");
+    assert_eq!(tokio::fs::read(&sentinel).await.expect("read sentinel"), b"preserve");
+    assert!(tokio::fs::symlink_metadata(done).await.expect("completion marker").is_file());
+}
+
+#[cfg(all(unix, feature = "test-util"))]
+#[tokio::test]
+async fn bootstrap_intent_retirement_requires_successful_directory_sync() {
+    let (_directory, disks) = local_disks(4).await;
+    let disk = disks[0].as_ref().expect("fixture disk");
+    crate::disk::os::set_local_dir_fsync_failure(disk, RUSTFS_META_BUCKET, "", std::io::ErrorKind::Other)
+        .expect("inject absent-intent directory-sync failure");
+    retire_format_bootstrap_authority(&disks[..1])
+        .await
+        .expect_err("visible absence requires a successful sync");
+    retire_format_bootstrap_authority(&disks[..1])
+        .await
+        .expect("retirement sync retries safely");
+
+    let plan = bootstrap_plan();
+    write_format_bootstrap_intents(&disks, &plan)
+        .await
+        .expect("persist creator intent");
+    crate::disk::os::set_local_dir_fsync_failure(disk, RUSTFS_META_BUCKET, "", std::io::ErrorKind::Other)
+        .expect("inject post-rename directory-sync failure");
+    retire_format_bootstrap_authority(&disks[..1])
+        .await
+        .expect_err("rename visibility cannot replace durability");
+    assert!(matches!(
+        disk.read_file(RUSTFS_META_BUCKET, FORMAT_BOOTSTRAP_FILE).await,
+        Err(DiskError::FileNotFound)
+    ));
+    crate::disk::os::set_local_dir_fsync_failure(disk, RUSTFS_META_BUCKET, "", std::io::ErrorKind::Other)
+        .expect("inject retirement retry sync failure");
+    retire_format_bootstrap_authority(&disks[..1])
+        .await
+        .expect_err("retry must repeat the durability barrier");
+    retire_format_bootstrap_authority(&disks[..1])
+        .await
+        .expect("retry consumes authority durably");
+}

crates/ecstore/src/store/mod.rs

@@ -435,6 +435,7 @@ pub use heal::{HealObjectAbsenceProof, HealObjectStorageResult};
 pub(crate) use heal::{bucket_heal_scope, bucket_heal_scope_for_object};
 mod heal_walk;
 pub use heal_walk::HealWalkVersion;
+mod format_bootstrap;
 mod init;
 pub(crate) mod init_format;
 pub(crate) mod list_objects;

crates/ecstore/src/store/peer.rs

@@ -46,6 +46,15 @@ impl BootstrapLocalTarget {
         Arc::ptr_eq(&self.ctx, &store.ctx)
     }
 
+    /// Atomically publish a format control file on this bootstrap instance.
+    ///
+    /// # Errors
+    /// Returns a disk error for an unsupported path, unavailable local endpoint,
+    /// publication failure, or failed durability barrier.
+    pub async fn write_local_format_control(&self, disk_ref: &str, path: &str, data: bytes::Bytes) -> DiskResult<()> {
+        write_local_format_control_with_ctx(&self.ctx, disk_ref, path, data).await
+    }
+
     pub async fn rename_local_data(
         &self,
         disk_ref: &str,
@@ -85,6 +94,15 @@ fn validate_bootstrap_volume(volume: &str) -> DiskResult<()> {
 }
 
 impl ECStore {
+    /// Atomically publish a format control file on this storage instance.
+    ///
+    /// # Errors
+    /// Returns a disk error for an unsupported path, unavailable local endpoint,
+    /// publication failure, or failed durability barrier.
+    pub async fn write_local_format_control(&self, disk_ref: &str, path: &str, data: bytes::Bytes) -> DiskResult<()> {
+        write_local_format_control_with_ctx(&self.ctx, disk_ref, path, data).await
+    }
+
     /// Resolve a quota lease against this request's captured storage instance.
     pub async fn quota_mutation_fence_disk(&self, disk_ref: &str) -> DiskResult<DiskStore> {
         let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
@@ -307,6 +325,18 @@ impl ECStore {
 }
 
 // The optional ID is a cold lookup to cache only after final admission.
+async fn write_local_format_control_with_ctx(
+    ctx: &Arc<InstanceContext>,
+    disk_ref: &str,
+    path: &str,
+    data: bytes::Bytes,
+) -> DiskResult<()> {
+    crate::disk::validate_format_control_path(path)?;
+    let (disk, id) = local_disk_candidate(ctx, disk_ref).await?;
+    let _owner = admit_local_disk(ctx, &disk, id, false).await?;
+    disk.write_format_control(path, data).await
+}
+
 async fn local_disk_candidate(ctx: &Arc<InstanceContext>, disk_ref: &str) -> DiskResult<(DiskStore, Option<Uuid>)> {
     let map = ctx.local_disk_map();
     let path_disk = map.read().await.get(disk_ref).and_then(Option::as_ref).cloned();

crates/protos/src/generated/proto_gen/node_service.rs

@@ -127,6 +127,15 @@ pub struct WriteAllRequest {
     pub data: ::prost::bytes::Bytes,
 }
 #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct WriteFormatControlRequest {
+    #[prost(string, tag = "1")]
+    pub disk: ::prost::alloc::string::String,
+    #[prost(string, tag = "2")]
+    pub path: ::prost::alloc::string::String,
+    #[prost(bytes = "bytes", tag = "3")]
+    pub data: ::prost::bytes::Bytes,
+}
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
 pub struct WriteAllResponse {
     #[prost(bool, tag = "1")]
     pub success: bool,
@@ -2221,6 +2230,21 @@ pub mod node_service_client {
                 .insert(GrpcMethod::new("node_service.NodeService", "WriteAll"));
             self.inner.unary(req, path, codec).await
         }
+        pub async fn write_format_control(
+            &mut self,
+            request: impl tonic::IntoRequest<super::WriteFormatControlRequest>,
+        ) -> std::result::Result<tonic::Response<super::WriteAllResponse>, tonic::Status> {
+            self.inner
+                .ready()
+                .await
+                .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
+            let codec = tonic_prost::ProstCodec::default();
+            let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/WriteFormatControl");
+            let mut req = request.into_request();
+            req.extensions_mut()
+                .insert(GrpcMethod::new("node_service.NodeService", "WriteFormatControl"));
+            self.inner.unary(req, path, codec).await
+        }
         pub async fn compare_and_update_file(
             &mut self,
             request: impl tonic::IntoRequest<super::CompareAndUpdateFileRequest>,
@@ -3801,6 +3825,10 @@ pub mod node_service_server {
             &self,
             request: tonic::Request<super::WriteAllRequest>,
         ) -> std::result::Result<tonic::Response<super::WriteAllResponse>, tonic::Status>;
+        async fn write_format_control(
+            &self,
+            request: tonic::Request<super::WriteFormatControlRequest>,
+        ) -> std::result::Result<tonic::Response<super::WriteAllResponse>, tonic::Status>;
         async fn compare_and_update_file(
             &self,
             request: tonic::Request<super::CompareAndUpdateFileRequest>,
@@ -4548,6 +4576,34 @@ pub mod node_service_server {
                     };
                     Box::pin(fut)
                 }
+                "/node_service.NodeService/WriteFormatControl" => {
+                    #[allow(non_camel_case_types)]
+                    struct WriteFormatControlSvc<T: NodeService>(pub Arc<T>);
+                    impl<T: NodeService> tonic::server::UnaryService<super::WriteFormatControlRequest> for WriteFormatControlSvc<T> {
+                        type Response = super::WriteAllResponse;
+                        type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
+                        fn call(&mut self, request: tonic::Request<super::WriteFormatControlRequest>) -> Self::Future {
+                            let inner = Arc::clone(&self.0);
+                            let fut = async move { <T as NodeService>::write_format_control(&inner, request).await };
+                            Box::pin(fut)
+                        }
+                    }
+                    let accept_compression_encodings = self.accept_compression_encodings;
+                    let send_compression_encodings = self.send_compression_encodings;
+                    let max_decoding_message_size = self.max_decoding_message_size;
+                    let max_encoding_message_size = self.max_encoding_message_size;
+                    let inner = self.inner.clone();
+                    let fut = async move {
+                        let method = WriteFormatControlSvc(inner);
+                        let codec = tonic_prost::ProstCodec::default();
+                        let mut grpc = tonic::server::Grpc::new(codec)
+                            .apply_compression_config(accept_compression_encodings, send_compression_encodings)
+                            .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
+                        let res = grpc.unary(method, req).await;
+                        Ok(res)
+                    };
+                    Box::pin(fut)
+                }
                 "/node_service.NodeService/CompareAndUpdateFile" => {
                     #[allow(non_camel_case_types)]
                     struct CompareAndUpdateFileSvc<T: NodeService>(pub Arc<T>);

crates/protos/src/lib.rs

@@ -1433,6 +1433,20 @@ pub fn canonical_write_all_request_body(
     Ok(body.finish())
 }
 
+/// Build the signed semantic body for atomic format-control publication.
+///
+/// # Errors
+/// Returns a length conversion error when a field exceeds the canonical wire limit.
+pub fn canonical_write_format_control_request_body(
+    request: &proto_gen::node_service::WriteFormatControlRequest,
+) -> Result<Vec<u8>, std::num::TryFromIntError> {
+    let mut body = CanonicalBodyBuilder::new(b"rustfs-write-format-control-request-v1\0");
+    body.push_str(&request.disk)?;
+    body.push_str(&request.path)?;
+    body.push_bytes(&request.data)?;
+    Ok(body.finish())
+}
+
 pub fn canonical_compare_and_update_file_request_body(
     request: &proto_gen::node_service::CompareAndUpdateFileRequest,
 ) -> Result<Vec<u8>, std::num::TryFromIntError> {
@@ -3781,6 +3795,36 @@ mod tests {
         fields
     }
 
+    #[test]
+    fn format_control_canonical_body_binds_every_field_and_operation() {
+        use proto_gen::node_service::{WriteAllRequest, WriteFormatControlRequest};
+        let request = WriteFormatControlRequest {
+            disk: "http://node-a:9000/data".into(),
+            path: "format.json.bootstrap".into(),
+            data: vec![1, 2, 3].into(),
+        };
+        let canonical = canonical_write_format_control_request_body(&request).expect("canonical format control");
+        for mutate in [
+            |r: &mut WriteFormatControlRequest| r.disk = "http://node-b:9000/data".into(),
+            |r: &mut WriteFormatControlRequest| r.path = "format.json".into(),
+            |r: &mut WriteFormatControlRequest| r.data = vec![1, 2, 4].into(),
+        ] {
+            let mut changed = request.clone();
+            mutate(&mut changed);
+            assert_ne!(
+                canonical_write_format_control_request_body(&changed).expect("changed canonical body"),
+                canonical
+            );
+        }
+        let ordinary = WriteAllRequest {
+            disk: request.disk,
+            volume: ".rustfs.sys".into(),
+            path: request.path,
+            data: request.data,
+        };
+        assert_ne!(canonical_write_all_request_body(&ordinary).expect("ordinary canonical body"), canonical);
+    }
+
     #[test]
     fn request_compat_send_site_manifest_covers_node_proto_bin_fields() {
         let mut manifest_fields = REQUEST_COMPAT_SEND_SITES
@@ -4007,20 +4051,9 @@ mod tests {
 
         let _test_guard = TLS02_TEST_LOCK.lock().await;
         let previous = runtime_sources::outbound_tls_state().await;
-        let cases = [
-            ("DNS", "127.0.0.1:0", "localhost", "localhost", true),
-            ("IPv4", "127.0.0.1:0", "127.0.0.1", "127.0.0.1", true),
-            ("IPv6", "[::1]:0", "[::1]", "::1", true),
-            ("IPv6 name mismatch", "[::1]:0", "[::1]", "localhost", false),
-        ];
-        let restore_offset = u64::try_from(cases.len() + 1).expect("bounded TLS case count");
-        let restore_generation = previous
-            .generation
-            .0
-            .checked_add(restore_offset)
-            .expect("test generation capacity");
+        let restore_generation = previous.generation.0.checked_add(3).expect("test generation capacity");
         let mut observations = Vec::new();
-        for (index, (case, listen_addr, dial_host, certificate_name, expected)) in cases.into_iter().enumerate() {
+        for (offset, certificate_name) in [(1, "::1"), (2, "localhost")] {
             let certificate =
                 rcgen::generate_simple_self_signed(vec![certificate_name.to_owned()]).expect("ephemeral TLS test certificate");
             let certificate_pem = certificate.cert.pem();
@@ -4028,11 +4061,10 @@ mod tests {
                 &certificate_pem,
                 certificate.signing_key.serialize_pem(),
             ));
-            let listener = tokio::net::TcpListener::bind(listen_addr)
+            let listener = tokio::net::TcpListener::bind("[::1]:0")
                 .await
-                .expect("loopback TLS listener");
-            let port = listener.local_addr().expect("TLS test server address").port();
-            let addr = format!("https://{dial_host}:{port}/data");
+                .expect("IPv6 loopback TLS listener");
+            let addr = format!("https://{}", listener.local_addr().expect("IPv6 test server address"));
             let accepted = Arc::new(AtomicUsize::new(0));
             let counter = Arc::clone(&accepted);
             let incoming = TcpListenerStream::new(listener).map(move |stream| {
@@ -4054,30 +4086,20 @@ mod tests {
                 root_ca_pem: certificate_pem.into_bytes(),
                 mtls_identity: None,
             };
-            let offset = u64::try_from(index + 1).expect("bounded generation offset");
             publish_global_outbound_tls_state(TlsGeneration(previous.generation.0 + offset), &material).await;
             let mut results = Vec::new();
             for key in [addr.clone(), bulk_cache_key(&addr, 0)] {
-                let (connected, reached_rpc) = match build_channel(&addr, &key).await {
-                    Ok(channel) => {
-                        let mut client = NodeServiceClient::new(channel);
-                        let response = tokio::time::timeout(
-                            Duration::from_secs(3),
-                            client.ping(proto_gen::node_service::PingRequest::default()),
-                        )
-                        .await;
-                        // The empty router answers only after TLS and HTTP/2 reach the server.
-                        let reached_rpc = matches!(response, Ok(Err(status)) if status.code() == tonic::Code::Unimplemented);
-                        (Ok(()), reached_rpc)
-                    }
-                    Err(error) => (Err(error.to_string()), false),
-                };
-                results.push((connected, reached_rpc));
+                results.push(
+                    build_channel(&addr, &key)
+                        .await
+                        .map(|_| ())
+                        .map_err(|error| error.to_string()),
+                );
                 evict_cached_channel(&key, ConnectionEvictionLogLevel::Debug).await;
             }
             server.abort();
             let _ = server.await;
-            observations.push((case, expected, results, accepted.load(Ordering::SeqCst)));
+            observations.push((results, accepted.load(Ordering::SeqCst)));
         }
         let restore = OutboundTlsMaterial {
             root_ca_pem: previous.root_ca_pem.unwrap_or_default(),
@@ -4085,31 +4107,19 @@ mod tests {
         };
         publish_global_outbound_tls_state(TlsGeneration(restore_generation), &restore).await;
 
-        let restored = runtime_sources::outbound_tls_state().await;
-        assert_eq!(restored.root_ca_pem.as_deref().unwrap_or_default(), restore.root_ca_pem.as_slice());
-        assert_eq!(
-            restored
-                .mtls_identity
-                .as_ref()
-                .map(|identity| (&identity.cert_pem, &identity.key_pem)),
-            restore
-                .mtls_identity
-                .as_ref()
-                .map(|identity| (&identity.cert_pem, &identity.key_pem)),
+        assert!(
+            observations[0].0.iter().all(Result::is_ok),
+            "IPv6 control and bulk channels must verify the URI host: {:?}",
+            observations[0].0
+        );
+        assert!(
+            observations[1].0.iter().all(Result::is_err),
+            "a trusted certificate for localhost must be rejected for ::1"
+        );
+        assert!(
+            observations.iter().all(|(_, accepted)| *accepted == 2),
+            "both certificate cases must exercise actual control and bulk TLS connections"
         );
-        assert!(restored.generation.0 > previous.generation.0);
-
-        for (case, expected, results, accepted) in observations {
-            for (connected, reached_rpc) in results {
-                assert_eq!(
-                    connected.is_ok(),
-                    expected,
-                    "{case} must verify the URI certificate identity: {connected:?}"
-                );
-                assert_eq!(reached_rpc, expected, "{case} must reach the RPC service only after successful TLS");
-            }
-            assert_eq!(accepted, 2, "{case} must exercise actual control and bulk connections");
-        }
     }
 
     #[tokio::test]

crates/protos/src/node.proto

@@ -116,6 +116,12 @@ message WriteAllRequest {
   bytes data = 4;
 }
 
+message WriteFormatControlRequest {
+  string disk = 1;
+  string path = 2;
+  bytes data = 3;
+}
+
 message WriteAllResponse {
   bool success = 1;
   optional Error error = 2;
@@ -1308,6 +1314,7 @@ service NodeService {
 
   rpc ReadAll(ReadAllRequest) returns (ReadAllResponse) {}; // auth-policy: read-only
   rpc WriteAll(WriteAllRequest) returns (WriteAllResponse) {}; // auth-policy: body-bound
+  rpc WriteFormatControl(WriteFormatControlRequest) returns (WriteAllResponse) {}; // auth-policy: body-bound
   rpc CompareAndUpdateFile(CompareAndUpdateFileRequest) returns (CompareAndUpdateFileResponse) {}; // auth-policy: body-bound
   rpc Delete(DeleteRequest) returns (DeleteResponse) {}; // auth-policy: body-bound
   rpc DeleteAtIncarnation(DeleteRequest) returns (DeleteResponse) {}; // auth-policy: body-bound

crates/s3-client/src/transition_api.rs

@@ -60,9 +60,7 @@ use rustfs_utils::{
         AMZ_CHECKSUM_SHA256,
     },
     net::get_endpoint_url,
-    retry::{
-        DEFAULT_RETRY_CAP, DEFAULT_RETRY_UNIT, MAX_JITTER, MAX_RETRY, RetryTimer, is_http_status_retryable, is_s3code_retryable,
-    },
+    retry::{DEFAULT_RETRY_CAP, DEFAULT_RETRY_UNIT, MAX_JITTER, MAX_RETRY, RetryTimer},
 };
 use rustls_pki_types::PrivateKeyDer;
 use rustls_pki_types::pem::PemObject;
@@ -121,33 +119,6 @@ fn transition_transport_error(err: hyper_util::client::legacy::Error) -> std::io
     std::io::Error::other(err)
 }
 
-fn is_retryable_transport_error(error: &std::io::Error) -> bool {
-    // Hyper wraps connection and body errors; the outer io::Error is often Other.
-    let mut current: Option<&(dyn StdError + 'static)> = Some(error);
-    while let Some(error) = current {
-        if error.downcast_ref::<std::io::Error>().is_some_and(|error| {
-            matches!(
-                error.kind(),
-                std::io::ErrorKind::Interrupted
-                    | std::io::ErrorKind::WouldBlock
-                    | std::io::ErrorKind::TimedOut
-                    | std::io::ErrorKind::ConnectionAborted
-                    | std::io::ErrorKind::ConnectionRefused
-                    | std::io::ErrorKind::ConnectionReset
-                    | std::io::ErrorKind::NotConnected
-                    | std::io::ErrorKind::UnexpectedEof
-            )
-        }) || error
-            .downcast_ref::<hyper::Error>()
-            .is_some_and(|error| error.is_incomplete_message() || error.is_closed())
-        {
-            return true;
-        }
-        current = error.source();
-    }
-    false
-}
-
 async fn next_response_body_data<B>(
     mut body: Pin<&mut B>,
     idle_timeout: Option<StdDuration>,
@@ -756,18 +727,6 @@ impl TransitionClient {
         Ok(resp)
     }
 
-    /// Executes a remote request, retrying transient failures up to `max_retries` total attempts.
-    ///
-    /// Only GET, HEAD, and PUT of a fixed multipart upload part with a buffered body
-    /// are replayed. Object readers and other mutations get one attempt: replaying
-    /// ordinary PUT or DELETE can create additional versions or delete markers.
-    /// Request construction failures and errors reading a successful response body
-    /// are not retried. Dropping this future stops further attempts and any pending
-    /// backoff; a failed or canceled request may still have reached the remote server.
-    ///
-    /// # Errors
-    /// Returns a request construction error, a permanent remote error, or the last
-    /// transient error when the attempt budget is exhausted.
     pub async fn execute_method(
         &self,
         method: http::Method,
@@ -779,27 +738,24 @@ impl TransitionClient {
             return Err(std::io::Error::other(s));
         }
 
-        let retryable = matches!(&metadata.content_body, ReaderImpl::Body(_))
-            && (method == http::Method::GET
-                || method == http::Method::HEAD
-                || (method == http::Method::PUT
-                    && metadata.query_values.get("uploadId").is_some_and(|value| !value.is_empty())
-                    && metadata.query_values.get("partNumber").is_some_and(|value| !value.is_empty())));
-        let req_retry = if retryable { self.max_retries.max(1) } else { 1 };
-        let mut last_error = None;
+        let retryable: bool;
+        //let mut body_seeker: BufferReader;
+        let mut req_retry = self.max_retries;
+        let mut resp: Response<Incoming>;
+
+        //if metadata.content_body != nil {
+        //body_seeker = BufferReader::new(metadata.content_body.read_all().await?);
+        retryable = true;
+        if !retryable {
+            req_retry = 1;
+        }
+        //}
 
         let mut retry_timer = RetryTimer::new(req_retry, DEFAULT_RETRY_UNIT, DEFAULT_RETRY_CAP, MAX_JITTER, self.random);
         while retry_timer.next().await.is_some() {
             let req = self.new_request(&method, metadata).await?;
 
-            let resp = match self.doit(req).await {
-                Ok(resp) => resp,
-                Err(error) if retryable && is_retryable_transport_error(&error) => {
-                    last_error = Some(error);
-                    continue;
-                }
-                Err(error) => return Err(error),
-            };
+            resp = self.doit(req).await?;
 
             for http_status in SUCCESS_STATUS {
                 if http_status == resp.status() {
@@ -810,17 +766,11 @@ impl TransitionClient {
             let resp_status = resp.status();
             let h = resp.headers().clone();
 
-            let body_vec = match self.collect_response_body(resp.into_body(), MAX_S3_ERROR_RESPONSE_SIZE).await {
-                Ok(body) => body,
-                Err(error) if retryable && is_retryable_transport_error(&error) => {
-                    last_error = Some(error);
-                    continue;
-                }
-                Err(error) => return Err(error),
-            };
+            let body_vec = self
+                .collect_response_body(resp.into_body(), MAX_S3_ERROR_RESPONSE_SIZE)
+                .await?;
             let parsed_error =
                 http_resp_to_error_response(resp_status, &h, body_vec, &metadata.bucket_name, &metadata.object_name);
-            let retryable_response = is_http_status_retryable(&resp_status) || is_s3code_retryable(parsed_error.code.as_str());
             let routing_region = parsed_error.region;
             let code = match parsed_error.code {
                 code if code.is_custom() => RemoteErrorCode::RESPONSE_INTERRUPTED,
@@ -835,11 +785,6 @@ impl TransitionClient {
                 ..Default::default()
             };
 
-            if retryable && retryable_response {
-                last_error = Some(std::io::Error::other(err_response));
-                continue;
-            }
-
             if self.region == "" {
                 return match err_response.code {
                     RemoteErrorCode::AUTHORIZATION_HEADER_MALFORMED | RemoteErrorCode::INVALID_ARGUMENT /*RemoteErrorCode::INVALID_REGION*/ => {
@@ -874,7 +819,7 @@ impl TransitionClient {
             return Err(std::io::Error::other(err_response));
         }
 
-        Err(last_error.unwrap_or_else(|| std::io::Error::other("remote tier request did not produce a response")))
+        Err(std::io::Error::other("remote tier request did not produce a response"))
     }
 
     pub async fn collect_response_body<B>(&self, body: B, limit: usize) -> Result<Vec<u8>, std::io::Error>

crates/s3-client/tests/transition_retries.rs

@@ -1,320 +0,0 @@
-// Copyright 2026 RustFS Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use bytes::Bytes;
-use http::{Method, StatusCode};
-use rustfs_s3_client::{
-    api_error_response::to_error_response,
-    credentials::{Credentials, SignatureType, Static, Value},
-    remote_error_code::RemoteErrorCode,
-    transition_api::{
-        BucketLookupType, MAX_S3_ERROR_RESPONSE_SIZE, ObjectReader, Options, ReaderImpl, RequestMetadata, TransitionClient,
-        TransitionClientTimeouts,
-    },
-};
-use std::time::Duration;
-use tokio::{
-    io::{AsyncReadExt, AsyncWriteExt},
-    net::TcpListener,
-};
-
-#[derive(Clone, Copy)]
-enum Reply {
-    Http(StatusCode, &'static str),
-    Disconnect,
-    Truncated,
-    StallHeaders,
-    MalformedHttp,
-    Oversized,
-}
-
-fn metadata(body: ReaderImpl, content_length: i64) -> RequestMetadata {
-    RequestMetadata {
-        bucket_name: "bucket".into(),
-        object_name: "object".into(),
-        bucket_location: "us-east-1".into(),
-        content_body: body,
-        content_length,
-        pre_sign_url: false,
-        query_values: Default::default(),
-        custom_header: Default::default(),
-        extra_pre_sign_header: None,
-        expires: 0,
-        content_md5_base64: String::new(),
-        content_sha256_hex: String::new(),
-        stream_sha256: false,
-        trailer: Default::default(),
-    }
-}
-
-fn empty_metadata() -> RequestMetadata {
-    metadata(ReaderImpl::Body(Bytes::new()), 0)
-}
-
-async fn run_case(
-    method: Method,
-    metadata: &mut RequestMetadata,
-    attempts: i64,
-    replies: Vec<Reply>,
-) -> (Result<StatusCode, std::io::Error>, Vec<Vec<u8>>) {
-    let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind retry fixture");
-    let endpoint = listener.local_addr().expect("fixture address").to_string();
-    let (stop, mut stopped) = tokio::sync::oneshot::channel();
-    let server = tokio::spawn(async move {
-        let mut requests = Vec::new();
-        let mut stalled_sockets = Vec::new();
-        loop {
-            let (mut socket, _) = tokio::select! {
-                _ = &mut stopped => break,
-                accepted = listener.accept() => accepted.expect("accept retry request"),
-            };
-            let mut request = Vec::new();
-            let header_end = loop {
-                let mut buffer = [0; 1024];
-                let size = socket.read(&mut buffer).await.expect("read request headers");
-                assert_ne!(size, 0, "client closed before sending headers");
-                request.extend_from_slice(&buffer[..size]);
-                if let Some(end) = request.windows(4).position(|part| part == b"\r\n\r\n") {
-                    break end + 4;
-                }
-            };
-            let headers = String::from_utf8_lossy(&request[..header_end]).to_ascii_lowercase();
-            assert!(headers.contains("authorization: aws4-hmac-sha256"), "each attempt must be signed");
-            let length = headers
-                .lines()
-                .find_map(|line| line.strip_prefix("content-length: "))
-                .map(|value| value.parse::<usize>().expect("numeric content length"))
-                .unwrap_or(0);
-            while request.len() < header_end + length {
-                let mut buffer = [0; 1024];
-                let size = socket.read(&mut buffer).await.expect("read request body");
-                assert_ne!(size, 0, "client closed before sending the complete body");
-                request.extend_from_slice(&buffer[..size]);
-            }
-            let reply = replies[requests.len().min(replies.len() - 1)];
-            requests.push(request);
-            let (status, body, extra_length) = match reply {
-                Reply::Disconnect => continue,
-                Reply::StallHeaders => {
-                    stalled_sockets.push(socket);
-                    continue;
-                }
-                Reply::MalformedHttp => {
-                    socket
-                        .write_all(b"invalid HTTP\r\n\r\n")
-                        .await
-                        .expect("write malformed response");
-                    continue;
-                }
-                Reply::Truncated => (StatusCode::SERVICE_UNAVAILABLE, "<Error>".to_string(), 100),
-                Reply::Oversized => (StatusCode::SERVICE_UNAVAILABLE, "x".repeat(MAX_S3_ERROR_RESPONSE_SIZE + 1), 0),
-                Reply::Http(status, body) => (status, body.to_string(), 0),
-            };
-            let is_head = requests.last().expect("recorded request").starts_with(b"HEAD ");
-            let response = format!(
-                "HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\nx-amz-request-id: fixture\r\n\r\n{}",
-                body.len() + extra_length,
-                if is_head { "" } else { &body },
-            );
-            socket.write_all(response.as_bytes()).await.expect("write fixture response");
-        }
-        requests
-    });
-    let client = TransitionClient::new_with_timeouts(
-        &endpoint,
-        Options {
-            creds: Credentials::new(Static(Value {
-                access_key_id: "access-key".into(),
-                secret_access_key: "secret-key".into(),
-                signer_type: SignatureType::SignatureV4,
-                ..Default::default()
-            })),
-            region: "us-east-1".into(),
-            bucket_lookup: BucketLookupType::BucketLookupPath,
-            max_retries: attempts,
-            ..Default::default()
-        },
-        "s3",
-        TransitionClientTimeouts::new(Duration::from_secs(1), Duration::from_secs(1), Duration::from_secs(1)),
-    )
-    .await
-    .expect("build real transition client");
-    let result = tokio::time::timeout(Duration::from_secs(5), client.execute_method(method, metadata)).await;
-    let _ = stop.send(());
-    let requests = tokio::time::timeout(Duration::from_secs(2), server)
-        .await
-        .expect("fixture must stop")
-        .expect("fixture must join");
-    (result.expect("retry loop must terminate").map(|response| response.status()), requests)
-}
-
-#[tokio::test]
-async fn retries_transient_http_statuses_then_succeeds() {
-    for method in [Method::GET, Method::HEAD] {
-        for status in [
-            StatusCode::SERVICE_UNAVAILABLE,
-            StatusCode::REQUEST_TIMEOUT,
-            StatusCode::TOO_MANY_REQUESTS,
-            StatusCode::INTERNAL_SERVER_ERROR,
-            StatusCode::BAD_GATEWAY,
-            StatusCode::GATEWAY_TIMEOUT,
-        ] {
-            let (result, requests) = run_case(
-                method.clone(),
-                &mut empty_metadata(),
-                2,
-                vec![Reply::Http(status, ""), Reply::Http(StatusCode::OK, "")],
-            )
-            .await;
-            assert_eq!(result.expect("transient status should recover"), StatusCode::OK);
-            assert_eq!(requests.len(), 2, "{method} {status} must issue a second request");
-        }
-    }
-}
-
-#[tokio::test]
-async fn retries_s3_error_codes_even_with_a_non_retryable_http_status() {
-    for body in [
-        "<Error><Code>SlowDown</Code></Error>",
-        "<Error><Code>Throttling</Code></Error>",
-        "<Error><Code>ThrottlingException</Code></Error>",
-        "<Error><Code>RequestTimeout</Code></Error>",
-    ] {
-        let (result, requests) = run_case(
-            Method::GET,
-            &mut empty_metadata(),
-            2,
-            vec![Reply::Http(StatusCode::BAD_REQUEST, body), Reply::Http(StatusCode::OK, "")],
-        )
-        .await;
-        assert_eq!(result.expect("transient S3 error should recover"), StatusCode::OK);
-        assert_eq!(requests.len(), 2, "{body} must trigger a retry");
-    }
-}
-
-#[tokio::test]
-async fn retries_disconnected_transport_and_interrupted_error_body() {
-    for first_reply in [Reply::Disconnect, Reply::Truncated, Reply::StallHeaders] {
-        let (result, requests) =
-            run_case(Method::GET, &mut empty_metadata(), 2, vec![first_reply, Reply::Http(StatusCode::OK, "")]).await;
-        assert_eq!(result.expect("interrupted request should recover"), StatusCode::OK);
-        assert_eq!(requests.len(), 2);
-    }
-}
-
-#[tokio::test]
-async fn retry_exhaustion_preserves_the_transport_timeout() {
-    let (result, requests) = run_case(Method::GET, &mut empty_metadata(), 2, vec![Reply::StallHeaders]).await;
-    assert_eq!(result.expect_err("both attempts timed out").kind(), std::io::ErrorKind::TimedOut);
-    assert_eq!(requests.len(), 2);
-}
-
-#[tokio::test]
-async fn malformed_and_oversized_responses_are_not_retried() {
-    for reply in [Reply::MalformedHttp, Reply::Oversized] {
-        let (result, requests) = run_case(Method::GET, &mut empty_metadata(), 3, vec![reply]).await;
-        let error = result.expect_err("invalid responses must fail");
-        if matches!(reply, Reply::Oversized) {
-            assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
-        }
-        assert_eq!(requests.len(), 1, "permanent protocol errors must not be replayed");
-    }
-}
-
-#[tokio::test]
-async fn invalid_request_headers_fail_before_dispatch() {
-    let mut metadata = empty_metadata();
-    metadata.content_md5_base64 = "invalid\nheader".into();
-    let (result, requests) = run_case(Method::GET, &mut metadata, 3, vec![Reply::Http(StatusCode::OK, "")]).await;
-    assert_eq!(result.expect_err("invalid header must fail").kind(), std::io::ErrorKind::InvalidInput);
-    assert!(requests.is_empty(), "request construction failures must not reach the server");
-}
-
-#[tokio::test]
-async fn retry_exhaustion_returns_the_last_s3_error() {
-    let (result, requests) = run_case(
-        Method::GET,
-        &mut empty_metadata(),
-        3,
-        vec![
-            Reply::Http(StatusCode::BAD_GATEWAY, ""),
-            Reply::Http(StatusCode::SERVICE_UNAVAILABLE, "<Error><Code>SlowDown</Code></Error>"),
-        ],
-    )
-    .await;
-    let error = to_error_response(&result.expect_err("all attempts failed"));
-    assert_eq!(requests.len(), 3, "max_retries counts total attempts");
-    assert_eq!(error.status_code, StatusCode::SERVICE_UNAVAILABLE);
-    assert_eq!(error.code, RemoteErrorCode::from_wire("SlowDown"));
-}
-
-#[tokio::test]
-async fn permanent_errors_and_single_attempts_are_not_retried() {
-    for (status, body, attempts) in [
-        (StatusCode::FORBIDDEN, "<Error><Code>AccessDenied</Code></Error>", 3),
-        (StatusCode::NOT_FOUND, "<Error><Code>NoSuchKey</Code></Error>", 3),
-        (StatusCode::BAD_REQUEST, "<Error><Code>InvalidArgument</Code></Error>", 3),
-        (StatusCode::SERVICE_UNAVAILABLE, "<Error><Code>SlowDown</Code></Error>", 1),
-        (StatusCode::OK, "", 3),
-    ] {
-        let (result, requests) = run_case(Method::GET, &mut empty_metadata(), attempts, vec![Reply::Http(status, body)]).await;
-        assert_eq!(result.is_ok(), status.is_success());
-        assert_eq!(requests.len(), 1, "{status} with {attempts} attempts");
-    }
-}
-
-#[tokio::test]
-async fn buffered_upload_parts_replay_identical_body_bytes() {
-    let mut metadata = metadata(ReaderImpl::Body(Bytes::from_static(b"part bytes")), 10);
-    metadata.query_values.insert("uploadId".into(), "upload-1".into());
-    metadata.query_values.insert("partNumber".into(), "1".into());
-    let (result, requests) = run_case(
-        Method::PUT,
-        &mut metadata,
-        2,
-        vec![
-            Reply::Http(StatusCode::SERVICE_UNAVAILABLE, ""),
-            Reply::Http(StatusCode::OK, ""),
-        ],
-    )
-    .await;
-    assert_eq!(result.expect("buffered upload part should recover"), StatusCode::OK);
-    assert_eq!(requests.len(), 2);
-    for request in requests {
-        assert!(request.ends_with(b"\r\n\r\npart bytes"), "every attempt must send the full body");
-    }
-}
-
-#[tokio::test]
-async fn non_replayable_bodies_and_object_mutations_are_not_retried() {
-    for method in [Method::GET, Method::PUT, Method::POST, Method::DELETE] {
-        let mut metadata = if method == Method::GET {
-            metadata(ReaderImpl::ObjectBody(ObjectReader::new(std::io::Cursor::new(b"stream"))), 6)
-        } else {
-            empty_metadata()
-        };
-        let (result, requests) =
-            run_case(method.clone(), &mut metadata, 3, vec![Reply::Http(StatusCode::SERVICE_UNAVAILABLE, "")]).await;
-        assert!(result.is_err(), "{method} must return the first failure");
-        assert_eq!(requests.len(), 1, "{method} must not be replayed");
-    }
-    let mut metadata = metadata(ReaderImpl::ObjectBody(ObjectReader::new(std::io::Cursor::new(b"stream"))), 6);
-    metadata.query_values.insert("uploadId".into(), "upload-1".into());
-    metadata.query_values.insert("partNumber".into(), "1".into());
-    let (result, requests) =
-        run_case(Method::PUT, &mut metadata, 3, vec![Reply::Http(StatusCode::SERVICE_UNAVAILABLE, "")]).await;
-    assert!(result.is_err(), "streaming upload parts must return the first failure");
-    assert_eq!(requests.len(), 1, "a consumed object reader must not be replayed");
-    assert!(requests[0].ends_with(b"\r\n\r\nstream"));
-}

crates/utils/src/retry.rs

@@ -65,10 +65,10 @@ impl Stream for RetryTimer {
 
         let jitter = self.jitter.clamp(NO_JITTER, MAX_JITTER);
         let attempt = self.max_retry - self.rem;
-        let mut sleep = u32::try_from(attempt)
-            .ok()
-            .and_then(|shift| 1_u32.checked_shl(shift))
-            .map_or(self.max_sleep, |factor| self.base_sleep.saturating_mul(factor).min(self.max_sleep));
+        let mut sleep = self.base_sleep * (1 << attempt);
+        if sleep > self.max_sleep {
+            sleep = self.max_sleep;
+        }
         if (jitter - NO_JITTER).abs() > 1e-9 {
             let sleep_ms = sleep.as_millis();
             let reduction = ((sleep_ms as f64) * (self.random as f64 * jitter / 100_f64)).round() as u128;
@@ -255,15 +255,6 @@ mod tests {
         assert_eq!(retry_timer.next().await, None);
     }
 
-    #[tokio::test]
-    async fn retry_timer_caps_backoff_for_large_attempt_budgets() {
-        let retry_timer = RetryTimer::new(40, Duration::from_millis(1), Duration::from_millis(2), NO_JITTER, 0);
-        let retries = timeout(Duration::from_secs(5), retry_timer.collect::<Vec<_>>())
-            .await
-            .expect("a large retry budget should finish without overflowing the backoff");
-        assert_eq!(retries.len(), 40);
-    }
-
     #[test]
     fn is_s3code_in_message_retryable_matches_each_retryable_code() {
         for code in [

docs/operations/pool-metadata-recovery.md

@@ -92,4 +92,18 @@ If a block persists, inspect the first block and subsequent recovery phase, rest
 3. Restore storage formats and the `pool.bin.identity` marker from the same deployment before rejoining it.
 4. Start the node and wait for it to load the verified committed generation and repair its replicas before touching another node.
 
-An initialized identity with every `pool.bin` missing is recovery required, as are existing storage formats with neither identity nor `pool.bin`. Format creation alone is not fresh-cluster proof: only the elected first topology node may create a durable `initialized=false` bootstrap identity with a fresh-bootstrap nonce, and only after every configured disk explicitly responds that it is unformatted. An unreachable peer, a non-elected distributed node, or an existing format is not sufficient proof. All-missing `pool.bin` replicas are accepted only by the same startup that proved the fresh topology and persisted that pending identity; when every `pool.bin` is missing, a later startup must recover even if the pending identity survived. This prevents a wiped or lagging node from rebuilding empty state and overwriting the cluster. Runtime reload, rebalance activation, and rebalance worker admission fail closed on this missing-authority condition; a clean probe alone cannot clear it.
+An initialized identity with every `pool.bin` missing requires metadata recovery. Existing storage formats with neither identity nor `pool.bin` also require recovery unless the pool retains a complete durable format-bootstrap intent. Runtime reload, rebalance activation, and rebalance worker admission preserve this authority check.
+
+Fresh formation first requires every configured disk to explicitly respond that it is unformatted and the legacy-format adoption check to complete. The pool's elected creator durably selects its plan in the first-slot `.rustfs.sys/format.json.bootstrap` anchor before publishing intents to the other slots. Each intent contains the deployment UUID, complete erasure topology and assigned disk UUID. Every configured intent must be durably acknowledged before any format is published. After an interruption with all drives still explicitly unformatted, the creator can complete missing replicas of the validated anchor plan using the original UUIDs. Once any format exists, a complete consistent intent set is required before restoring explicitly unformatted slots and recovering creator authority. A missing anchor with partial replicas, corrupt or conflicting evidence, and unreachable configured drives withhold authority. A pending `pool.bin.identity` alone continues to require verified creator authority.
+
+The elected metadata writer commits `pool.bin`, durably retires one bootstrap intent per pool to `format.json.bootstrap.done`, and then publishes initialized identity on every configured pool. Every node waits for that complete initialized-identity set before proceeding with startup. Since replay requires every intent, one durably consumed slot per pool is sufficient. A crash during retirement leaves committed metadata available for the next startup to finish identity publication. Established startup validates initialized identity through metadata reads, and creators probe the local first-slot intent before opening any replay-read fanout. The `.done` files are internal tombstones and should remain with the deployment.
+
+An established cluster already has a committed metadata predecessor. During expansion or identity repair, the elected writer first retires bootstrap authority, repairs initialized identity on every physical participant, and then saves the updated topology. This preserves the native metadata writer's identity admission requirement. Followers wait for a committed snapshot matching the configured topology and initialized identity on every pool. The startup phase is derived from the observed metadata and identity state. An already complete identity skips retirement for both writers and followers.
+
+Bootstrap format and intent publication uses the authenticated `WriteFormatControl` RPC, restricted to the two format control paths. Each operation performs its staged write, file sync, atomic rename and directory sync in one dedicated per-disk fsync-lane closure with a pinned parent. Unsupported peers return an error before control publication. These critical control writes always execute their durability barriers, including deployments using legacy full-off durability, because resumable creator authority requires persistent file and namespace state.
+
+Format publication and retirement share a namespace owner keyed by physical-root identity, including reopened handles. The native task retains that owner through file and directory sync and cache invalidation after caller cancellation. Admission runs under the same owner: pending plans accept equivalent retries, terminal `.done` records reject bootstrap re-creation, and existing format or intent identities must agree with the incoming deployment, layout and exact drive slot. This prevents an older cancelled or delayed publication from replacing a newer accepted plan. The pinned parent used for evidence reads also supplies the publication directory handle.
+
+Replacement-drive healing retains the existing staged `WriteAll` plus durable `RenameFile` sequence so configured peers can be upgraded independently. Bootstrap publication uses the new atomic operation directly and fails closed on unsupported peers.
+
+Use the same protocol-capable version on all nodes for fresh formation. Finish initialization and intent retirement before rolling back to an older binary, which does not enforce the consumption barrier. Existing deployments retain their current format and identity encodings. Historical interrupted deployments created by older versions, including the format-plus-UUID-temporary-file state in issue #9270, require operator-confirmed recovery because their files contain no durable creator intent. Preserve the drives and follow the metadata-recovery procedure above when any existing data or topology history needs to be retained.

docs/operations/release-pipeline.md

@@ -14,7 +14,7 @@ The builder workflows (`build.yml`, `package.yml`, `docker.yml`, `helm-package.y
 
 ## Freeze and build
 
-Select a reviewed source commit whose Cargo version equals the intended final version. Finish exact-source CI before dispatching the candidate: a successful `push` or `workflow_dispatch` run of `ci.yml` for that commit on `main` or the release branch whose `Quick Checks`, `Test and Lint` and `End-to-End Tests (full merge gate)` all succeeded. A commit that landed through the merge queue has a push run whose validation jobs were skipped as proven; `Quick Checks` and `Test and Lint` must still succeed there, and the evidence recorded is the successful `merge_group` run of that branch's queue whose head commit has the same tree, which must itself have passed all three jobs. `scripts/ci_merge_group_proof.py` defines that match for both the push plan and the candidate. The candidate workflow itself must run from upstream main. Unrelated main commits do not require rebuilding an accepted source.
+Select a reviewed source commit whose Cargo version equals the intended final version. Finish exact-source CI before dispatching the candidate. The candidate workflow itself must run from upstream main. Unrelated main commits do not require rebuilding an accepted source.
 
 ```bash
 gh workflow run release-candidate.yml --ref main \

docs/testing/ci-gates.md

@@ -39,8 +39,6 @@ The aggregate requires the validation lanes already selected by `ci.yml`; this c
 
 Superseded PR attempts are cancelled. A running main validation finishes, with only the newest pending main run retained, so frequent merges cannot continuously cancel the full baseline. Having a `merge_group` trigger does not itself require use of the merge queue.
 
-A push whose tree a successful `merge_group` run of `ci.yml` already validated does not validate it again. A merge-queue landing fast-forwards the branch to the group's commit, so `Select CI scope` looks that run up (`scripts/ci_merge_group_proof.py`, with the job's read-only Actions token) and plans mode `proven`: `Select CI scope`, `Typos` and `Quick Checks` run, every validation job is skipped, and `Test and Lint` passes only if each of them was skipped. The plan and the aggregate's summary name the proving run; a job skipped for any other reason keeps its own reason. A push that did not come through the queue, such as a direct administrator merge, has no such run, and any doubt about the lookup (no token, an API or network error, a rate limit, no matching successful run, a different tree) plans the full validation.
-
 Protocol matrix jobs finish independently when a sibling fails. Each executed protocol test step preserves its log and available JUnit under a separate, attempt-specific artifact; a skipped test step cannot upload a cached report.
 
 The four site-replication state-writer concurrency proofs reserve the available nextest slots in both local and CI profiles. This isolates their durable IO from unrelated test processes while retaining each proof's internal two-writer race, production five-second lock-acquisition limit, assertions, and zero retries.
@@ -72,7 +70,7 @@ The four site-replication state-writer concurrency proofs reserve the available
 | PR touching `paths` in `oidc-keycloak.yml` | `OIDC Keycloak live gate` | `oidc-keycloak.yml` `oidc-keycloak-live` | Report-only | `cargo build --locked -p rustfs --bin rustfs`, then `bash scripts/test/oidc_keycloak_live.sh ./target/debug/rustfs` |
 | PR touching `paths` in `targets-integration.yml` | `PostgreSQL, MySQL, AMQP, and NATS` | `targets-integration.yml` `targets-live` | Report-only | start the containers as in the job, export the `RUSTFS_TEST_*` DSNs, then the job's `cargo test --locked -p rustfs-targets --test <name> -- --ignored --test-threads=1` commands |
 | PR, documentation-only selection | `Quick Checks`, `Typos`, `Test and Lint` | `ci.yml` `quick-checks`, `typos`, `required-checks` | Required directly or via aggregate | Quick Checks commands; `python3 scripts/ci_gate.py --self-test` |
-| `merge_group`; push to `main` or `gateway/integration` that no successful `merge_group` run of the same tree proved | `End-to-End Tests (full merge gate)` | `ci.yml` `e2e-full` | Via aggregate on these events; skipped on a proven push, whose evidence is the proving run | `python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-full -p e2e_test` (CI adds `--binary "$RUSTFS_E2E_STARTUP_CAS_BINARY"` for the downloaded binary and sidecar) |
+| `merge_group`; push to `main` or `gateway/integration` | `End-to-End Tests (full merge gate)` | `ci.yml` `e2e-full` | Via aggregate on these events | `python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-full -p e2e_test` (CI adds `--binary "$RUSTFS_E2E_STARTUP_CAS_BINARY"` for the downloaded binary and sidecar) |
 
 e2e filters live in `.config/nextest.toml`; extend a profile instead of adding a second selector. Before a profile runs, `scripts/check_test_wiring.py` compares its listing to the committed digest in `.config/e2e-<profile>-selection.txt`, so a silent test drop fails closed.
 
@@ -88,7 +86,7 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run. The v
 
 | Workflow (cadence) | Jobs | Verdict and artifacts | Registry monitor | Reproduce |
 |---|---|---|---|---|
-| `ci.yml` (nightly) | full matrix, including the schedule/dispatch-only rio-v2 jobs `build-rustfs-debug-binary-rio-v2` and `e2e-tests-rio-v2` | strict aggregate; the full E2E lane runs on dispatch, merge groups, and main pushes that no merge group proved | yes | dispatch `ci.yml` |
+| `ci.yml` (nightly) | full matrix, including the schedule/dispatch-only rio-v2 jobs `build-rustfs-debug-binary-rio-v2` and `e2e-tests-rio-v2` | strict aggregate; the full E2E lane runs on dispatch, merge groups, and main pushes | yes | dispatch `ci.yml` |
 | `cache-warm.yml` (every six hours, plus main pushes that change `Cargo.lock`, `rust-toolchain.toml` or the setup action) | `warm-ci-dev`, `warm-ci-feat-rio`, `warm-ci-feat-proto`, `warm-ci-uring` | no verdict; sole writer of the `rust-cache` keys that `ci.yml` restores | yes | dispatch `cache-warm.yml` |
 | `build.yml` (nightly) | `build-rustfs` over the six-target platform matrix in `prepare-platform-matrix` (four Linux, macOS aarch64, Windows x86_64) | build/package integrity | yes | dispatch `build.yml` with an exact platform set |
 | `e2e-replication-nightly.yml` (nightly) | `repl-nightly`, `cluster-nightly`, `protocols-nightly` | three independent gates; JUnit, membership listing, server logs | yes | `python3 scripts/e2e_binary.py build --bins`, then `python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-repl-nightly -p e2e_test`; `--features e2e-test-hooks` on both steps with `--profile e2e-nightly`; `--features ftps,webdav,sftp` on both steps with `-j 1 --profile e2e-protocols` |

rustfs/src/admin/service/session_policy.rs

@@ -13,33 +13,14 @@
 // limitations under the License.
 
 use rustfs_iam::sys::SESSION_POLICY_NAME;
-use rustfs_policy::policy::{ID, Policy, Statement};
+use rustfs_policy::policy::Policy;
 use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result, s3_error};
-use serde::Deserialize;
 use serde_json::Value;
 use std::collections::HashMap;
 
-#[derive(Deserialize)]
-#[serde(deny_unknown_fields)]
-struct SessionPolicyInput {
-    #[serde(default, rename = "Id", alias = "ID")]
-    id: ID,
-    #[serde(default, rename = "Version")]
-    version: String,
-    #[serde(default, rename = "Statement")]
-    statements: Vec<Statement>,
-}
-
-fn parse_session_policy(input: &str) -> rustfs_policy::error::Result<Policy> {
-    let SessionPolicyInput { id, version, statements } = serde_json::from_str(input)?;
-    let policy = Policy { id, version, statements };
-    policy.validate()?;
-    Ok(policy)
-}
-
 pub(crate) fn populate_session_policy(claims: &mut HashMap<String, Value>, policy: &str) -> S3Result<()> {
     if !policy.is_empty() {
-        let session_policy = parse_session_policy(policy).map_err(|e| {
+        let session_policy = Policy::parse_config(policy.as_bytes()).map_err(|e| {
             let error_msg = format!(
                 "Failed to parse session policy: {}. Please check that the policy is valid JSON format with standard brackets [] for arrays.",
                 e
@@ -78,50 +59,27 @@ mod tests {
     }
 
     #[test]
-    fn session_policy_claim_preserves_the_iam_policy_format() {
-        for id_key in [Some("Id"), Some("ID"), None] {
-            let mut input = serde_json::json!({
-                "Version": "2012-10-17",
-                "Statement": [{"Effect": "Allow", "Action": ["s3:GetObject"], "Resource": ["arn:aws:s3:::bucket/*"]}]
-            });
-            if let Some(key) = id_key {
-                input[key] = "read-objects".into();
-            }
-            if id_key == Some("Id") {
-                assert!(Policy::parse_config(input.to_string().as_bytes()).is_err());
-            }
-            let mut claims = HashMap::new();
-            populate_session_policy(&mut claims, &input.to_string()).expect("valid session policy");
+    fn valid_policy_is_encoded_in_the_session_claim() {
+        let mut claims = HashMap::new();
+        let policy = r#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::bucket/*"]}]}"#;
 
-            let encoded = claims[SESSION_POLICY_NAME].as_str().expect("encoded session policy");
-            let decoded = base64_simd::URL_SAFE_NO_PAD
-                .decode_to_vec(encoded.as_bytes())
-                .expect("base64url claim");
-            let json: Value = serde_json::from_slice(&decoded).expect("JSON claim");
-            assert!(json.get("Id").is_none());
-            assert_eq!(json.get("ID").and_then(Value::as_str), id_key.map(|_| "read-objects"));
-            let policy = Policy::parse_config(&decoded).expect("claim must be readable by the IAM policy parser");
-            assert_eq!(policy.version, "2012-10-17");
-            assert_eq!(policy.statements.len(), 1);
-        }
+        populate_session_policy(&mut claims, policy).expect("valid policy should be accepted");
+
+        let encoded = claims
+            .get(SESSION_POLICY_NAME)
+            .and_then(Value::as_str)
+            .expect("session policy claim should be present");
+        let decoded = base64_simd::URL_SAFE_NO_PAD
+            .decode_to_vec(encoded.as_bytes())
+            .expect("session policy claim should be base64url encoded");
+        let decoded_policy: Value = serde_json::from_slice(&decoded).expect("session policy claim should contain JSON");
+        assert_eq!(decoded_policy["Version"], "2012-10-17");
     }
 
     #[test]
     fn invalid_policy_is_rejected_as_invalid_request() {
-        for input in [
-            "not-json",
-            r#"{"Id":"a","ID":"b","Version":"2012-10-17"}"#,
-            r#"{"Id":"a","Id":"b","Version":"2012-10-17"}"#,
-            r#"{"id":"a","Version":"2012-10-17"}"#,
-            r#"{"Id":42,"Version":"2012-10-17"}"#,
-            r#"{"Id":"a"}"#,
-            r#"{"Id":"a","Version":"invalid"}"#,
-            r#"{"Id":"a","Version":"2012-10-17","Statement":[{"Effect":"Allow"}]}"#,
-        ] {
-            let mut claims = HashMap::new();
-            let error = populate_session_policy(&mut claims, input).expect_err("invalid session policy");
-            assert_eq!(error.code(), &S3ErrorCode::InvalidRequest, "{input}");
-            assert!(claims.is_empty());
-        }
+        let mut claims = HashMap::new();
+        let error = populate_session_policy(&mut claims, "not-json").expect_err("invalid policy should fail");
+        assert_eq!(error.code(), &S3ErrorCode::InvalidRequest);
     }
 }

rustfs/src/storage/rpc/node_service.rs

@@ -1585,6 +1585,13 @@ impl Node for NodeService {
         self.handle_write_all(request).await
     }
 
+    async fn write_format_control(
+        &self,
+        request: Request<WriteFormatControlRequest>,
+    ) -> Result<Response<WriteAllResponse>, Status> {
+        self.handle_write_format_control(request).await
+    }
+
     async fn compare_and_update_file(
         &self,
         request: Request<CompareAndUpdateFileRequest>,
@@ -3279,7 +3286,7 @@ mod tests {
         SignalServiceRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
         StartDecommissionRequest, StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationAbortRequest,
         TierMutationFailureClass, TierMutationPeerState, TierMutationPrepareRequest, UpdateMetacacheListingRequest,
-        UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, WriteRequest,
+        UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteFormatControlRequest, WriteMetadataRequest, WriteRequest,
         heal_control_service_client::HealControlServiceClient,
         heal_control_service_server::{HealControlService as _, HealControlServiceServer},
         node_service_client::NodeServiceClient,
@@ -3297,7 +3304,7 @@ mod tests {
     use tonic::{Request, Response, Status};
     use uuid::Uuid;
 
-    const DISK_MUTATION_RPC_METHODS: [&str; 34] = [
+    const DISK_MUTATION_RPC_METHODS: [&str; 35] = [
         "renamedata",
         "renamemultipartdata",
         "recovermultipartsource",
@@ -3316,6 +3323,7 @@ mod tests {
         "writemetadata",
         "updatemetadata",
         "writeall",
+        "writeformatcontrol",
         "compareandupdatefile",
         "delete",
         "deletepaths",
@@ -5147,6 +5155,15 @@ mod tests {
             },
             rustfs_protos::canonical_write_all_request_body
         );
+        assert_gated!(
+            write_format_control,
+            WriteFormatControlRequest {
+                disk: disk.clone(),
+                path: "format.json.bootstrap".into(),
+                data: vec![0x01, 0x02].into(),
+            },
+            rustfs_protos::canonical_write_format_control_request_body
+        );
         assert_gated!(
             compare_and_update_file,
             CompareAndUpdateFileRequest {
@@ -7799,6 +7816,146 @@ mod tests {
         assert!(matches!(disk.read_all(volume, &path).await, Err(DiskError::FileNotFound)));
     }
 
+    #[tokio::test]
+    async fn format_control_rpc_publishes_unformatted_listener_target_in_one_call() {
+        use crate::storage::storage_api::{init_local_disks_with_instance_ctx, new_instance_ctx};
+        use rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClient;
+        use std::sync::atomic::{AtomicUsize, Ordering};
+        use tower_http::trace::TraceLayer;
+
+        let root = tempfile::tempdir().expect("format-control RPC disk");
+        let mut endpoint = Endpoint::try_from(root.path().to_str().expect("UTF-8 disk path")).expect("local endpoint");
+        endpoint.set_pool_index(0);
+        endpoint.set_set_index(0);
+        endpoint.set_disk_index(0);
+        let instance = new_instance_ctx();
+        let pool = PoolEndpoints {
+            legacy: false,
+            set_count: 1,
+            drives_per_set: 1,
+            endpoints: Endpoints::from(vec![endpoint.clone()]),
+            cmd_line: "format-control-rpc".into(),
+            platform: "test".into(),
+        };
+        init_local_disks_with_instance_ctx(&instance, EndpointServerPools::from(vec![pool]))
+            .await
+            .expect("register unformatted listener target");
+        assert!(!root.path().join(".rustfs.sys/format.json").exists());
+        let slot = super::ServerContextSlot::with_instance_context(instance);
+        let service = super::make_server_for_slot(slot);
+        let listener = TcpListener::bind("127.0.0.1:0").await.expect("format-control listener");
+        let address = listener.local_addr().expect("listener address");
+        let calls = Arc::new(AtomicUsize::new(0));
+        let recorded = calls.clone();
+        let trace = TraceLayer::new_for_grpc().on_request(move |request: &http::Request<_>, _: &tracing::Span| {
+            if request.uri().path() == "/node_service.NodeService/WriteFormatControl" {
+                recorded.fetch_add(1, Ordering::SeqCst);
+            }
+        });
+        let (shutdown, stopped) = tokio::sync::oneshot::channel();
+        let server = tokio::spawn(async move {
+            tonic::transport::Server::builder()
+                .layer(trace)
+                .add_service(NodeServiceServer::new(service))
+                .serve_with_incoming_shutdown(TcpListenerStream::new(listener), async {
+                    let _ = stopped.await;
+                })
+                .await
+                .expect("serve format-control target");
+        });
+        let mut client = NodeServiceClient::connect(format!("http://{address}"))
+            .await
+            .expect("connect real format-control client");
+        let drive_id = Uuid::new_v4();
+        let payload = Bytes::from(
+            serde_json::to_vec(&serde_json::json!({
+                "version": "1",
+                "format": "xl-single",
+                "id": Uuid::new_v4(),
+                "xl": {
+                    "version": "3",
+                    "this": drive_id,
+                    "sets": [[drive_id]],
+                    "distributionAlgo": "SIPMOD+PARITY",
+                },
+            }))
+            .expect("encode a valid single-drive format plan"),
+        );
+        let message = WriteFormatControlRequest {
+            disk: endpoint.to_string(),
+            path: "format.json.bootstrap".into(),
+            data: payload.clone(),
+        };
+        let sign = |message: WriteFormatControlRequest| {
+            let mut request = Request::new(message);
+            let canonical =
+                rustfs_protos::canonical_write_format_control_request_body(request.get_ref()).expect("canonical control request");
+            set_tonic_canonical_body_digest(&mut request, &canonical).expect("body digest");
+            mark_v2_authenticated(&mut request);
+            request
+        };
+        let metrics = rustfs_io_metrics::internode_metrics::global_internode_metrics();
+        let before = metrics.snapshot();
+        let response = super::timeout(Duration::from_secs(10), client.write_format_control(sign(message.clone())))
+            .await
+            .expect("bounded atomic control RPC")
+            .expect("atomic control response")
+            .into_inner();
+        assert!(response.success, "{:?}", response.error);
+        assert_eq!(calls.load(Ordering::SeqCst), 1, "one atomic publication uses one unary RPC");
+        let successful = metrics.snapshot();
+        assert_eq!(successful.incoming_requests_total - before.incoming_requests_total, 1);
+        assert_eq!(
+            successful.recv_bytes_total - before.recv_bytes_total,
+            u64::try_from(payload.len()).expect("payload length")
+        );
+        assert_eq!(successful.errors_total, before.errors_total);
+        assert_eq!(
+            tokio::fs::read(root.path().join(".rustfs.sys/format.json.bootstrap"))
+                .await
+                .expect("published control"),
+            payload
+        );
+        assert!(
+            !root.path().join(".rustfs.sys/format.json").exists(),
+            "an endpoint route needs no disk UUID"
+        );
+
+        let mut invalid = message.clone();
+        invalid.path = "../outside".into();
+        let error = client
+            .write_format_control(sign(invalid))
+            .await
+            .expect_err("invalid control path rejected before disk work");
+        assert_eq!(error.code(), tonic::Code::InvalidArgument);
+        let mut missing = message.clone();
+        missing.disk = "unregistered-format-target".into();
+        let response = client
+            .write_format_control(sign(missing))
+            .await
+            .expect("typed missing-target response")
+            .into_inner();
+        assert!(!response.success);
+        assert_eq!(response.error.expect("typed target failure").code, DiskError::DiskNotFound.to_u32());
+        let error = client
+            .write_format_control(Request::new(message))
+            .await
+            .expect_err("unsigned control mutation rejected");
+        assert_eq!(error.code(), tonic::Code::PermissionDenied);
+        let completed = metrics.snapshot();
+        assert_eq!(completed.incoming_requests_total - before.incoming_requests_total, 2);
+        assert_eq!(
+            completed.recv_bytes_total - before.recv_bytes_total,
+            2 * u64::try_from(payload.len()).expect("payload length")
+        );
+        assert_eq!(completed.errors_total - before.errors_total, 1);
+        shutdown.send(()).expect("stop format-control server");
+        super::timeout(Duration::from_secs(10), server)
+            .await
+            .expect("bounded server shutdown")
+            .expect("server task");
+    }
+
     #[cfg(not(windows))]
     async fn check_remote_bucket_creation(interrupted: bool) {
         use crate::storage::storage_api::{

rustfs/src/storage/rpc/node_service/disk.rs

@@ -18,6 +18,7 @@ use crate::storage::storage_api::ecstore_disk::{
     MultipartSourceOperation, MultipartSourceRequest, POOL_META_NATIVE_CAS_METADATA_SUFFIX,
     PoolMetaNativeRecoveryRequest as NativeRecoveryRequest, QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET,
     RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET, is_quota_mutation_fence_path, require_source_condition,
+    validate_format_control_path,
 };
 use crate::storage::storage_api::rpc_consumer::node_service::{
     BatchReadVersionReq, BatchReadVersionResp, ConditionalFileUpdate, DeleteOptions, DiskError, DiskInfoOptions,
@@ -76,6 +77,14 @@ fn startup_cas_rename_observation(
 }
 
 impl LocalMutationTarget {
+    async fn write_local_format_control(&self, disk_ref: &str, path: &str, data: Bytes) -> Result<(), DiskError> {
+        match self {
+            Self::Ready(store) => store.write_local_format_control(disk_ref, path, data).await,
+            Self::Bootstrap(target) => target.write_local_format_control(disk_ref, path, data).await,
+            Self::Unbound => Err(DiskError::DiskNotFound),
+        }
+    }
+
     async fn recover_local_multipart_source(
         &self,
         disk_ref: &str,
@@ -2748,6 +2757,48 @@ impl NodeService {
         }
     }
 
+    pub(super) async fn handle_write_format_control(
+        &self,
+        request: Request<WriteFormatControlRequest>,
+    ) -> Result<Response<WriteAllResponse>, Status> {
+        let _duration = InternodeRpcDurationGuard::handler(INTERNODE_OPERATION_GRPC_WRITE_ALL);
+        let canonical = rustfs_protos::canonical_write_format_control_request_body(request.get_ref())
+            .map_err(|_| Status::invalid_argument("format control request length cannot be represented"))?;
+        verify_tonic_canonical_body_digest(&request, &canonical)
+            .map_err(|_| Status::permission_denied("format control requires a signed canonical body"))?;
+        validate_format_control_path(&request.get_ref().path)
+            .map_err(|_| Status::invalid_argument("unsupported format control filename"))?;
+        let request = request.into_inner();
+        let metrics = runtime_sources::current_internode_metrics();
+        metrics.record_incoming_request_for_operation_and_backend(
+            INTERNODE_OPERATION_GRPC_WRITE_ALL,
+            INTERNODE_TRANSPORT_BACKEND_GRPC,
+        );
+        metrics.record_recv_bytes_for_operation_and_backend(
+            INTERNODE_OPERATION_GRPC_WRITE_ALL,
+            INTERNODE_TRANSPORT_BACKEND_GRPC,
+            request.data.len(),
+        );
+        let target = self.local_mutation_target();
+        let result = target
+            .write_local_format_control(&request.disk, &request.path, request.data)
+            .await;
+        match result {
+            Ok(()) => Ok(Response::new(WriteAllResponse {
+                success: true,
+                error: None,
+            })),
+            Err(error) => {
+                metrics
+                    .record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
+                Ok(Response::new(WriteAllResponse {
+                    success: false,
+                    error: Some(error.into()),
+                }))
+            }
+        }
+    }
+
     pub(super) async fn handle_compare_and_update_file(
         &self,
         request: Request<CompareAndUpdateFileRequest>,
@@ -3028,6 +3079,82 @@ mod tests {
         }
     }
 
+    #[test]
+    #[serial]
+    fn format_control_handler_metrics_count_admitted_payload_once() {
+        use metrics_util::debugging::{DebugValue, DebuggingRecorder};
+        use rustfs_protos::proto_gen::node_service::WriteFormatControlRequest;
+        let recorder = DebuggingRecorder::new();
+        let snapshotter = recorder.snapshotter();
+        let enabled = rustfs_io_metrics::metrics_enabled();
+        rustfs_io_metrics::set_metrics_enabled(true);
+        metrics::with_local_recorder(&recorder, || {
+            let runtime = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("handler metrics runtime");
+            runtime.block_on(async {
+                let server = make_server();
+                let message = WriteFormatControlRequest {
+                    disk: "missing-format-target".into(),
+                    path: "format.json.bootstrap".into(),
+                    data: bytes::Bytes::from_static(b"format-plan"),
+                };
+                let mut request = Request::new(message.clone());
+                let canonical =
+                    rustfs_protos::canonical_write_format_control_request_body(request.get_ref()).expect("canonical control");
+                crate::storage::storage_api::set_tonic_canonical_body_digest(&mut request, &canonical).expect("control digest");
+                request
+                    .metadata_mut()
+                    .insert("x-rustfs-rpc-auth-version", "2".parse().expect("v2 header"));
+                let response = server
+                    .handle_write_format_control(request)
+                    .await
+                    .expect("typed target failure")
+                    .into_inner();
+                assert!(!response.success);
+                let error = server
+                    .handle_write_format_control(Request::new(message))
+                    .await
+                    .expect_err("unsigned control rejection");
+                assert_eq!(error.code(), tonic::Code::PermissionDenied);
+            });
+        });
+        rustfs_io_metrics::set_metrics_enabled(enabled);
+        let entries = snapshotter.snapshot().into_vec();
+        let value = |name: &str| {
+            &entries
+                .iter()
+                .find(|(key, _, _, _)| {
+                    key.key().name() == name
+                        && key
+                            .key()
+                            .labels()
+                            .any(|label| label.key() == "operation" && label.value() == "grpc_write_all")
+                })
+                .expect("format-control handler metric")
+                .3
+        };
+        for (name, expected) in [
+            ("rustfs_system_network_internode_operation_requests_incoming_total", 1),
+            ("rustfs_system_network_internode_operation_recv_bytes_total", 11),
+            ("rustfs_system_network_internode_operation_errors_total", 1),
+        ] {
+            assert!(matches!(value(name), DebugValue::Counter(count) if *count == expected), "{name}");
+        }
+        assert!(
+            matches!(value("rustfs_system_network_internode_operation_server_duration_ms"), DebugValue::Histogram(samples) if samples.len() == 2)
+        );
+        assert!(
+            matches!(value("rustfs_system_network_internode_rpc_active"), DebugValue::Gauge(active) if active.into_inner() == 0.0)
+        );
+        assert!(
+            entries
+                .iter()
+                .all(|(key, _, _, _)| key.key().name() != "rustfs_system_network_internode_operation_duration_ms")
+        );
+    }
+
     #[test]
     #[serial]
     fn handlers_record_duration_on_missing_disk_errors() {

rustfs/src/storage/storage_api.rs

@@ -554,7 +554,7 @@ pub(crate) mod ecstore_disk {
         RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RemoteBackgroundIo,
         RenameDataResp, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, get_object_disk_read_timeout,
         is_quota_mutation_fence_path, require_source_condition, validate_batch_read_version_item_count,
-        with_remote_background_io,
+        validate_format_control_path, with_remote_background_io,
     };
     #[cfg(test)]
     pub(crate) use rustfs_ecstore::api::disk::{

scripts/ci_gate.py

@@ -33,7 +33,7 @@ NIGHTLY_JOBS = ("connect-feature-boundaries", "fips-provider", "uring-integratio
 # against this revision's expected job contract. The workflow refuses a plan
 # that does not carry the current marker and falls back to the full matrix, so
 # a base planner predating the contract cannot authorize a narrower selection.
-PLAN_CONTRACT = "5"
+PLAN_CONTRACT = "4"
 # These jobs still exercise the server and its black-box harness when only E2E
 # Rust sources change. Production code, manifests and shared test configuration
 # always select the full matrix. No production package depends on e2e_test.
@@ -311,8 +311,7 @@ def closure(seeds: set[str], graph: dict[str, set[str]], reverse: bool = False)
 def expected_results(mode: str, event: str, ref: str, connect: bool = True, action: str = "", rehearse_shards: bool = False) -> dict[str, str]:
     if event not in EVENTS:
         raise ValueError(f"unsupported CI event: {event!r}")
-    # `proven`: a push whose tree a successful merge-group run already validated.
-    if mode not in ("docs", "e2e", "full", "proven") or (mode != "full" and event != ("push" if mode == "proven" else "pull_request")):
+    if mode not in ("docs", "e2e", "full") or (mode != "full" and event != "pull_request"):
         raise ValueError(f"invalid CI selection: {mode!r} for {event!r}")
     if type(rehearse_shards) is not bool or (rehearse_shards and event != "workflow_dispatch"):
         raise ValueError("shard rehearsal requires an explicit manual event")
@@ -377,8 +376,7 @@ def configurations(jobs: dict[str, str]) -> list[dict]:
              "default_features": True, "features": features} for identity, job, profile, features in values if jobs[job] == "success"]
 
 
-def select_plan(event: str, base: str, head: str, ref: str, root: Path, action: str = "", rehearse_shards: bool = False,
-                *, proof: dict | None = None) -> dict:
+def select_plan(event: str, base: str, head: str, ref: str, root: Path, action: str = "", rehearse_shards: bool = False) -> dict:
     if event not in EVENTS or not SHA.fullmatch(head):
         raise ValueError("unsupported event or missing candidate SHA")
     tree = git(root, "rev-parse", f"{head}^{{tree}}").strip()
@@ -475,10 +473,6 @@ def select_plan(event: str, base: str, head: str, ref: str, root: Path, action:
     effective_packages = all_packages if shadow and mode != "docs" else sorted(selected)
     effective_connect = mode == "full" if shadow else connect
     effective_full = mode == "full" if shadow else full
-    proven = event == "push" and mode == "full" and isinstance(proof, dict) and proof.get("tree") == tree
-    if proven:
-        mode = "proven"
-        reasons.append(f"merge_group run {proof['url']} already validated tree {tree}")
     jobs = expected_results(mode, event, ref, effective_connect, action, rehearse_shards)
     plan = {"schema_version": 1, "event": event, "action": action, "ref": ref, "base": base if SHA.fullmatch(base) else "",
             "head": head, "tree": tree, "policy": {"revision": policy_revision, "rollout": "shadow" if shadow else "enforced"},
@@ -487,9 +481,7 @@ def select_plan(event: str, base: str, head: str, ref: str, root: Path, action:
             "gateway_smoke": gateway_smoke_selection(root, event, base, head, changed),
             "configurations": configurations(jobs), "jobs": jobs, "proposed": proposed,
             "reasons": ["shadow rollout preserves current validation responsibility"] + reasons if shadow else reasons,
-            "not_applicable": {job: f"proven by {proof['url']}" if proven else f"not required for {mode}/{event}"
-                               for job, result in jobs.items() if result == "skipped"},
-            "proven_by": proof if proven else None}
+            "not_applicable": {job: f"not required for {mode}/{event}" for job, result in jobs.items() if result == "skipped"}}
     return plan
 
 
@@ -521,12 +513,6 @@ def validate_plan(plan: object, event: str, ref: str, head: str = "", tree: str
         raise ValueError("invalid plan packages")
     if plan.get("mode") != "docs" and not packages:
         raise ValueError("code validation plan must not be empty")
-    proof = plan.get("proven_by")
-    if (proof is not None or plan.get("mode") == "proven") and not (
-            isinstance(proof, dict) and plan.get("mode") == "proven" and proof.get("tree") == plan["tree"]
-            and type(proof.get("run_id")) is int and proof["run_id"] > 0
-            and re.fullmatch(rf"https://github\.com/[\w.-]+/[\w.-]+/actions/runs/{proof['run_id']}", str(proof.get("url")))):
-        raise ValueError("a proven push requires the merge-group run that validated its tree")
     expected = expected_results(plan.get("mode"), event, ref, plan["connect"], plan.get("action", ""), plan["rehearse_shards"])
     if plan.get("jobs") != expected:
         raise ValueError("plan job contract differs from selected responsibilities")
@@ -814,7 +800,7 @@ def check_workflow(root: Path) -> list[str]:
     # Truth tables protect behavior while allowing equivalent expression and YAML formatting.
     for event, rehearse_shards in [(event, rehearsal) for event in EVENTS
                                    for rehearsal in ((False, True) if event == "workflow_dispatch" else (False,))]:
-        for mode in {"pull_request": ("docs", "e2e", "full"), "push": ("full", "proven")}.get(event, ("full",)):
+        for mode in (("docs", "e2e", "full") if event == "pull_request" else ("full",)):
             for ref in ("refs/heads/main", "refs/heads/release"):
                 for connect in (True, False) if event == "pull_request" and mode == "full" else (mode == "full",):
                     for action in ("opened", "closed") if event == "pull_request" else ("",):
@@ -891,14 +877,9 @@ def main() -> int:
         return not unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful()
     if sys.argv[1:] == ["select"]:
         try:
-            # scripts/ci_merge_group_proof.py output; select_plan accepts it only for a push of the same tree.
-            try:
-                proof = json.loads(os.environ.get("CI_MERGE_GROUP_PROOF") or "null")
-            except ValueError:
-                proof = None
             plan = select_plan(os.environ.get("GITHUB_EVENT_NAME", ""), os.environ.get("CI_BASE_SHA", ""),
-                               os.environ.get("GITHUB_SHA", ""), os.environ.get("GITHUB_REF", ""), Path.cwd(),
-                               os.environ.get("CI_EVENT_ACTION", ""), os.environ.get("CI_REHEARSE_SHARDS", "false") == "true", proof=proof)
+                               os.environ.get("GITHUB_SHA", ""), os.environ.get("GITHUB_REF", ""),
+                               Path.cwd(), os.environ.get("CI_EVENT_ACTION", ""), os.environ.get("CI_REHEARSE_SHARDS", "false") == "true")
             with open(os.environ["GITHUB_OUTPUT"], "a") as output:
                 for name, value in (("mode", plan["mode"]), ("plan", json.dumps(plan, separators=(",", ":"))),
                                     ("connect", str(plan["connect"]).lower()), ("packages", json.dumps(plan["packages"], separators=(",", ":"))),
@@ -926,16 +907,7 @@ def main() -> int:
                 base = (payload["pull_request"]["base"]["sha"] if event == "pull_request" else
                         payload["merge_group"]["base_sha"] if event == "merge_group" else payload.get("before", ""))
                 action = payload.get("action", "")
-            needs = json.loads(os.environ["CI_NEEDS"])
-            errors = verify_results(needs, event, os.environ.get("GITHUB_REF", ""), head, tree, base, action)
-            plan = json.loads(needs["classify-changes"]["outputs"].get("plan") or "null")
-            if not errors and isinstance(plan, dict) and plan["mode"] == "proven":
-                # The check list shows these jobs as skipped like any other; say why.
-                note = f"Validation jobs were skipped as proven: merge_group run {plan['proven_by']['url']} passed tree {plan['tree']}."
-                print(note)
-                if os.environ.get("GITHUB_STEP_SUMMARY"):
-                    with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as summary:
-                        summary.write(note + "\n")
+            errors = verify_results(json.loads(os.environ["CI_NEEDS"]), event, os.environ.get("GITHUB_REF", ""), head, tree, base, action)
         except (OSError, KeyError, TypeError, ValueError, subprocess.CalledProcessError) as error:
             errors = [str(error)]
     elif sys.argv[1:] == ["--check-workflow"]:

scripts/ci_merge_group_proof.py

@@ -1,66 +0,0 @@
-#!/usr/bin/env python3
-"""Find the successful merge-group run of ci.yml that already validated a pushed tree.
-
-A merge-queue landing fast-forwards the branch to the group's commit, whose
-tree that run validated. Any doubt (no token, an API or network error, a
-rate limit, an unexpected payload, no matching run) returns None, and the
-caller then validates the push in full. A push that did not come through the
-queue has no such run.
-"""
-import json
-import os
-import re
-import subprocess
-import urllib.request
-
-SHA = re.compile(r"[0-9a-f]{40}")
-
-
-def lookup(repository: str, tree: str, branch: str, token: str, opener=None) -> dict | None:
-    if not (token and SHA.fullmatch(tree) and re.fullmatch(r"[\w.-]+/[\w.-]+", repository)
-            and re.fullmatch(r"[\w.-]+(/[\w.-]+)*", branch) and ".." not in branch):
-        return None
-    request = urllib.request.Request(
-        f"https://api.github.com/repos/{repository}/actions/workflows/ci.yml/runs?event=merge_group&status=success&per_page=100",
-        headers={"Authorization": f"Bearer {token}", "Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28"})
-    try:
-        with (opener or urllib.request.urlopen)(request, timeout=20) as response:
-            runs = json.load(response)["workflow_runs"]
-    except (OSError, ValueError, KeyError, TypeError):
-        return None
-    if not isinstance(runs, list):
-        return None
-    for run in runs:
-        if proves(run, repository, branch, tree):
-            return {"run_id": run["id"], "url": run["html_url"], "tree": tree}
-    return None
-
-
-def proves(run, repository: str, branch: str, tree: str) -> bool:
-    """Whether `run`, listed from ci.yml's runs, is a successful merge-queue run of `branch` in `repository` on `tree`.
-
-    The push plan and the release pipeline both accept a tree on this rule alone.
-    """
-    return (isinstance(run, dict) and run.get("event") == "merge_group" and run.get("status") == "completed"
-            and run.get("conclusion") == "success" and isinstance(run.get("head_branch"), str)
-            and run["head_branch"].startswith(f"gh-readonly-queue/{branch}/")
-            and isinstance(run.get("head_commit"), dict) and run["head_commit"].get("tree_id") == tree
-            and type(run.get("id")) is int and run["id"] > 0 and isinstance(run.get("html_url"), str)
-            and run["html_url"] == f"https://github.com/{repository}/actions/runs/{run['id']}")
-
-
-def main() -> int:
-    """Print the proof for GITHUB_SHA on GITHUB_REF as JSON, or null; never fail the step."""
-    ref = os.environ.get("GITHUB_REF", "")
-    try:
-        tree = subprocess.run(["git", "rev-parse", "--verify", "--end-of-options", f"{os.environ.get('GITHUB_SHA', '')}^{{tree}}"],
-                              capture_output=True, text=True, check=True).stdout.strip()
-    except (OSError, subprocess.CalledProcessError):
-        tree = ""
-    print(json.dumps(lookup(os.environ.get("GITHUB_REPOSITORY", ""), tree,
-                            ref.removeprefix("refs/heads/") if ref.startswith("refs/heads/") else "", os.environ.get("GITHUB_TOKEN", ""))))
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

scripts/release/pipeline.py

@@ -12,7 +12,6 @@ from pathlib import Path
 import re
 import shutil
 import subprocess
-import sys
 import tempfile
 import urllib.error
 import zipfile
@@ -20,9 +19,6 @@ import zipfile
 import candidate as c
 import release_gate
 
-sys.path.insert(0, str(c.ROOT / "scripts"))
-import ci_merge_group_proof as proof
-
 
 def current_run(workflow):
     c.require(os.environ.get("GITHUB_REPOSITORY") == c.REPOSITORY and os.environ.get("GITHUB_REF") == "refs/heads/main",
@@ -48,26 +44,9 @@ def output_inputs(inputs):
         c.emit_output(field, inputs[field])
 
 
-CI_GATE = ("Quick Checks", "Test and Lint", "End-to-End Tests (full merge gate)")
-
-
-def run_jobs(run):
-    jobs = []
-    for page in range(1, 11):
-        batch = c.github(f"repos/{c.REPOSITORY}/actions/runs/{run['id']}/attempts/{run['run_attempt']}/jobs?per_page=100&page={page}")["jobs"]
-        jobs.extend(batch)
-        if len(batch) < 100:
-            break
-    return jobs
-
-
-def succeeded(jobs, names):
-    selected = {name: [job for job in jobs if job.get("name") == name] for name in names}
-    return all(len(found) == 1 and found[0].get("status") == "completed" and found[0].get("conclusion") == "success" for found in selected.values())
-
-
 def source_ci(source, version):
     runs = c.github(f"repos/{c.REPOSITORY}/actions/workflows/ci.yml/runs?head_sha={source}&status=success&per_page=10")["workflow_runs"]
+    required = {"Quick Checks", "Test and Lint", "End-to-End Tests (full merge gate)"}
     for run in runs:
         if (run.get("head_sha") != source or run.get("path") != ".github/workflows/ci.yml"
                 or run.get("head_branch") not in ("main", f"release/{version}")
@@ -75,18 +54,15 @@ def source_ci(source, version):
                 or run.get("conclusion") != "success" or run.get("status") != "completed"
                 or (run.get("repository") or {}).get("full_name") != c.REPOSITORY):
             continue
-        jobs = run_jobs(run)
-        if succeeded(jobs, CI_GATE):
+        jobs = []
+        for page in range(1, 11):
+            batch = c.github(f"repos/{c.REPOSITORY}/actions/runs/{run['id']}/attempts/{run['run_attempt']}/jobs?per_page=100&page={page}")["jobs"]
+            jobs.extend(batch)
+            if len(batch) < 100:
+                break
+        selected = {name: [job for job in jobs if job.get("name") == name] for name in required}
+        if all(len(found) == 1 and found[0].get("status") == "completed" and found[0].get("conclusion") == "success" for found in selected.values()):
             return {"run_id": run["id"], "run_attempt": run["run_attempt"]}
-        # The full gate of a queue-landed commit may have run only in the merge-group run of its tree, which
-        # its push run skips as proven. That run is found again here (scripts/ci_merge_group_proof.py) and,
-        # if it passed the whole gate itself, is the evidence.
-        if run["event"] == "push" and succeeded(jobs, CI_GATE[:2]):
-            tree = c.command(["git", "rev-parse", "--verify", "--end-of-options", f"{source}^{{tree}}"], cwd=c.ROOT)
-            queued = c.github(f"repos/{c.REPOSITORY}/actions/workflows/ci.yml/runs?event=merge_group&status=success&per_page=100")
-            for proving in queued["workflow_runs"]:
-                if proof.proves(proving, c.REPOSITORY, run["head_branch"], tree) and succeeded(run_jobs(proving), CI_GATE):
-                    return {"run_id": proving["id"], "run_attempt": proving["run_attempt"]}
     raise ValueError("no complete exact-source CI found; run ci.yml on main or the release branch before creating a candidate")
 
 

scripts/release/test_pipeline.py

@@ -175,56 +175,6 @@ class SourceCITests(unittest.TestCase):
             with self.assertRaises(ValueError):
                 pipeline.source_ci("a" * 40, "1.2.3")
 
-    def test_a_queue_landed_push_is_accepted_only_through_the_queue_run_of_its_tree(self):
-        tree = "d" * 40
-        gate = ("Quick Checks", "Test and Lint", "End-to-End Tests (full merge gate)")
-        push = dict(run_metadata(60, "ci.yml", "a" * 40), event="push")
-
-        def queued(run_id, branch, tree_id=tree):
-            return dict(run_metadata(run_id, "ci.yml", "a" * 40), event="merge_group", run_attempt=3,
-                        head_branch=f"gh-readonly-queue/{branch}/pr-1-" + "c" * 40, head_commit={"tree_id": tree_id},
-                        html_url=f"https://github.com/{c.REPOSITORY}/actions/runs/{run_id}")
-        listing = [queued(62, "main", "e" * 40), queued(63, "release/1.2.3"), queued(61, "main")]
-        # Keyed by (run id, attempt): reading another attempt's jobs is a KeyError, not evidence.
-        jobs = {(60, 2): [{"name": name, "status": "completed", "conclusion": "skipped" if name == gate[2] else "success"} for name in gate]}
-        jobs.update({(run["id"], 3): [{"name": name, "status": "completed", "conclusion": "success"} for name in gate] for run in listing})
-
-        def api(path):
-            if "/jobs?" in path:
-                run_id, attempt = path.split("/runs/")[1].split("/jobs?")[0].split("/attempts/")
-                return {"jobs": jobs[(int(run_id), int(attempt))]}
-            return {"workflow_runs": listing if "event=merge_group" in path else [push]}
-
-        def source_ci():
-            with patch.object(c, "github", side_effect=api), patch.object(c, "command", return_value=tree) as git:
-                found = pipeline.source_ci("a" * 40, "1.2.3")
-            self.assertEqual(git.call_args.args[0], ["git", "rev-parse", "--verify", "--end-of-options", "a" * 40 + "^{tree}"])
-            return found
-        self.assertEqual(source_ci(), {"run_id": 61, "run_attempt": 3})
-        listing.reverse()
-        self.assertEqual(source_ci(), {"run_id": 61, "run_attempt": 3})
-        push["head_branch"] = "release/1.2.3"
-        self.assertEqual(source_ci(), {"run_id": 63, "run_attempt": 3})
-        del listing[listing.index(next(run for run in listing if run["id"] == 63))]
-        with self.assertRaises(ValueError):
-            source_ci()
-        push["head_branch"] = "main"
-        match = next(run for run in listing if run["id"] == 61)
-        refusals = [(match, "head_commit", {"tree_id": "e" * 40}), (match, "head_branch", "gh-readonly-queue/release/1.2.3/pr-1-x"),
-                    (match, "conclusion", "failure"), (match, "event", "push"), (push, "event", "workflow_dispatch")]
-        for record, key, wrong in refusals:
-            old = record[key]; record[key] = wrong
-            with self.subTest(key=key, wrong=wrong), self.assertRaises(ValueError):
-                source_ci()
-            record[key] = old
-        for key, index, conclusion in (((61, 3), 2, "skipped"), ((61, 3), 0, "failure"), ((60, 2), 1, "failure")):
-            old = jobs[key][index]["conclusion"]; jobs[key][index]["conclusion"] = conclusion
-            with self.subTest(run=key, job=gate[index]), self.assertRaises(ValueError):
-                source_ci()
-            jobs[key][index]["conclusion"] = old
-        del jobs[(61, 3)][2]
-        with self.assertRaises(ValueError):
-            source_ci()
 
 class SupplyChainTests(unittest.TestCase):
     def test_actual_generator_uses_frozen_lock_and_all_package_formats(self):

scripts/test_ci_merge_group_proof.py

@@ -1,103 +0,0 @@
-#!/usr/bin/env python3
-"""Fail-closed cases for the merge-group proof of a pushed tree."""
-import io
-import json
-import os
-import subprocess
-import unittest
-import urllib.error
-from unittest.mock import patch
-
-import ci_merge_group_proof as proof
-
-TREE = "b" * 40
-
-
-def run(**changes):
-    value = {"id": 7, "html_url": "https://github.com/rustfs/rustfs/actions/runs/7", "event": "merge_group",
-             "status": "completed", "conclusion": "success", "head_branch": "gh-readonly-queue/main/pr-1-" + "c" * 40,
-             "head_commit": {"tree_id": TREE}}
-    value.update(changes)
-    return value
-
-
-class Response(io.BytesIO):
-    def __enter__(self):
-        return self
-
-    def __exit__(self, *_):
-        self.close()
-
-
-def answering(payload):
-    requests = []
-
-    def opener(request, timeout):
-        requests.append((request, timeout))
-        if isinstance(payload, Exception):
-            raise payload
-        return Response(payload if isinstance(payload, bytes) else json.dumps(payload).encode())
-    return opener, requests
-
-
-class ProofTests(unittest.TestCase):
-    def lookup(self, payload, **overrides):
-        arguments = {"repository": "rustfs/rustfs", "tree": TREE, "branch": "main", "token": "token"}
-        arguments.update(overrides)
-        opener, requests = answering(payload)
-        return proof.lookup(**arguments, opener=opener), requests
-
-    def test_a_successful_merge_group_run_on_the_same_tree_proves_it(self):
-        found, requests = self.lookup({"workflow_runs": [run(id=6, head_commit={"tree_id": "d" * 40}), run()]})
-        self.assertEqual(found, {"run_id": 7, "url": "https://github.com/rustfs/rustfs/actions/runs/7", "tree": TREE})
-        request, timeout = requests[0]
-        self.assertEqual(request.full_url, "https://api.github.com/repos/rustfs/rustfs/actions/workflows/ci.yml/runs"
-                                           "?event=merge_group&status=success&per_page=100")
-        self.assertEqual(request.get_header("Authorization"), "Bearer token")
-        self.assertLessEqual(timeout, 30)
-
-    def test_any_run_that_is_not_a_successful_queue_run_of_this_tree_proves_nothing(self):
-        for changes in ({"head_commit": {"tree_id": "d" * 40}}, {"head_commit": None}, {"head_commit": {}},
-                        {"conclusion": "failure"}, {"conclusion": "cancelled"}, {"conclusion": None},
-                        {"status": "in_progress"}, {"event": "push"}, {"event": "pull_request"},
-                        {"head_branch": "main"}, {"head_branch": "gh-readonly-queue/release/pr-1-x"},
-                        {"head_branch": "gh-readonly-queue/mainline/pr-1-x"},
-                        {"head_branch": None}, {"id": "7"}, {"id": 0}, {"id": True},
-                        {"html_url": "https://example.invalid/runs/7"}, {"html_url": None}):
-            with self.subTest(changes=changes):
-                self.assertIsNone(self.lookup({"workflow_runs": [run(**changes)]})[0])
-
-    def test_every_doubt_about_the_answer_runs_everything(self):
-        rate_limited = urllib.error.HTTPError("https://api.github.com", 403, "rate limit exceeded", {}, None)
-        for payload in (rate_limited, urllib.error.HTTPError("u", 500, "error", {}, None), urllib.error.URLError("dns"),
-                        TimeoutError(), b"not json", b"[]", {"workflow_runs": None}, {"workflow_runs": [None, "x"]}, {}, {"workflow_runs": []}):
-            with self.subTest(payload=payload):
-                self.assertIsNone(self.lookup(payload)[0])
-
-    def test_no_request_without_a_token_or_with_a_malformed_identity(self):
-        for overrides in ({"token": ""}, {"tree": ""}, {"tree": "B" * 40}, {"tree": "b" * 39}, {"branch": ""},
-                          {"branch": "main/../x"}, {"repository": "rustfs"}, {"repository": "a/b/c"}):
-            with self.subTest(overrides=overrides):
-                found, requests = self.lookup({"workflow_runs": [run()]}, **overrides)
-                self.assertIsNone(found)
-                self.assertEqual(requests, [])
-
-    def test_the_step_prints_the_proof_of_the_pushed_commit_or_null(self):
-        head = subprocess.run(["git", "rev-parse", "HEAD"], capture_output=True, text=True, check=True).stdout.strip()
-        tree = subprocess.run(["git", "rev-parse", "HEAD^{tree}"], capture_output=True, text=True, check=True).stdout.strip()
-        environment = {"GITHUB_REPOSITORY": "rustfs/rustfs", "GITHUB_REF": "refs/heads/main", "GITHUB_SHA": head, "GITHUB_TOKEN": "token"}
-        cases = [({}, {"workflow_runs": [run(head_commit={"tree_id": tree})]}, {"run_id": 7, "url": run()["html_url"], "tree": tree}),
-                 ({}, urllib.error.URLError("dns"), None),
-                 ({"GITHUB_SHA": "0" * 40}, {"workflow_runs": [run(head_commit={"tree_id": tree})]}, None),
-                 ({"GITHUB_SHA": "--output=x"}, {"workflow_runs": [run(head_commit={"tree_id": tree})]}, None),
-                 ({"GITHUB_REF": "refs/tags/main"}, {"workflow_runs": [run(head_commit={"tree_id": tree})]}, None)]
-        for overrides, payload, expected in cases:
-            opener, _ = answering(payload)
-            with self.subTest(overrides=overrides, payload=payload), patch.dict(os.environ, dict(environment, **overrides), clear=True), \
-                    patch.object(proof.urllib.request, "urlopen", opener), patch("sys.stdout", new_callable=io.StringIO) as output:
-                self.assertEqual(proof.main(), 0)
-                self.assertEqual(json.loads(output.getvalue()), expected)
-
-
-if __name__ == "__main__":
-    unittest.main()

scripts/test_ci_plan.py

@@ -44,8 +44,8 @@ class Repository:
         self.git('commit', '-qm', 'fixture')
         return self.git('rev-parse', 'HEAD')
 
-    def plan(self, head=None, event='pull_request', base=None, action='opened', proof=None):
-        return gate.select_plan(event, base or self.base, head or self.git('rev-parse', 'HEAD'), 'refs/heads/main', self.root, action, proof=proof)
+    def plan(self, head=None, event='pull_request', base=None, action='opened'):
+        return gate.select_plan(event, base or self.base, head or self.git('rev-parse', 'HEAD'), 'refs/heads/main', self.root, action)
 
 
 class PlannerTests(unittest.TestCase):
@@ -182,24 +182,6 @@ class PlannerTests(unittest.TestCase):
         for event in ('push', 'merge_group', 'schedule', 'workflow_dispatch'):
             self.assertTrue(self.repo.plan(head, event=event, base=baseline)['full_validation'])
 
-    def test_push_skips_validation_only_for_the_tree_a_merge_group_run_proved(self):
-        head, tree = self.repo.git('rev-parse', 'HEAD'), self.repo.git('rev-parse', 'HEAD^{tree}')
-        proof = {'run_id': 7, 'url': 'https://github.com/rustfs/rustfs/actions/runs/7', 'tree': tree}
-        plan = self.repo.plan(head, event='push', proof=proof)
-        self.assertEqual((plan['mode'], plan['proven_by']), ('proven', proof))
-        self.assertEqual({job for job, state in plan['jobs'].items() if state == 'success'}, set(gate.ALWAYS_JOBS))
-        self.assertTrue(all(proof['url'] in plan['not_applicable'][job] for job in gate.CODE_JOBS))
-        self.assertEqual(gate.validate_plan(plan, 'push', 'refs/heads/main', head, tree), plan['jobs'])
-        # A direct push has no proving run; a proof of another tree, or on any other event, proves nothing.
-        for event, candidate in [('push', None), ('push', dict(proof, tree='0' * 40)), ('merge_group', proof),
-                                 ('schedule', proof), ('workflow_dispatch', proof), ('pull_request', proof)]:
-            with self.subTest(event=event, proof=candidate):
-                other = self.repo.plan(head, event=event, proof=candidate)
-                self.assertNotEqual(other['mode'], 'proven')
-                self.assertIsNone(other['proven_by'])
-                if event != 'pull_request':
-                    self.assertTrue(all(other['jobs'][job] == 'success' for job in (gate.LINT_JOB, gate.ARCHIVE_PRODUCER, 'workspace-tests')))
-
     def test_individual_stable_version_fields_without_new_rpm_history_are_metadata(self):
         baseline, files = self.release_metadata_base()
         for path, content in files.items():
@@ -482,8 +464,7 @@ class GateTests(unittest.TestCase):
                 'base': 'c' * 40, 'action': '', 'mode': mode, 'connect': connect, 'full_validation': mode == 'full', 'packages': ['rustfs'] if mode != 'docs' else [],
                 'rehearse_shards': False,
                 'policy': {'revision': 'c' * 40, 'rollout': 'enforced'}, 'jobs': jobs, 'configurations': gate.configurations(jobs),
-                'reasons': ['fixture'], 'not_applicable': {job: 'fixture' for job, state in jobs.items() if state == 'skipped'},
-                'proven_by': {'run_id': 7, 'url': 'https://github.com/rustfs/rustfs/actions/runs/7', 'tree': 'b' * 40} if mode == 'proven' else None}
+                'reasons': ['fixture'], 'not_applicable': {job: 'fixture' for job, state in jobs.items() if state == 'skipped'}}
 
     def needs(self, plan):
         needs = {job: {'result': state} for job, state in plan['jobs'].items()}
@@ -491,7 +472,7 @@ class GateTests(unittest.TestCase):
         return needs
 
     def test_failure_cancel_missing_extra_or_unexpected_skip_is_rejected(self):
-        for event, mode in [('pull_request', 'docs'), ('pull_request', 'e2e'), ('pull_request', 'full'), ('push', 'full'), ('push', 'proven'), ('merge_group', 'full'), ('schedule', 'full'), ('workflow_dispatch', 'full')]:
+        for event, mode in [('pull_request', 'docs'), ('pull_request', 'e2e'), ('pull_request', 'full'), ('push', 'full'), ('merge_group', 'full'), ('schedule', 'full'), ('workflow_dispatch', 'full')]:
             plan = self.plan(event, mode)
             good = self.needs(plan)
             self.assertEqual(gate.verify_results(good, event, plan['ref'], plan['head'], plan['tree']), [])
@@ -521,28 +502,6 @@ class GateTests(unittest.TestCase):
         self.assertTrue(gate.verify_results(good, plan['event'], plan['ref'], base='d' * 40))
         self.assertTrue(gate.verify_results(good, plan['event'], plan['ref'], action='closed'))
 
-    def test_a_proven_push_needs_its_own_tree_proof_and_nothing_else_may_carry_one(self):
-        proven, full = self.plan('push', 'proven'), self.plan('push', 'full')
-        self.assertEqual(gate.verify_results(self.needs(proven), 'push', proven['ref'], proven['head'], proven['tree']), [])
-        proof = proven['proven_by']
-        bad = [dict(proven, proven_by=None), dict(proven, proven_by=dict(proof, tree='c' * 40)),
-               dict(proven, proven_by=dict(proof, run_id='7')), dict(proven, proven_by=dict(proof, run_id=0)),
-               dict(proven, proven_by=dict(proof, url='https://example.invalid/7')), dict(proven, proven_by={}),
-               dict(full, proven_by=proof)]
-        for plan in bad:
-            with self.subTest(plan=plan['proven_by']), self.assertRaises(ValueError):
-                gate.validate_plan(plan, 'push', plan['ref'])
-        for event in ('pull_request', 'merge_group', 'schedule', 'workflow_dispatch'):
-            with self.subTest(event=event), self.assertRaises(ValueError):
-                gate.expected_results('proven', event, 'refs/heads/main')
-
-    def test_the_queue_run_that_proves_a_push_validates_everything_that_push_would(self):
-        queued = {job for job, result in gate.expected_results('full', 'merge_group', 'refs/heads/main').items() if result == 'success'}
-        for ref in gate.INTEGRATION_PUSH_REFS:
-            pushed = {job for job, result in gate.expected_results('full', 'push', ref).items() if result == 'success'}
-            with self.subTest(ref=ref):
-                self.assertLessEqual(pushed, queued)
-
     def test_legacy_base_cannot_authorize_skipped_new_jobs(self):
         expected = gate.expected_results('full', 'pull_request', 'refs/heads/main')
         needs = {job: {'result': result} for job, result in expected.items()}
@@ -585,11 +544,11 @@ class GateTests(unittest.TestCase):
             elif job in gate.OPTIONAL_JOBS[:2]:
                 expression = "contains(fromJSON('[\"schedule\",\"workflow_dispatch\"]'), github.event_name)"
             elif job == 'e2e-full':
-                expression = "needs.classify-changes.outputs.mode == 'full' && (github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main'))"
+                expression = "github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')"
             elif job in gate.SHARD_JOBS:
                 expression = "needs.classify-changes.outputs.rehearse_shards == 'true'"
             elif job in gate.E2E_JOBS:
-                expression = "contains(fromJSON('[\"full\",\"e2e\"]'), needs.classify-changes.outputs.mode)"
+                expression = "needs.classify-changes.outputs.mode != 'docs'"
             else:
                 expression = "needs.classify-changes.outputs.mode == 'full'"
             expression = f"({expression}) && (github.event_name != 'pull_request' || github.event.action != 'closed')"