diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d12a8b68..bb4a25553 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -582,13 +582,59 @@ jobs: install-build-packaging-tools: 'false' - name: Build debug binary - run: cargo build -p rustfs --bins --features e2e-test-hooks + run: | + python3 - <<'PYBUILD' + import hashlib + import json + import os + import pathlib + import subprocess + + def git(*args): + return subprocess.check_output(["git", *args], text=True).strip() + + def sha256(path): + digest = hashlib.sha256() + with pathlib.Path(path).open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + argv = ["cargo", "build", "-p", "rustfs", "--bins", "--features", "e2e-test-hooks"] + commit, tree = git("rev-parse", "HEAD"), git("rev-parse", "HEAD^{tree}") + clean_before = not git("status", "--porcelain", "--untracked-files=normal") + if not clean_before: + raise SystemExit("hooks binary requires a clean build checkout") + lock_sha256 = sha256("Cargo.lock") + lock_git_blob = git("hash-object", "Cargo.lock") + rustc = subprocess.check_output(["rustc", "-vV"], text=True) + host = next(line.removeprefix("host: ") for line in rustc.splitlines() if line.startswith("host: ")) + if os.environ.get("CARGO_BUILD_TARGET") or pathlib.Path(os.environ.get("CARGO_TARGET_DIR", "target")).resolve() != pathlib.Path("target").resolve(): + raise SystemExit("this artifact requires the native target/debug output") + subprocess.run(argv, check=True) + clean_after = not git("status", "--porcelain", "--untracked-files=normal") + if not clean_after or commit != git("rev-parse", "HEAD") or tree != git("rev-parse", "HEAD^{tree}") or lock_sha256 != sha256("Cargo.lock"): + raise SystemExit("hooks binary source changed while building") + manifest = { + "schema": 1, "commit": commit, "tree": tree, + "clean_before": clean_before, "clean_after": clean_after, + "lock_sha256": lock_sha256, "lock_git_blob": lock_git_blob, + "argv": argv, "profile": "debug", "target": host, + "features": ["e2e-test-hooks"], + "rustc_verbose": rustc, + "build_flags": {key: os.environ[key] for key in ("RUSTFLAGS", "CARGO_ENCODED_RUSTFLAGS", "CARGO_BUILD_TARGET", "CARGO_TARGET_DIR", "RUSTUP_TOOLCHAIN") if key in os.environ}, + "binary_sha256": sha256("target/debug/rustfs"), + } + pathlib.Path("target/debug/rustfs.e2e-startup-cas-build.json").write_text(json.dumps(manifest, indent=2) + "\n") + PYBUILD - name: Upload debug binary uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: name: rustfs-debug-binary - path: target/debug/rustfs + path: | + target/debug/rustfs + target/debug/rustfs.e2e-startup-cas-build.json if-no-files-found: error retention-days: 1 @@ -906,6 +952,36 @@ jobs: - name: Make binary executable run: chmod +x ./target/debug/rustfs + - name: Preserve startup CAS binary input + env: + STARTUP_CAS_INPUT: ${{ runner.temp }}/rustfs-startup-cas-input + run: | + python3 - <<'PYINPUT' + import hashlib + import json + import os + import pathlib + import shutil + import subprocess + + source = pathlib.Path("target/debug/rustfs") + manifest_path = source.with_name("rustfs.e2e-startup-cas-build.json") + manifest = json.loads(manifest_path.read_text()) + target = pathlib.Path(os.environ["STARTUP_CAS_INPUT"]) + target.mkdir(parents=True, exist_ok=True) + binary = target / "rustfs" + shutil.copy2(source, binary) + digest = hashlib.sha256() + with binary.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() + if manifest["binary_sha256"] != digest.hexdigest() or manifest["commit"] != commit: + raise SystemExit("downloaded hooks binary identity mismatch") + shutil.copy2(manifest_path, target / manifest_path.name) + binary.chmod(0o755) + PYINPUT + - name: Verify e2e full membership env: NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-full-list.json @@ -918,6 +994,10 @@ jobs: # extend that filter, never add ad-hoc e2e jobs here. Reuses the downloaded # debug binary; each test spawns its own rustfs server on a random port. - name: Run e2e full suite + env: + RUSTFS_E2E_STARTUP_CAS_BINARY: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs + RUSTFS_E2E_STARTUP_CAS_BUILD_MANIFEST: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json + RUSTFS_E2E_STARTUP_CAS_ARTIFACT_DIR: ${{ runner.temp }}/rustfs-startup-cas-evidence run: cargo nextest run --profile e2e-full -p e2e_test - name: Upload junit @@ -930,6 +1010,17 @@ jobs: ${{ runner.temp }}/rustfs-e2e-full-list.json retention-days: 7 + - name: Upload startup CAS evidence + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: fresh-startup-cas-evidence-${{ github.run_number }} + path: | + ${{ runner.temp }}/rustfs-startup-cas-evidence + ${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json + if-no-files-found: warn + retention-days: 7 + e2e-tests-rio-v2: name: End-to-End Tests (rio-v2) # Inherits the schedule/dispatch-only gate through needs: on every other diff --git a/Cargo.lock b/Cargo.lock index a9a0d63a7..d68959ecc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4057,6 +4057,7 @@ dependencies = [ "sha1 0.11.0", "sha2 0.11.0", "suppaftp", + "tempfile", "time", "tokio", "tokio-stream", diff --git a/crates/e2e_test/Cargo.toml b/crates/e2e_test/Cargo.toml index ac68cc646..70d1bbeb9 100644 --- a/crates/e2e_test/Cargo.toml +++ b/crates/e2e_test/Cargo.toml @@ -144,3 +144,6 @@ russh = { workspace = true, features = ["serde"] } russh-sftp = { workspace = true } zip.workspace = true clap = { workspace = true, features = ["derive", "env"] } + +[dev-dependencies] +tempfile.workspace = true diff --git a/crates/e2e_test/src/distributed_startup_regression_test.rs b/crates/e2e_test/src/distributed_startup_regression_test.rs index ff2da755b..c76aadc28 100644 --- a/crates/e2e_test/src/distributed_startup_regression_test.rs +++ b/crates/e2e_test/src/distributed_startup_regression_test.rs @@ -195,4 +195,1198 @@ mod tests { info!("RT-10c PASS: bucket visible from all 4 nodes"); Ok(()) } + + /// A real elected CAS must cross a signed peer RPC while the receiver still + /// holds its Bootstrap capability, before IAM installs any AppContext. + #[tokio::test] + async fn test_fresh_four_node_bootstrap_metadata_cas() -> TestResult { + use futures::FutureExt; + use std::path::PathBuf; + + init_logging(); + let nonce = uuid::Uuid::new_v4().to_string(); + let artifact = std::env::var_os("RUSTFS_E2E_STARTUP_CAS_ARTIFACT_DIR") + .map(PathBuf::from) + .unwrap_or_else(std::env::temp_dir) + .join(format!("fresh-startup-cas-{nonce}")); + std::fs::create_dir_all(&artifact)?; + let binary_dir = tempfile::tempdir()?; + let binary = prepare_startup_cas_binary(binary_dir.path(), &artifact)?; + probe_startup_cas_binary(&binary, &nonce, &artifact).await?; + + let mut cluster = RustFSTestClusterEnvironment::new(4).await?; + let mut logs = Vec::new(); + let mut disks = Vec::new(); + let mut endpoints = Vec::new(); + let mut releases = StartupCasReleases(Vec::new()); + for i in 0..4 { + let disk = PathBuf::from(&cluster.nodes[i].data_dir); + assert!(std::fs::read_dir(&disk)?.next().is_none(), "node {i} must start with an empty disk"); + let log = artifact.join(format!("node-{i}.log")); + let release = artifact.join(format!("release-{i}")); + cluster.set_node_capture_log_path(i, log.to_string_lossy())?; + cluster.set_node_env(i, "RUSTFS_E2E_STARTUP_CAS_RELEASE", release.to_string_lossy())?; + endpoints.push(format!("http://{}{}", cluster.nodes[i].address, cluster.nodes[i].data_dir)); + disks.push(disk); + logs.push(log); + releases.0.push(release); + } + assert!( + cluster.rustfs_volumes_arg().starts_with(&endpoints[0]), + "node 0 must own the elected first endpoint" + ); + cluster.set_env("RUSTFS_E2E_STARTUP_CAS_NONCE", &nonce); + cluster.set_env("RUSTFS_OBS_LOG_DIRECTORY", ""); + cluster.set_env("RUSTFS_OBS_LOG_STDOUT_ENABLED", "true"); + cluster.set_env("RUST_LOG", "rustfs=info,rustfs_ecstore=trace"); + for key in [ + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "http_proxy", + "https_proxy", + "all_proxy", + ] { + cluster.set_env(key, ""); + } + for key in ["NO_PROXY", "no_proxy"] { + cluster.set_env(key, "127.0.0.1,localhost"); + } + + let attempt = tokio::time::timeout( + Duration::from_secs(240), + std::panic::AssertUnwindSafe(async { + let mut startup = Box::pin(cluster.start_with_binary(&binary)); + let mut controller = Box::pin(wait_startup_cas(&logs, &disks, &endpoints, &nonce, &artifact)); + let (observed, startup_finished) = tokio::select! { + observed = tokio::time::timeout(Duration::from_secs(120), &mut controller) => { + (observed.map_err(std::io::Error::other).and_then(|result| result), false) + } + result = &mut startup => { + let error = match result { + Ok(()) => "startup returned before the unreleased CAS gates".to_owned(), + Err(error) => error.to_string(), + }; + // Preserve a real pool.bin rejection instead of relabeling + // an earlier identity failure or generic readiness timeout. + let observed = match tokio::time::timeout(Duration::from_secs(5), &mut controller).await { + Ok(Err(error)) => Err(error), + _ => Err(std::io::Error::other(format!("PRECONDITION: startup ended without causal proof: {error}"))), + }; + (observed, true) + } + }; + let release_result = releases.release(); + let drained = if startup_finished { + Ok(()) + } else { + let deadline = if observed.is_ok() { 60 } else { 5 }; + tokio::time::timeout(Duration::from_secs(deadline), &mut startup) + .await + .map_err(std::io::Error::other) + .map_err(|error| -> Box { error.into() }) + .and_then(|result| result) + }; + drop(startup); + observed?; + release_result?; + drained?; + for (i, node) in cluster.nodes.iter().enumerate() { + let pid = node + .process + .as_ref() + .ok_or_else(|| std::io::Error::other("missing child process"))? + .id(); + let records = startup_cas_log(&logs[i])?; + assert!( + records + .iter() + .any(|r| r["kind"] == "observer-ready" && r["nonce"] == nonce && r["pid"] == pid), + "node {i} observation must belong to the actual harness child" + ); + } + let bucket = format!("fresh-cas-{nonce}"); + tokio::time::timeout(Duration::from_secs(10), cluster.create_test_bucket(&bucket)) + .await + .map_err(std::io::Error::other)??; + for (i, client) in cluster.create_all_clients()?.iter().enumerate() { + let key = format!("node-{i}"); + let body = format!("fresh four-node body {i} {nonce}").into_bytes(); + tokio::time::timeout(Duration::from_secs(10), async { + client + .put_object() + .bucket(&bucket) + .key(&key) + .body(ByteStream::from(body.clone())) + .send() + .await?; + let received = client + .get_object() + .bucket(&bucket) + .key(&key) + .send() + .await? + .body + .collect() + .await? + .into_bytes(); + assert_eq!(received.as_ref(), body, "node {i} must return the complete object body"); + Ok::<_, Box>(()) + }) + .await + .map_err(std::io::Error::other)??; + } + Ok::<(), Box>(()) + }) + .catch_unwind(), + ) + .await; + // Drop the borrowed startup future before stopping its child processes. + // All logs are outside the cluster directory which Drop removes. + let release_result = releases.release(); + let pids: Vec<_> = cluster + .nodes + .iter() + .map(|node| node.process.as_ref().map(std::process::Child::id)) + .collect(); + let process_record = serde_json::to_vec(&pids) + .map_err(std::io::Error::other) + .and_then(|bytes| std::fs::write(artifact.join("processes.json"), bytes)); + cluster.stop(); + eprintln!("fresh startup CAS evidence: {}", artifact.display()); + match attempt { + Ok(Ok(result)) => { + result?; + release_result?; + process_record?; + Ok(()) + } + Ok(Err(panic)) => std::panic::resume_unwind(panic), + Err(error) => Err(std::io::Error::other(format!("fresh startup CAS fixture deadline: {error}")).into()), + } + } + + struct StartupCasReleases(Vec); + + impl StartupCasReleases { + fn release(&mut self) -> std::io::Result<()> { + let mut failure = None; + for path in &self.0 { + if let Err(error) = std::fs::write(path, b"release") { + failure.get_or_insert(error); + } + } + failure.map_or(Ok(()), Err) + } + } + + impl Drop for StartupCasReleases { + fn drop(&mut self) { + let _ = self.release(); + } + } + + fn startup_cas_sha256(path: &std::path::Path) -> std::io::Result { + use sha2::{Digest, Sha256}; + use std::io::Read; + let mut file = std::fs::File::open(path)?; + let mut hash = Sha256::new(); + let mut buf = [0; 65536]; + loop { + let len = file.read(&mut buf)?; + if len == 0 { + break; + } + hash.update(&buf[..len]); + } + Ok(rustfs_utils::crypto::hex(hash.finalize())) + } + + fn startup_cas_git(args: &[&str]) -> std::io::Result { + let result = std::process::Command::new("git") + .args(args) + .current_dir(crate::common::workspace_root()) + .output()?; + if !result.status.success() { + return Err(std::io::Error::other("cannot verify startup fixture checkout identity")); + } + String::from_utf8(result.stdout) + .map(|value| value.trim().to_owned()) + .map_err(std::io::Error::other) + } + + fn prepare_startup_cas_binary(dir: &std::path::Path, artifact: &std::path::Path) -> std::io::Result { + use serde_json::Value; + use std::path::PathBuf; + let explicit = std::env::var_os("RUSTFS_E2E_STARTUP_CAS_BINARY") + .or_else(|| std::env::var_os("CARGO_BIN_EXE_rustfs")) + .ok_or_else(|| { + std::io::Error::other("PRECONDITION: provide the existing hooks binary; this fixture never invokes Cargo") + })?; + let binary = std::fs::canonicalize(explicit)?; + if let Some(other) = std::env::var_os("CARGO_BIN_EXE_rustfs") { + if binary != std::fs::canonicalize(other)? { + return Err(std::io::Error::other("PRECONDITION: conflicting startup binary paths")); + } + } + let manifest_path = std::env::var_os("RUSTFS_E2E_STARTUP_CAS_BUILD_MANIFEST") + .ok_or_else(|| std::io::Error::other("PRECONDITION: missing hooks binary build manifest"))?; + let manifest_bytes = std::fs::read(manifest_path)?; + let manifest: Value = serde_json::from_slice(&manifest_bytes)?; + std::fs::write(artifact.join("binary-build.json"), &manifest_bytes)?; + let checkout = crate::common::workspace_root(); + let sha = startup_cas_sha256(&binary)?; + let valid = manifest["schema"] == 1 + && manifest["clean_before"] == true + && manifest["clean_after"] == true + && env!("RUSTFS_E2E_BUILD_DIRTY") == "false" + && manifest["commit"] == env!("RUSTFS_E2E_BUILD_COMMIT") + && manifest["commit"] == startup_cas_git(&["rev-parse", "HEAD"])? + && manifest["tree"] == startup_cas_git(&["rev-parse", "HEAD^{tree}"])? + && startup_cas_git(&["status", "--porcelain", "--untracked-files=normal"])?.is_empty() + && manifest["lock_git_blob"] == env!("RUSTFS_E2E_BUILD_LOCK") + && manifest["lock_git_blob"] == startup_cas_git(&["hash-object", "Cargo.lock"])? + && manifest["lock_sha256"] == startup_cas_sha256(&checkout.join("Cargo.lock"))? + && manifest["binary_sha256"] == sha + && manifest["target"] == env!("RUSTFS_E2E_BUILD_TARGET") + && manifest["profile"] == "debug" + && manifest["features"] + .as_array() + .is_some_and(|features| features.iter().any(|f| f == "e2e-test-hooks")) + && manifest["argv"] + .as_array() + .is_some_and(|argv| argv.iter().any(|arg| arg == "--features") && argv.iter().any(|arg| arg == "e2e-test-hooks")) + && manifest["rustc_verbose"].as_str().is_some_and(|value| !value.is_empty()) + && manifest["build_flags"].is_object(); + if !valid { + return Err(std::io::Error::other( + "PRECONDITION: hooks binary identity does not match this clean test checkout", + )); + } + let target = dir.join(format!("rustfs{}", std::env::consts::EXE_SUFFIX)); + std::fs::copy(&binary, &target)?; + if startup_cas_sha256(&target)? != sha { + return Err(std::io::Error::other("PRECONDITION: binary changed during fixture copy")); + } + std::fs::write( + artifact.join("runner-build.json"), + serde_json::to_vec(&serde_json::json!({ + "commit": env!("RUSTFS_E2E_BUILD_COMMIT"), "lock_git_blob": env!("RUSTFS_E2E_BUILD_LOCK"), + "target": env!("RUSTFS_E2E_BUILD_TARGET"), "profile": env!("RUSTFS_E2E_BUILD_PROFILE"), + "features": env!("RUSTFS_E2E_BUILD_FEATURES"), "binary_sha256": sha, + "binary": PathBuf::from(&target), + }))?, + )?; + Ok(target) + } + + async fn probe_startup_cas_binary(binary: &std::path::Path, nonce: &str, artifact: &std::path::Path) -> std::io::Result<()> { + struct Probe(std::process::Child); + impl Drop for Probe { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } + } + let path = artifact.join("capability-probe.log"); + let log = std::fs::File::create(&path)?; + let mut child = Probe( + std::process::Command::new(binary) + .arg("--help") + .env("RUSTFS_E2E_STARTUP_CAS_PROBE", nonce) + .stdout(log.try_clone()?) + .stderr(log) + .spawn()?, + ); + let status = tokio::time::timeout(Duration::from_secs(10), async { + loop { + if let Some(status) = child.0.try_wait()? { + return Ok::<_, std::io::Error>(status); + } + sleep(Duration::from_millis(25)).await; + } + }) + .await + .map_err(|_| std::io::Error::other("PRECONDITION: binary capability probe timed out"))??; + let records = startup_cas_log(&path)?; + let matching: Vec<_> = records + .iter() + .filter(|r| r["nonce"] == nonce && r["kind"] == "capability" && r["schema"] == "fresh-startup-cas/v1") + .collect(); + if !status.success() || matching.len() != 1 { + return Err(std::io::Error::other( + "PRECONDITION: binary lacks the startup CAS hooks; no cluster was started", + )); + } + Ok(()) + } + + fn startup_cas_log(path: &std::path::Path) -> std::io::Result> { + let text = match std::fs::read_to_string(path) { + Ok(text) => text, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(error), + }; + if text.len() > 64 * 1024 * 1024 { + return Err(std::io::Error::other("startup observation log exceeds fixture bound")); + } + let mut records = Vec::new(); + for line in text.split_inclusive('\n').filter_map(|line| line.strip_suffix('\n')) { + if let Some(json) = line.strip_prefix("RUSTFS_E2E_STARTUP_CAS ") { + records.push(serde_json::from_str(json)?); + } else if let Ok(json) = serde_json::from_str::(line) { + // The existing remote-disk trace is JSON with flattened fields. + records.push(json); + } + } + Ok(records) + } + + fn startup_cas_remote_matches(sender: &[serde_json::Value], receiver: &serde_json::Value) -> usize { + sender + .iter() + .filter(|event| { + event["target"] + .as_str() + .is_some_and(|target| target.split("::").eq(["rustfs_ecstore", "cluster", "rpc", "remote_disk"])) + && event["op"] == "rename_data" + && event["state"] == "started" + && event["endpoint"] == receiver["disk"] + && ["src_volume", "src_path", "dst_volume", "dst_path"] + .iter() + .all(|key| event[*key] == receiver[*key]) + }) + .count() + } + + async fn wait_startup_cas( + paths: &[std::path::PathBuf], + disks: &[std::path::PathBuf], + endpoints: &[String], + nonce: &str, + artifact: &std::path::Path, + ) -> std::io::Result<()> { + loop { + let logs: Vec<_> = paths + .iter() + .map(|path| startup_cas_log(path)) + .collect::>()?; + let events: Vec> = logs + .iter() + .map(|records| records.iter().filter(|r| r["nonce"] == nonce).collect()) + .collect(); + let source = &events[0]; + for (node, events) in events.iter().enumerate() { + for event in events.iter().filter(|r| r["kind"] == "cas") { + if node != 0 { + return Err(std::io::Error::other(format!( + "PRECONDITION: non-elected node {node} executed CAS: {event}" + ))); + } + if event["ok"] == false { + let object = event["object"].as_str().unwrap_or_default(); + let receiver = logs.iter().skip(1).flat_map(|records| records.iter()).find(|r| { + r["nonce"] == nonce + && r["kind"] == "receiver" + && r["dst_path"] == object + && r["ok"] == false + && r["target"] == "bootstrap" + && startup_cas_remote_matches(&logs[0], r) == 1 + }); + if let Some(receiver) = receiver { + let identity_ok = source + .iter() + .take_while(|r| !std::ptr::eq(**r, *event)) + .any(|r| r["kind"] == "cas" && r["phase"] == "identity_cas" && r["ok"] == true); + let class = if object == "pool.bin" && identity_ok { + "POOL_BIN_CAUSAL_REJECTION" + } else { + "PRECONDITION_IDENTITY_OR_STARTUP_FAILURE" + }; + std::fs::write( + artifact.join("cas-rejection.json"), + serde_json::to_vec_pretty(&serde_json::json!({ + "class": class, "sender": event, "receiver": receiver, + }))?, + )?; + return Err(std::io::Error::other(format!("{class}: sender={event}; receiver={receiver}"))); + } + } + } + } + if events + .iter() + .all(|records| records.iter().any(|r| r["kind"] == "gate" && r["slot_installed"] == false)) + { + let mut pids = std::collections::HashSet::new(); + for records in &events { + let ready: Vec<_> = records.iter().filter(|r| r["kind"] == "observer-ready").collect(); + if ready.len() != 1 + || !pids.insert( + ready[0]["pid"] + .as_u64() + .ok_or_else(|| std::io::Error::other("missing child PID"))?, + ) + { + return Err(std::io::Error::other( + "PRECONDITION: four independent observer-capable child processes required", + )); + } + if records.iter().any(|r| r["pid"] != ready[0]["pid"]) { + return Err(std::io::Error::other("PRECONDITION: observation process mismatch")); + } + } + let prepare: Vec<_> = source + .iter() + .filter(|r| r["kind"] == "cas" && r["phase"] == "prepare_cas" && r["object"] == "pool.bin") + .collect(); + let commit: Vec<_> = source + .iter() + .filter(|r| r["kind"] == "cas" && r["phase"] == "commit_cas" && r["object"] == "pool.bin") + .collect(); + if prepare.len() != 1 || commit.len() != 1 { + return Err(std::io::Error::other("PRECONDITION: startup CAS evidence is absent or ambiguous")); + } + let (prepare, commit) = (*prepare[0], *commit[0]); + for cas in [prepare, commit] { + if cas["ok"] != true + || cas["tail_drained"] != true + || cas["no_lock"] != true + || cas["etag"].as_str().is_none_or(str::is_empty) + { + return Err(std::io::Error::other(format!("actual startup CAS did not complete: {cas}"))); + } + } + if prepare["if_none_match"] != "*" + || !prepare["if_match"].is_null() + || commit["if_match"] != prepare["etag"] + || !commit["if_none_match"].is_null() + || prepare["etag"] == commit["etag"] + || prepare["payload_sha256"] == commit["payload_sha256"] + { + return Err(std::io::Error::other( + "actual prepare/commit conditional revisions do not form the fresh CAS chain", + )); + } + let confirmed = source.iter().find(|r| { + r["kind"] == "confirmed" + && r["payload_sha256"] == commit["payload_sha256"] + && r["generation"].as_u64().is_some_and(|g| g > 0) + }); + if confirmed.is_none() { + return Err(std::io::Error::other("actual quorum reload did not confirm the committed payload")); + } + for node in 1..4 { + let mut accepted = Vec::new(); + for cas in [prepare, commit] { + let matching: Vec<_> = events[node] + .iter() + .filter(|r| { + r["kind"] == "receiver" + && r["dst_volume"] == ".rustfs.sys" + && r["dst_path"] == "pool.bin" + && r["etag"] == cas["etag"] + }) + .collect(); + if matching.len() != 1 { + break; + } + let received = *matching[0]; + if received["ok"] != true + || received["target"] != "bootstrap" + || received["disk"] != endpoints[node] + || received["body_sha256"].as_str().is_none_or(|hash| hash.len() != 64) + || startup_cas_remote_matches(&logs[0], received) != 1 + { + break; + } + accepted.push(received); + } + if accepted.len() == 2 { + let raw = std::fs::read(disks[node].join(".rustfs.sys/pool.bin/xl.meta"))?; + std::fs::write(artifact.join(format!("node-{node}-committed-xl.meta")), &raw)?; + let file_info = rustfs_filemeta::get_file_info( + &raw, + ".rustfs.sys", + "pool.bin", + "", + rustfs_filemeta::FileInfoOpts { + data: false, + include_free_versions: false, + include_part_checksums: false, + }, + ) + .map_err(std::io::Error::other)?; + if raw.is_empty() + || file_info.metadata.get("etag").map(String::as_str) != commit["etag"].as_str() + || file_info + .mod_time + .map(|time| time.unix_timestamp_nanos().to_string()) + .as_deref() + != accepted[1]["mod_time"].as_str() + || accepted[1]["mod_time"].is_null() + { + return Err(std::io::Error::other( + "latest physical target metadata does not match the accepted commit", + )); + } + std::fs::write( + artifact.join("cas-proof.json"), + serde_json::to_vec_pretty(&serde_json::json!({ + "sender": 0, "receiver": node, "prepare": prepare, "commit": commit, "accepted": accepted, "confirmed": confirmed, + }))?, + )?; + return Ok(()); + } + } + // A started trace may still be flushing asynchronously. Keep + // waiting for its real tuple; the enclosing deadline is finite. + } + sleep(Duration::from_millis(25)).await; + } + } + + /// A stopped, initialized pool loses only its pool.bin object replicas. + /// Restart must repair both pools through their actual conditional writes. + #[tokio::test] + async fn test_two_pool_restart_repairs_missing_pool_metadata_via_bootstrap_cas() -> TestResult { + use crate::common::ClusterTopology; + use futures::FutureExt; + use std::path::PathBuf; + + init_logging(); + let run = uuid::Uuid::new_v4().to_string(); + let artifact = std::env::var_os("RUSTFS_E2E_STARTUP_CAS_ARTIFACT_DIR") + .map(PathBuf::from) + .unwrap_or_else(std::env::temp_dir) + .join(format!("repair-startup-cas-{run}")); + std::fs::create_dir_all(&artifact)?; + let binary_dir = tempfile::tempdir()?; + let binary = prepare_startup_cas_binary(binary_dir.path(), &artifact)?; + probe_startup_cas_binary(&binary, &run, &artifact).await?; + let mut owned = RepairStartupCluster(Some( + RustFSTestClusterEnvironment::with_topology(ClusterTopology::per_node_pools(2, vec![vec![0], vec![1]])).await?, + )); + let cluster = owned.0.as_mut().expect("fixture owns its cluster"); + assert_eq!(cluster.nodes.len(), 2); + for (pool, node) in cluster.nodes.iter().enumerate() { + assert_eq!(node.pool_idx, pool); + assert_eq!(node.data_dirs.len(), 2); + } + let volumes = cluster.rustfs_volumes_arg(); + assert_eq!(volumes.split_whitespace().count(), 2); + cluster.set_env("RUSTFS_OBS_LOG_DIRECTORY", ""); + cluster.set_env("RUSTFS_OBS_LOG_STDOUT_ENABLED", "true"); + cluster.set_env("RUST_LOG", "rustfs=info,rustfs_ecstore=trace"); + for key in [ + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "http_proxy", + "https_proxy", + "all_proxy", + ] { + cluster.set_env(key, ""); + } + for key in ["NO_PROXY", "no_proxy"] { + cluster.set_env(key, "127.0.0.1,localhost"); + } + let attempt = tokio::time::timeout( + Duration::from_secs(600), + std::panic::AssertUnwindSafe(async { + // Preserve pool0's exact command line when it later joins pool1. + // Fresh pools with different format leaders cannot yet combine + // their bootstrap authority, so prepare through normal expansion. + let first_pool = volumes.split_whitespace().next().expect("pool0 volume argument"); + cluster.set_env("RUSTFS_VOLUMES", first_pool); + let mut seed = RepairStartupPhase::new(cluster, &artifact, "seed", 1)?; + let seeded = run_repair_startup_phase(cluster, &binary, &mut seed, None, true).await?; + let stopped = stop_repair_cluster(cluster)?; + assert_eq!(stopped.len(), 1, "the seed child must be reaped before expansion"); + std::fs::write(artifact.join("seed-stopped.json"), serde_json::to_vec_pretty(&stopped)?)?; + let seed_formats = cluster.nodes[0] + .data_dirs + .iter() + .map(|disk| std::fs::read(std::path::Path::new(disk).join(".rustfs.sys/format.json"))) + .collect::>>()?; + cluster.extra_env.retain(|(key, _)| key != "RUSTFS_VOLUMES"); + assert_eq!(cluster.rustfs_volumes_arg(), volumes); + let mut first = RepairStartupPhase::new(cluster, &artifact, "expansion", 2)?; + assert_ne!(seed.nonce, first.nonce); + let previous = run_repair_startup_phase(cluster, &binary, &mut first, Some(&seeded), true).await?; + for (disk, expected) in cluster.nodes[0].data_dirs.iter().zip(seed_formats) { + assert_eq!(std::fs::read(std::path::Path::new(disk).join(".rustfs.sys/format.json"))?, expected); + } + let bucket = format!("repair-cas-{run}"); + let key = "preserved/body"; + let body = vec![0x6bu8; 256 * 1024]; + tokio::time::timeout(Duration::from_secs(20), async { + cluster.create_test_bucket(&bucket).await?; + cluster.create_all_clients()?[0] + .put_object() + .bucket(&bucket) + .key(key) + .body(ByteStream::from(body.clone())) + .send() + .await?; + repair_full_get(cluster, &bucket, key, &body).await + }) + .await + .map_err(std::io::Error::other)??; + + // Retain each Child until wait confirms exit; only then mutate + // the stopped fixture's exact pool1 object subtrees. + let stopped = stop_repair_cluster(cluster)?; + assert_eq!(stopped.len(), 2, "both initial children must be reaped before disk mutation"); + std::fs::write(artifact.join("expansion-stopped.json"), serde_json::to_vec_pretty(&stopped)?)?; + let mut before = Vec::new(); + for (pool, node) in cluster.nodes.iter().enumerate() { + for disk in &node.data_dirs { + let root = std::fs::canonicalize(disk)?; + assert!(root.starts_with(std::fs::canonicalize(&cluster.temp_dir)?)); + let snapshot = repair_disk_snapshot(&root)?; + assert!(snapshot.contains_key(std::path::Path::new(".rustfs.sys/format.json"))); + assert!(snapshot.contains_key(std::path::Path::new(".rustfs.sys/pool.bin.identity/xl.meta"))); + assert!(snapshot.contains_key(std::path::Path::new(".rustfs.sys/pool.bin/xl.meta"))); + before.push((pool, root, snapshot)); + } + } + std::fs::write(artifact.join("before-removal.json"), serde_json::to_vec_pretty(&before)?)?; + for (pool, root, _) in &before { + if *pool == 1 { + let object = root.join(".rustfs.sys/pool.bin"); + assert_eq!(std::fs::canonicalize(&object)?, object, "no aliased deletion target"); + std::fs::remove_dir_all(&object)?; + assert!(!object.try_exists()?); + } + } + for (pool, root, snapshot) in &before { + let mut expected = snapshot.clone(); + if *pool == 1 { + expected.retain(|path, _| !path.starts_with(".rustfs.sys/pool.bin")); + } + assert_eq!(repair_disk_snapshot(root)?, expected, "only pool1's complete pool.bin objects may change"); + } + assert_eq!(cluster.rustfs_volumes_arg(), volumes, "repair reuses identical topology, ports and roots"); + let mut restart = RepairStartupPhase::new(cluster, &artifact, "repair", 2)?; + assert_ne!(restart.nonce, first.nonce); + let repaired = run_repair_startup_phase(cluster, &binary, &mut restart, Some(&previous), false).await?; + tokio::time::timeout(Duration::from_secs(20), repair_full_get(cluster, &bucket, key, &body)) + .await + .map_err(std::io::Error::other)??; + std::fs::write( + artifact.join("repair-proof.json"), + serde_json::to_vec_pretty(&serde_json::json!({ + "seed": seeded, "expansion": previous, "repair": repaired, "volumes": volumes, "seed_volumes": first_pool, + "negative_control": "NOT_RUN", "body_length": body.len(), "full_get_nodes": [0, 1], + }))?, + )?; + Ok::<(), Box>(()) + }) + .catch_unwind(), + ) + .await; + // Borrowed startup futures and their release guards have dropped here. + // Attempt every child even when an earlier wait or assertion failed. + let stopped = stop_repair_cluster(cluster); + eprintln!("two-pool repair CAS evidence: {}", artifact.display()); + let cleanup_receipt = match &stopped { + Ok(receipts) => serde_json::json!({"ok": true, "waited": receipts}), + Err(error) => serde_json::json!({"ok": false, "error": error.to_string()}), + }; + let cleanup_recorded = serde_json::to_vec_pretty(&cleanup_receipt) + .map_err(std::io::Error::other) + .and_then(|bytes| std::fs::write(artifact.join("final-cleanup.json"), bytes)); + if let Err(error) = &cleanup_recorded { + eprintln!("repair cleanup receipt could not be saved: {error}"); + } + match attempt { + Ok(Ok(result)) => { + result?; + let stopped = stopped?; + cleanup_recorded?; + std::fs::write(artifact.join("repair-stopped.json"), serde_json::to_vec_pretty(&stopped)?)?; + Ok(()) + } + Ok(Err(panic)) => std::panic::resume_unwind(panic), + Err(error) => Err(std::io::Error::other(format!("two-pool repair fixture deadline: {error}")).into()), + } + } + + // Unlike the general harness stop_node, keep a failed Child wait attached + // so Drop can retry without deleting a possibly active disk directory. + struct RepairStartupCluster(Option); + + impl Drop for RepairStartupCluster { + fn drop(&mut self) { + if let Some(mut cluster) = self.0.take() { + if let Err(error) = stop_repair_cluster(&mut cluster) { + eprintln!("repair child cleanup failed; preserving {}: {error}", cluster.temp_dir); + } + if cluster.nodes.iter().any(|node| node.process.is_some()) { + std::mem::forget(cluster); + } + } + } + } + + fn stop_repair_cluster(cluster: &mut RustFSTestClusterEnvironment) -> std::io::Result> { + let mut stopped = Vec::new(); + let mut failure = None; + for (node, state) in cluster.nodes.iter_mut().enumerate() { + let Some(child) = state.process.as_mut() else { continue }; + let pid = child.id(); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + let waited = (|| { + if child.try_wait()?.is_none() { + // Exit can race kill; the actual wait below decides whether + // the child is gone, rather than a successful signal alone. + let _ = child.kill(); + } + loop { + if child.try_wait()?.is_some() { + return child.wait(); + } + if std::time::Instant::now() >= deadline { + return Err(std::io::Error::other(format!("node {node} pid {pid} did not exit"))); + } + std::thread::sleep(Duration::from_millis(10)); + } + })(); + match waited { + Ok(status) => { + stopped.push(serde_json::json!({"node": node, "pid": pid, "status": status.to_string()})); + state.process = None; + } + Err(error) => { + failure.get_or_insert(error); + } + } + } + failure.map_or(Ok(stopped), Err) + } + + fn repair_disk_snapshot( + root: &std::path::Path, + ) -> std::io::Result> { + let mut snapshot = std::collections::BTreeMap::new(); + let mut pending = vec![std::path::PathBuf::new()]; + while let Some(relative) = pending.pop() { + if relative.components().count() > 64 || snapshot.len() > 10_000 { + return Err(std::io::Error::other("repair fixture snapshot exceeds its bound")); + } + for entry in std::fs::read_dir(root.join(&relative))? { + let entry = entry?; + let path = relative.join(entry.file_name()); + let metadata = std::fs::symlink_metadata(entry.path())?; + if metadata.is_dir() { + snapshot.insert(path.clone(), (0, "directory".to_owned())); + pending.push(path); + } else if metadata.is_file() { + snapshot.insert(path, (metadata.len(), startup_cas_sha256(&entry.path())?)); + } else { + return Err(std::io::Error::other("repair fixture contains a symlink or special file")); + } + } + } + Ok(snapshot) + } + + async fn repair_full_get(cluster: &RustFSTestClusterEnvironment, bucket: &str, key: &str, body: &[u8]) -> TestResult { + for (node, client) in cluster.create_all_clients()?.iter().enumerate() { + let received = client + .get_object() + .bucket(bucket) + .key(key) + .send() + .await? + .body + .collect() + .await? + .into_bytes(); + assert_eq!(received.as_ref(), body, "node {node} must return every preserved object byte"); + } + Ok(()) + } + + struct RepairStartupPhase { + nonce: String, + artifact: std::path::PathBuf, + logs: Vec, + disks: Vec>, + endpoints: Vec>, + releases: StartupCasReleases, + } + + impl RepairStartupPhase { + fn new( + cluster: &mut RustFSTestClusterEnvironment, + artifact: &std::path::Path, + phase: &str, + pool_count: usize, + ) -> Result> { + assert!(matches!(pool_count, 1 | 2)); + let artifact = artifact.join(phase); + std::fs::create_dir(&artifact)?; + let nonce = uuid::Uuid::new_v4().to_string(); + cluster.set_env("RUSTFS_E2E_STARTUP_CAS_NONCE", &nonce); + let mut result = Self { + nonce, + artifact, + logs: Vec::new(), + disks: Vec::new(), + endpoints: Vec::new(), + releases: StartupCasReleases(Vec::new()), + }; + for node in 0..pool_count { + let log = result.artifact.join(format!("node-{node}.log")); + let release = result.artifact.join(format!("release-{node}")); + assert!(!release.try_exists()?, "each startup has a new unreleased gate"); + cluster.set_node_capture_log_path(node, log.to_string_lossy())?; + cluster.set_node_env(node, "RUSTFS_E2E_STARTUP_CAS_RELEASE", release.to_string_lossy())?; + result + .disks + .push(cluster.nodes[node].data_dirs.iter().map(std::path::PathBuf::from).collect()); + result.endpoints.push( + cluster.nodes[node] + .data_dirs + .iter() + .map(|disk| format!("http://{}{disk}", cluster.nodes[node].address)) + .collect(), + ); + result.logs.push(log); + result.releases.0.push(release); + } + Ok(result) + } + } + + async fn run_repair_startup_phase( + cluster: &mut RustFSTestClusterEnvironment, + binary: &std::path::Path, + phase: &mut RepairStartupPhase, + previous: Option<&serde_json::Value>, + topology_update: bool, + ) -> Result> { + let pool_count = phase.logs.len(); + let mut startup = Box::pin(async { + if pool_count == 1 { + cluster.start_node_from_binary(0, binary).await + } else { + cluster.start_with_binary(binary).await + } + }); + let mut observer = Box::pin(wait_repair_startup_cas(phase, previous, topology_update)); + let (observed, finished) = tokio::select! { + observed = tokio::time::timeout(Duration::from_secs(120), &mut observer) => { + (observed.map_err(std::io::Error::other).and_then(|result| result), false) + } + result = &mut startup => { + (Err(std::io::Error::other(format!("startup ended before the unreleased phase gates: {result:?}"))), true) + } + }; + drop(observer); + let released = phase.releases.release(); + let drained = if finished { + Ok(()) + } else { + tokio::time::timeout(Duration::from_secs(if observed.is_ok() { 60 } else { 5 }), &mut startup) + .await + .map_err(std::io::Error::other) + .map_err(|error| -> Box { error.into() }) + .and_then(|result| result) + }; + drop(startup); + let observed = observed?; + released?; + drained?; + for (node, process) in cluster.nodes.iter().take(pool_count).enumerate() { + assert_eq!(observed["pids"][node], process.process.as_ref().expect("live phase child").id()); + } + Ok(observed) + } + + async fn wait_repair_startup_cas( + phase: &RepairStartupPhase, + previous: Option<&serde_json::Value>, + topology_update: bool, + ) -> std::io::Result { + use serde_json::{Value, json}; + let pool_count = phase.logs.len(); + loop { + let logs = phase + .logs + .iter() + .map(|path| startup_cas_log(path)) + .collect::>>()?; + let events: Vec> = logs + .iter() + .map(|log| log.iter().filter(|event| event["nonce"] == phase.nonce).collect()) + .collect(); + if events + .iter() + .flat_map(|events| events.iter()) + .any(|event| event["kind"] == "cas" && event["ok"] == false) + { + return Err(std::io::Error::other("normal two-pool startup produced a failed CAS; inspect phase logs")); + } + if !events.iter().all(|events| { + events + .iter() + .any(|event| event["kind"] == "gate" && event["slot_installed"] == false) + }) { + sleep(Duration::from_millis(25)).await; + continue; + } + let mut pids = Vec::new(); + for records in &events { + let ready: Vec<_> = records.iter().filter(|event| event["kind"] == "observer-ready").collect(); + assert_eq!(ready.len(), 1, "one real process per phase log"); + assert!(ready[0]["pid"].as_u64().is_some()); + assert!(records.iter().all(|event| event["pid"] == ready[0]["pid"])); + pids.push(ready[0]["pid"].clone()); + } + if pool_count == 2 { + assert_ne!(pids[0], pids[1]); + } + let source = &events[0]; + let classified: Vec<_> = source + .iter() + .copied() + .filter(|event| event["kind"] == "startup-classifier") + .collect(); + assert_eq!(classified.len(), 1, "the normal startup must classify once, without retrying failures"); + let classified = classified[0]; + let attempt = &classified["attempt"]; + uuid::Uuid::parse_str(attempt.as_str().expect("real init attempt UUID")).expect("valid init attempt UUID"); + assert_eq!(classified["elected_writer"], true); + assert_eq!(classified["needs_repair"], previous.is_some()); + assert_eq!(classified["repair_write_safe"], true); + assert_eq!(classified["topology_update"], topology_update); + assert!( + events + .iter() + .skip(1) + .flatten() + .all(|event| event["kind"] != "cas" || event["object"] != "pool.bin"), + "only elected pool0 may persist pool.bin" + ); + let initial: Vec<_> = source + .iter() + .copied() + .filter(|event| { + event["kind"] == "replica-read" && event["startup_phase"] == "load" && event["attempt"] == *attempt + }) + .collect(); + assert_eq!(initial.len(), pool_count, "complete initial reads of every real pool"); + if pool_count == 2 { + assert_eq!(initial[0]["batch"], initial[1]["batch"]); + } + let mut prepares = Vec::new(); + let mut commits = Vec::new(); + for pool in 0..pool_count { + let read: Vec<_> = initial.iter().filter(|event| event["pool"] == pool).collect(); + assert_eq!(read.len(), 1); + let read = *read[0]; + if let Some(previous) = previous.filter(|_| pool == 0) { + assert_eq!(read["state"], "valid"); + assert_eq!(read["committed"], true); + for field in [ + "payload_sha256", + "raw_sha256", + "version", + "cluster_id", + "epoch", + "generation", + "transaction_id", + "etag", + ] { + assert_eq!(read[field], previous["replicas"][0][field], "pool0 baseline {field} is retained"); + } + } else { + assert_eq!(read["state"], "missing"); + assert_eq!(read["cas"], "missing"); + } + let mut pair = Vec::new(); + for stage in ["prepare_cas", "commit_cas"] { + let matching: Vec<_> = source + .iter() + .copied() + .filter(|event| { + event["kind"] == "cas" + && event["object"] == "pool.bin" + && event["phase"] == stage + && event["pool"] == pool + && event["attempt"] == *attempt + && event["startup_phase"] == "persist" + }) + .collect(); + assert_eq!(matching.len(), 1, "exactly one successful {stage} on actual pool {pool}"); + let cas = matching[0]; + assert_eq!(cas["ok"], true); + assert_eq!(cas["tail_drained"], true); + assert_eq!(cas["no_lock"], true); + assert!(cas["etag"].as_str().is_some_and(|value| !value.is_empty())); + assert!(cas["mod_time"].as_str().is_some_and(|value| !value.is_empty())); + pair.push(cas); + } + if previous.is_some() && pool == 0 { + assert_eq!(read["cas"], "existing"); + assert_eq!(pair[0]["if_match"], read["etag"]); + assert!(pair[0]["if_none_match"].is_null()); + } else { + assert_eq!(pair[0]["if_none_match"], "*"); + assert!(pair[0]["if_match"].is_null()); + } + assert_eq!(pair[1]["if_match"], pair[0]["etag"]); + assert!(pair[1]["if_none_match"].is_null()); + assert_ne!(pair[0]["etag"], pair[1]["etag"]); + assert_ne!(pair[0]["payload_sha256"], pair[1]["payload_sha256"]); + prepares.push(pair[0]); + commits.push(pair[1]); + } + assert_eq!( + source + .iter() + .filter(|event| event["kind"] == "cas" && event["object"] == "pool.bin") + .count(), + 2 * pool_count + ); + if pool_count == 2 { + assert_eq!(prepares[0]["payload_sha256"], prepares[1]["payload_sha256"]); + assert_eq!(commits[0]["payload_sha256"], commits[1]["payload_sha256"]); + } + let mut replicas = Vec::new(); + for pool in 0..pool_count { + let matching: Vec<_> = source + .iter() + .copied() + .filter(|event| { + event["kind"] == "replica-read" + && event["startup_phase"] == "persist" + && event["attempt"] == *attempt + && event["pool"] == pool + && event["payload_sha256"] == commits[pool]["payload_sha256"] + }) + .collect(); + assert_eq!(matching.len(), 1, "actual final complete decoded read for pool {pool}"); + let replica = matching[0]; + assert_eq!(replica["state"], "valid"); + assert_eq!(replica["committed"], true); + assert_eq!(replica["pool_count"], pool_count); + assert_eq!(replica["raw_sha256"], replica["payload_sha256"]); + assert_eq!(replica["etag"], commits[pool]["etag"]); + assert_eq!(replica["cas"], "existing"); + assert!( + replica["cluster_id"] + .as_str() + .is_some_and(|value| uuid::Uuid::parse_str(value).is_ok()) + ); + assert!( + replica["transaction_id"] + .as_str() + .is_some_and(|value| uuid::Uuid::parse_str(value).is_ok()) + ); + assert!(replica["epoch"].as_u64().is_some_and(|value| value > 0)); + let expected_generation = match previous { + Some(previous) => { + previous["replicas"][0]["generation"] + .as_u64() + .expect("previous decoded generation") + + 1 + } + None => 1, + }; + assert_eq!(replica["generation"], expected_generation); + if let Some(previous) = previous { + assert_eq!(replica["cluster_id"], previous["replicas"][0]["cluster_id"]); + assert_eq!(replica["epoch"], previous["replicas"][0]["epoch"]); + assert_ne!(replica["transaction_id"], previous["replicas"][0]["transaction_id"]); + } + replicas.push(replica); + } + for field in [ + "batch", + "version", + "cluster_id", + "epoch", + "generation", + "transaction_id", + "payload_sha256", + ] { + if pool_count == 2 { + assert_eq!(replicas[0][field], replicas[1][field], "same decoded final revision: {field}"); + } + } + let confirmed: Vec<_> = source + .iter() + .filter(|event| { + event["kind"] == "confirmed" + && event["attempt"] == *attempt + && event["payload_sha256"] == replicas[0]["payload_sha256"] + && event["generation"] == replicas[0]["generation"] + && event["transaction_id"] == replicas[0]["transaction_id"] + }) + .collect(); + assert_eq!(confirmed.len(), 1); + let mut receivers = Vec::new(); + for drive in 0..phase.disks.get(1).map_or(0, Vec::len) { + for cas in [prepares[1], commits[1]] { + let matching: Vec<_> = events[1] + .iter() + .copied() + .filter(|event| { + event["kind"] == "receiver" + && event["disk"] == phase.endpoints[1][drive] + && event["dst_volume"] == ".rustfs.sys" + && event["dst_path"] == "pool.bin" + && event["etag"] == cas["etag"] + }) + .collect(); + assert!(matching.len() <= 1, "one actual receiver per disk/CAS"); + if let Some(receiver) = matching.first() { + assert_eq!(receiver["ok"], true); + assert_eq!(receiver["target"], "bootstrap"); + assert_eq!(receiver["mod_time"], cas["mod_time"]); + assert!(receiver["body_sha256"].as_str().is_some_and(|hash| hash.len() == 64)); + if startup_cas_remote_matches(&logs[0], receiver) == 1 { + receivers.push(*receiver); + } + } + } + } + if receivers.len() != 2 * phase.disks.get(1).map_or(0, Vec::len) { + // Remote started tracing may flush after direct receiver JSON. + sleep(Duration::from_millis(25)).await; + continue; + } + for (pool, disks) in phase.disks.iter().enumerate() { + for (drive, disk) in disks.iter().enumerate() { + let raw = std::fs::read(disk.join(".rustfs.sys/pool.bin/xl.meta"))?; + let latest = rustfs_filemeta::get_file_info( + &raw, + ".rustfs.sys", + "pool.bin", + "", + rustfs_filemeta::FileInfoOpts { + data: false, + include_free_versions: false, + include_part_checksums: false, + }, + ) + .map_err(std::io::Error::other)?; + assert!(!raw.is_empty()); + assert_eq!(latest.metadata.get("etag").map(String::as_str), commits[pool]["etag"].as_str()); + assert_eq!( + latest.mod_time.map(|time| time.unix_timestamp_nanos().to_string()).as_deref(), + commits[pool]["mod_time"].as_str() + ); + std::fs::write(phase.artifact.join(format!("pool-{pool}-drive-{drive}-latest-xl.meta")), raw)?; + } + } + let proof = json!({"nonce": phase.nonce, "pids": pids, "classifier": classified, "initial": initial, "prepare": prepares, "commit": commits, "replicas": replicas, "receivers": receivers, "confirmed": confirmed}); + std::fs::write(phase.artifact.join("cas-proof.json"), serde_json::to_vec_pretty(&proof)?)?; + return Ok(proof); + } + } } diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index 4925b283e..0b9d2871c 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -118,6 +118,8 @@ hotpath-cpu = [ # injection, xl.meta transition assertions) via `api::tier::test_util`. # Enable only from `[dev-dependencies]` (rustfs/backlog#1148 ilm-6). test-util = [] +# Observes real startup CAS only in the dedicated E2E binary. +e2e-test-hooks = [] [dependencies] hotpath.workspace = true diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index e0cc06d26..04c1bfe94 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -368,6 +368,8 @@ pub mod data_usage { pub mod disk { pub use crate::disk::disk_store::get_object_disk_read_timeout; pub use crate::disk::local::ScanGuard; + #[cfg(all(feature = "test-util", not(windows)))] + pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage}; pub use crate::disk::{ BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, ConditionalFileUpdate, DeleteOptions, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, @@ -544,8 +546,8 @@ pub mod storage { pub use crate::core::pools::HealLifecycleExpiryContext; pub use crate::store::HealWalkVersion; pub use crate::store::{ - ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk, all_local_disk_path, - find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients, + BootstrapLocalTarget, ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk, + all_local_disk_path, find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map, prewarm_local_disk_id_map_with_instance_ctx, }; } diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index c4235bd40..39463a3ab 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -5108,7 +5108,49 @@ async fn read_pool_meta_replicas(pools: Vec>, no_lock: bool) -> Vec "missing", + PoolMetaCasToken::Existing(_) => "existing", + PoolMetaCasToken::Unsafe => "unsafe", + }, + "etag": match &read.cas { PoolMetaCasToken::Existing(etag) => Some(etag), _ => None }, + }); + match &read.replica { + PoolMetaReplica::Valid { + raw, + canonical, + meta, + revision, + committed, + .. + } => { + observation["state"] = serde_json::json!("valid"); + observation["committed"] = serde_json::json!(committed); + observation["version"] = serde_json::json!(revision.version); + observation["cluster_id"] = serde_json::json!(revision.cluster_id); + observation["epoch"] = serde_json::json!(revision.epoch); + observation["generation"] = serde_json::json!(revision.generation); + observation["transaction_id"] = serde_json::json!(revision.transaction_id); + observation["pool_count"] = serde_json::json!(meta.pools.len()); + observation["payload_sha256"] = serde_json::json!(rustfs_utils::crypto::hex(Sha256::digest(canonical))); + observation["raw_sha256"] = serde_json::json!(rustfs_utils::crypto::hex(Sha256::digest(raw))); + } + PoolMetaReplica::Missing => observation["state"] = serde_json::json!("missing"), + PoolMetaReplica::Corrupt(_) => observation["state"] = serde_json::json!("corrupt"), + PoolMetaReplica::Incompatible(_) => observation["state"] = serde_json::json!("incompatible"), + PoolMetaReplica::Unreadable(_) => observation["state"] = serde_json::json!("unreadable"), + } + startup_cas_test_observe(observation); + } + } + reads } fn select_pool_meta_replicas_observing(write_state: &mut PoolMetaWriteState, replicas: Vec) -> Result @@ -5480,6 +5522,60 @@ fn pool_meta_cas_preconditions(token: &PoolMetaCasToken, object: &str) -> Result } } +#[cfg(feature = "e2e-test-hooks")] +struct StartupCasObservation { + attempt: uuid::Uuid, + phase: &'static str, + pools: Vec, +} + +#[cfg(feature = "e2e-test-hooks")] +tokio::task_local! { + static STARTUP_CAS_OBSERVATION: StartupCasObservation; +} + +// This scope follows only the directly polled startup future. Spawned work +// does not inherit it; receiver evidence retains its existing RPC tuple. +#[cfg(feature = "e2e-test-hooks")] +pub(crate) async fn startup_cas_test_scope( + attempt: uuid::Uuid, + phase: &'static str, + pools: &[Arc], + future: F, +) -> F::Output { + STARTUP_CAS_OBSERVATION + .scope( + StartupCasObservation { + attempt, + phase, + // These identities are never dereferenced or logged. The + // caller and operation keep the same pool Arcs alive. + pools: pools.iter().map(|pool| Arc::as_ptr(pool) as usize).collect(), + }, + future, + ) + .await +} + +// Direct JSON diagnostics are independent of the startup tracing subscriber. +#[cfg(feature = "e2e-test-hooks")] +pub(crate) fn startup_cas_test_observe(mut observation: serde_json::Value) { + let Some(nonce) = std::env::var("RUSTFS_E2E_STARTUP_CAS_NONCE") + .ok() + .and_then(|value| uuid::Uuid::parse_str(&value).ok()) + else { + return; + }; + observation["nonce"] = serde_json::json!(nonce); + observation["pid"] = serde_json::json!(std::process::id()); + let _ = STARTUP_CAS_OBSERVATION.try_with(|scope| { + observation["attempt"] = serde_json::json!(scope.attempt); + observation["startup_phase"] = serde_json::json!(scope.phase); + }); + let line = format!("RUSTFS_E2E_STARTUP_CAS {observation}\n"); + let _ = std::io::Write::write_all(&mut std::io::stderr().lock(), line.as_bytes()); +} + async fn save_pool_meta_object_cas( pool: Arc, object: &str, @@ -5500,13 +5596,43 @@ where ..Default::default() }; fence.add_to_options(&mut opts); + #[cfg(feature = "e2e-test-hooks")] + let observation = std::env::var_os("RUSTFS_E2E_STARTUP_CAS_NONCE").map(|_| { + serde_json::json!({ + "kind": "cas", "object": object, "phase": phase, + "pool": STARTUP_CAS_OBSERVATION.try_with(|scope| { + scope.pools.iter().position(|identity| *identity == Arc::as_ptr(&pool) as usize) + }).ok().flatten(), + "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(&data)), + "if_match": opts.http_preconditions.as_ref().and_then(|p| p.if_match.as_deref()), + "if_none_match": opts.http_preconditions.as_ref().and_then(|p| p.if_none_match.as_deref()), + "tail_drained": opts.write_completion == crate::object_api::WriteCompletion::TailDrained, + "no_lock": opts.no_lock, + }) + }); let result = save_config_with_opts_and_metadata(pool, object, data, &opts).await; if matches!(&result, Err(Error::PreconditionFailed)) { record_pool_meta_stale_write_rejection(phase); } - let object_info = result?; - fence.ensure_held()?; - Ok(object_info) + let result = result.and_then(|object_info| { + fence.ensure_held()?; + Ok(object_info) + }); + #[cfg(feature = "e2e-test-hooks")] + if let Some(mut observation) = observation { + observation["ok"] = serde_json::json!(result.is_ok()); + observation["etag"] = serde_json::json!(result.as_ref().ok().and_then(|info| info.etag.as_deref())); + observation["mod_time"] = serde_json::json!( + result + .as_ref() + .ok() + .and_then(|info| info.mod_time) + .map(|time| time.unix_timestamp_nanos().to_string()) + ); + observation["error"] = serde_json::json!(result.as_ref().err().map(ToString::to_string)); + startup_cas_test_observe(observation); + } + result } async fn persist_pool_meta_identity( @@ -6805,6 +6931,13 @@ impl PoolMeta { }; if confirmed.revision == revision && confirmed.canonical.as_ref() == Some(&durable) { persist_pool_meta_identity(pools, write_state, true, fence).await?; + #[cfg(feature = "e2e-test-hooks")] + startup_cas_test_observe(serde_json::json!({ + "kind": "confirmed", "object": POOL_META_NAME, + "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(&durable)), + "generation": confirmed.revision.generation, + "transaction_id": confirmed.revision.transaction_id, + })); return Ok(confirmed.meta); } if !commit_succeeded { diff --git a/crates/ecstore/src/disk/os.rs b/crates/ecstore/src/disk/os.rs index ff9d3d5d6..3ac953582 100644 --- a/crates/ecstore/src/disk/os.rs +++ b/crates/ecstore/src/disk/os.rs @@ -247,7 +247,7 @@ pub(crate) mod fsync_dir_recorder { } /// Pause a real namespace mutation inside its physical executor. -#[cfg(all(test, not(windows)))] +#[cfg(all(any(test, feature = "test-util"), not(windows)))] pub(crate) mod prepared_publication_test_hooks { use super::*; @@ -256,7 +256,9 @@ pub(crate) mod prepared_publication_test_hooks { PreparedRename, Rename, Remove, + #[cfg(test)] Rollback, + #[cfg(test)] DirFsync, } @@ -272,6 +274,7 @@ pub(crate) mod prepared_publication_test_hooks { } } + #[cfg(test)] pub(crate) fn install(path: &Path, hook: impl FnOnce() + Send + 'static) -> Guard { install_at(Stage::PreparedRename, path, hook) } @@ -330,6 +333,51 @@ pub(crate) mod prepared_publication_test_hooks { } } +/// Controlled application-test pause at an existing physical executor boundary. +#[cfg(all(feature = "test-util", not(windows)))] +pub struct LocalPublicationPause { + _hook: prepared_publication_test_hooks::Guard, + entered: oneshot::Receiver<()>, + _release: std::sync::mpsc::Sender<()>, +} + +#[cfg(all(feature = "test-util", not(windows)))] +#[derive(Clone, Copy)] +pub enum LocalPublicationStage { + PreparedRename, + Rename, + Remove, +} + +#[cfg(all(feature = "test-util", not(windows)))] +impl LocalPublicationPause { + pub fn install(disk: &crate::disk::Disk, volume: &str, path: &str, stage: LocalPublicationStage) -> Result { + let path = disk + .get_object_path_for_io_if_local(volume, path) + .ok_or(DiskError::DiskNotFound)??; + let stage = match stage { + LocalPublicationStage::PreparedRename => prepared_publication_test_hooks::Stage::PreparedRename, + LocalPublicationStage::Rename => prepared_publication_test_hooks::Stage::Rename, + LocalPublicationStage::Remove => prepared_publication_test_hooks::Stage::Remove, + }; + let (entered_tx, entered) = oneshot::channel(); + let (release, release_rx) = std::sync::mpsc::channel::<()>(); + let hook = prepared_publication_test_hooks::install_at(stage, &path, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv(); + }); + Ok(Self { + _hook: hook, + entered, + _release: release, + }) + } + + pub async fn entered(&mut self) -> std::result::Result<(), oneshot::error::RecvError> { + (&mut self.entered).await + } +} + #[cfg(all(test, windows))] pub(crate) mod windows_rename_test_hooks { use super::*; @@ -1956,7 +2004,7 @@ pub(crate) async fn remove_file_with_owner( let path = path.as_ref().to_path_buf(); let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await; run_blocking_namespace_operation(lease, move || { - #[cfg(all(test, not(windows)))] + #[cfg(all(any(test, feature = "test-util"), not(windows)))] prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Remove, &path); std::fs::remove_file(path) }) @@ -2216,7 +2264,7 @@ pub(crate) async fn rename_all_with_prepared_source( move || { validate_prepared_rename_source(&prepared_source, &src_file_path)?; let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?; - #[cfg(test)] + #[cfg(any(test, feature = "test-util"))] prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::PreparedRename, &dst_file_path); rename_prepared(&src_file_path, &dst_file_path, &preparation) } @@ -2349,7 +2397,7 @@ async fn reliable_rename_inner_with_lease( let preparation = prepare_rename_with_retry(&src_file_path, &dst_file_path, &base_dir, &publication_root)?; #[cfg(all(test, not(windows)))] prepared_publication_test_hooks::run_rename_destination(&src_file_path, &dst_file_path); - #[cfg(all(test, not(windows)))] + #[cfg(all(any(test, feature = "test-util"), not(windows)))] { prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &src_file_path); prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, &dst_file_path); diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 86662f182..a38ba4077 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -630,14 +630,27 @@ impl ECStore { .pools .first() .is_some_and(|pool| pool_first_endpoint_is_local(&pool.endpoints)); + #[cfg(feature = "e2e-test-hooks")] + let startup_attempt = uuid::Uuid::new_v4(); let (meta, pool_meta_replica_state) = { let mut write_state = self.pool_meta_save_gate.lock().await; establish_pool_meta_bootstrap_identity_if_proven(self.pools.clone(), &mut write_state, should_persist_pool_meta) .await .map_err(|err| Error::other(format!("store init failed during establish_pool_meta_bootstrap_identity: {err}")))?; - load_pool_meta_for_startup(self.pools.clone(), &mut write_state).await? + let load = load_pool_meta_for_startup(self.pools.clone(), &mut write_state); + #[cfg(feature = "e2e-test-hooks")] + let load = crate::core::pools::startup_cas_test_scope(startup_attempt, "load", &self.pools, load); + load.await? }; let update = meta.validate(self.pools.clone())?; + #[cfg(feature = "e2e-test-hooks")] + crate::core::pools::startup_cas_test_observe(serde_json::json!({ + "kind": "startup-classifier", "attempt": startup_attempt, + "elected_writer": should_persist_pool_meta, + "needs_repair": pool_meta_replica_state.needs_repair, + "repair_write_safe": pool_meta_replica_state.repair_write_safe, + "topology_update": update, + })); let endpoints = runtime_sources::endpoint_pools_or_default(); let mut installed_pool_meta = if update { @@ -649,15 +662,17 @@ impl ECStore { // distributed startup can race on the same lock and replay the prior init bug. { let mut write_state = self.pool_meta_save_gate.lock().await; - installed_pool_meta = persist_pool_meta_for_startup_if_safe( + let persist = persist_pool_meta_for_startup_if_safe( &installed_pool_meta, self.pools.clone(), pool_meta_replica_state, &mut write_state, update, should_persist_pool_meta, - ) - .await?; + ); + #[cfg(feature = "e2e-test-hooks")] + let persist = crate::core::pools::startup_cas_test_scope(startup_attempt, "persist", &self.pools, persist); + installed_pool_meta = persist.await?; } { diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs index b54a69438..530362f39 100644 --- a/crates/ecstore/src/store/mod.rs +++ b/crates/ecstore/src/store/mod.rs @@ -442,7 +442,7 @@ pub(crate) mod utils; use peer::init_local_peer; pub use peer::{ - all_local_disk, all_local_disk_path, find_local_disk_by_ref, get_disk_infos, init_local_disks, + BootstrapLocalTarget, all_local_disk, all_local_disk_path, find_local_disk_by_ref, get_disk_infos, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map, prewarm_local_disk_id_map_with_instance_ctx, }; @@ -1787,7 +1787,7 @@ mod tests { // Build a minimal ECStore carrying an explicit instance context. Empty // pools/disks are sufficient: the Phase 5 accessors read only `self.ctx`. - fn build_store_with_ctx(ctx: Arc) -> Arc { + pub(super) fn build_store_with_ctx(ctx: Arc) -> Arc { let endpoint_pools = EndpointServerPools::default(); Arc::new(ECStore { id: uuid::Uuid::new_v4(), diff --git a/crates/ecstore/src/store/peer.rs b/crates/ecstore/src/store/peer.rs index d8ba54d1f..858b14be7 100644 --- a/crates/ecstore/src/store/peer.rs +++ b/crates/ecstore/src/store/peer.rs @@ -13,7 +13,10 @@ // limitations under the License. use super::*; -use crate::runtime::instance::InstanceContext; +use crate::bucket::utils::has_bad_path_component; +use crate::disk::error::{DiskError, Result as DiskResult}; +use crate::disk::{DeleteOptions, Disk, RenameDataGuards, RenameDataResp}; +use crate::runtime::instance::{InstanceContext, NamespaceCommitGuard}; use crate::runtime::sources as runtime_sources; use tracing::{debug, error}; @@ -22,6 +25,203 @@ const LOG_SUBSYSTEM_DISK_STARTUP: &str = "disk_startup"; const EVENT_LOCAL_DISK_ID_PREWARM_SKIPPED: &str = "local_disk_id_prewarm_skipped"; const EVENT_LOCK_CLIENT_INITIALIZATION_FAILED: &str = "lock_client_initialization_failed"; +/// An instance-bound capability for internal writes before ECStore/IAM startup. +/// Its private context and volume checks cannot be replaced by a caller guard. +#[derive(Clone)] +pub struct BootstrapLocalTarget { + ctx: Arc, +} + +impl BootstrapLocalTarget { + pub fn new(ctx: Arc) -> Self { + Self { ctx } + } + + pub fn is_for_store(&self, store: &ECStore) -> bool { + Arc::ptr_eq(&self.ctx, &store.ctx) + } + + pub async fn rename_local_data( + &self, + disk_ref: &str, + source: (&str, &str), + fi: &FileInfo, + destination: (&str, &str), + scanner_token: Option, + ) -> DiskResult { + if scanner_token.is_some() { + return Err(DiskError::other("bootstrap rename cannot use a scanner publication lease")); + } + validate_bootstrap_volume(source.0)?; + validate_bootstrap_volume(destination.0)?; + rename_local_data_with_ctx(&self.ctx, disk_ref, source, fi, destination, RenameDataGuards::default()).await + } + + pub async fn undo_local_write( + &self, + disk_ref: &str, + volume: &str, + path: &str, + fi: FileInfo, + opts: DeleteOptions, + ) -> DiskResult<()> { + validate_bootstrap_volume(volume)?; + undo_local_write_with_ctx(&self.ctx, disk_ref, volume, path, fi, opts).await + } +} + +fn validate_bootstrap_volume(volume: &str) -> DiskResult<()> { + // Prefix membership alone permits aliases such as .rustfs.sys/../bucket. + // Validate both raw rename volumes before any disk lookup or admission. + if has_bad_path_component(volume) || !is_meta_bucketname(volume) { + return Err(DiskError::FileAccessDenied); + } + Ok(()) +} + +impl ECStore { + /// Execute on this instance's active local disk through the physical owner. + pub async fn rename_local_data( + &self, + disk_ref: &str, + source: (&str, &str), + fi: &FileInfo, + destination: (&str, &str), + scanner_token: Option, + ) -> DiskResult { + let external_guard: Option> = if let Some(token) = scanner_token { + Some(Arc::new( + self.acquire_scanner_publication_lease_guard(token) + .await + .map_err(|err| DiskError::other(err.to_string()))?, + )) + } else { + None + }; + rename_local_data_with_ctx( + &self.ctx, + disk_ref, + source, + fi, + destination, + RenameDataGuards { + scanner_publication_lease_token: scanner_token, + external_guard, + namespace_owner: None, + }, + ) + .await + } + + pub async fn undo_local_write( + &self, + disk_ref: &str, + volume: &str, + path: &str, + fi: FileInfo, + opts: DeleteOptions, + ) -> DiskResult<()> { + undo_local_write_with_ctx(&self.ctx, disk_ref, volume, path, fi, opts).await + } +} + +// The optional ID is a cold lookup to cache only after final admission. +async fn local_disk_candidate(ctx: &Arc, disk_ref: &str) -> DiskResult<(DiskStore, Option)> { + let map = ctx.local_disk_map(); + if let Some(disk) = map.read().await.get(disk_ref).and_then(Option::as_ref).cloned() { + return Ok((disk, None)); + } + let disk_id = Uuid::parse_str(disk_ref).map_err(|_| DiskError::DiskNotFound)?; + let cached_path = ctx.local_disk_id_map().read().await.get(&disk_id).cloned(); + if let Some(path) = cached_path { + let cached_disk = map.read().await.get(&path).and_then(Option::as_ref).cloned(); + if let Some(disk) = cached_disk + && matches!(disk.as_ref(), Disk::Local(_)) + && disk.get_disk_id().await? == Some(disk_id) + { + return Ok((disk, None)); + } + } + let disks: Vec<_> = map.read().await.values().filter_map(Clone::clone).collect(); + // Disk identity may perform format I/O. No registry guard spans this await. + for disk in disks { + if matches!(disk.as_ref(), Disk::Local(_)) && disk.get_disk_id().await.ok().flatten() == Some(disk_id) { + return Ok((disk, Some(disk_id))); + } + } + Err(DiskError::DiskNotFound) +} + +async fn admit_local_disk( + ctx: &Arc, + disk: &DiskStore, + disk_id: Option, + volume: &str, +) -> DiskResult>> { + if !matches!(disk.as_ref(), Disk::Local(_)) { + return Err(DiskError::DiskNotFound); + } + let map = ctx.local_disk_map(); + let active = map.read().await; + if !active + .get(&disk.endpoint().to_string()) + .and_then(Option::as_ref) + .is_some_and(|current| Arc::ptr_eq(current, disk)) + { + return Err(DiskError::DiskNotFound); + } + // Preserve registry -> ID-cache lock order; no filesystem I/O under either. + if let Some(disk_id) = disk_id { + ctx.local_disk_id_map() + .write() + .await + .insert(disk_id, disk.endpoint().to_string()); + } + // Admission linearizes under the registry read: replacement/quarantine + // before this point rejects; later changes do not revoke physical I/O. + Ok((!is_meta_bucketname(volume)).then(|| ctx.begin_namespace_commit())) +} + +async fn rename_local_data_with_ctx( + ctx: &Arc, + disk_ref: &str, + source: (&str, &str), + fi: &FileInfo, + destination: (&str, &str), + mut guards: RenameDataGuards, +) -> DiskResult { + let (disk, disk_id) = local_disk_candidate(ctx, disk_ref).await?; + let owner = admit_local_disk(ctx, &disk, disk_id, destination.0).await?; + guards.namespace_owner = owner.as_ref().map(|owner| owner.clone() as Arc); + let result = disk + .rename_data_borrowed_with_fence_observed(source.0, source.1, fi, destination.0, destination.1, guards) + .await + .result; + drop(owner); + result +} + +async fn undo_local_write_with_ctx( + ctx: &Arc, + disk_ref: &str, + volume: &str, + path: &str, + fi: FileInfo, + opts: DeleteOptions, +) -> DiskResult<()> { + if !opts.undo_write { + return Err(DiskError::other("target undo requires undo_write")); + } + let (disk, disk_id) = local_disk_candidate(ctx, disk_ref).await?; + let owner = admit_local_disk(ctx, &disk, disk_id, volume).await?; + let physical_owner = owner.as_ref().map(|owner| owner.clone() as Arc); + let result = disk + .undo_write_with_namespace_owner(volume, path, fi, opts, physical_owner) + .await; + drop(owner); + result +} + async fn remember_local_disk_id(disk: &DiskStore) -> Option { remember_local_disk_id_with_instance_ctx(&crate::runtime::global::current_ctx(), disk).await } @@ -265,6 +465,522 @@ mod tests { }]) } + async fn target_disk(ctx: &Arc, root: &std::path::Path, id: Uuid) -> DiskStore { + let mut format = crate::layout::format::FormatV3::new(1, 1); + format.erasure.this = id; + format.erasure.sets[0][0] = id; + let meta = root.join(crate::disk::RUSTFS_META_BUCKET); + tokio::fs::create_dir_all(&meta).await.expect("create format volume"); + tokio::fs::write( + meta.join(crate::disk::FORMAT_CONFIG_FILE), + serde_json::to_vec(&format).expect("encode format"), + ) + .await + .expect("write real disk identity"); + let mut endpoint = Endpoint::try_from(root.to_str().expect("UTF-8 root")).expect("endpoint"); + endpoint.set_pool_index(0); + endpoint.set_set_index(0); + endpoint.set_disk_index(0); + let disk = new_disk( + &endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("open real local disk"); + assert_eq!(disk.get_disk_id().await.expect("read disk format identity"), Some(id)); + ctx.local_disk_map() + .write() + .await + .insert(disk.endpoint().to_string(), Some(disk.clone())); + disk + } + + fn target_file_info(object: &str, version: Uuid, body: &'static [u8]) -> FileInfo { + let mut fi = FileInfo::new(object, 1, 0); + fi.erasure.index = 1; + fi.version_id = Some(version); + fi.mod_time = Some(OffsetDateTime::now_utc()); + fi.size = i64::try_from(body.len()).expect("fixture length"); + fi.parts = vec![rustfs_filemeta::ObjectPartInfo { + number: 1, + size: body.len(), + actual_size: fi.size, + ..Default::default() + }]; + fi.data = Some(bytes::Bytes::from_static(body)); + fi.set_inline_data(); + fi + } + + async fn seed_target(disk: &DiskStore, volume: &str, object: &str, fi: FileInfo) -> Vec { + let dir = disk.path().join(volume); + tokio::fs::create_dir_all(&dir).await.expect("real fixture volume"); + disk.write_metadata(volume, volume, object, fi.clone()) + .await + .expect("seed real metadata"); + let read = disk + .read_version( + volume, + volume, + object, + &fi.version_id.expect("fixture version").to_string(), + &crate::disk::ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("read fixture before mutation"); + assert_eq!(read.data, fi.data, "fixture must contain readable inline bytes"); + tokio::fs::read(dir.join(object).join(crate::disk::STORAGE_FORMAT_FILE)) + .await + .expect("seeded metadata bytes") + } + + #[tokio::test] + async fn target_uuid_lookup_binds_real_disk_and_owner_to_one_instance() { + for warm in [false, true] { + let ctx_a = Arc::new(InstanceContext::new()); + let ctx_b = Arc::new(InstanceContext::new()); + let a = tempfile::tempdir().expect("A root"); + let b = tempfile::tempdir().expect("B root"); + let id = Uuid::new_v4(); + let disk_a = target_disk(&ctx_a, a.path(), id).await; + let disk_b = target_disk(&ctx_b, b.path(), id).await; + if warm { + assert!(record_local_disk_id_if_active(&ctx_a, &disk_a, id).await); + assert!(record_local_disk_id_if_active(&ctx_b, &disk_b, id).await); + } + let version = Uuid::new_v4(); + let fi = target_file_info("destination", version, b"new-A"); + for disk in [&disk_a, &disk_b] { + seed_target(disk, "target-bucket", "staged", fi.clone()).await; + } + let b_before = seed_target( + &disk_b, + "target-bucket", + "destination", + target_file_info("destination", version, b"old-B"), + ) + .await; + let store = super::super::tests::build_store_with_ctx(ctx_a.clone()); + store + .rename_local_data(&id.to_string(), ("target-bucket", "staged"), &fi, ("target-bucket", "destination"), None) + .await + .expect("rename on A"); + let read = disk_a + .read_version( + "target-bucket", + "target-bucket", + "destination", + &version.to_string(), + &crate::disk::ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("read committed A"); + assert_eq!(read.data, fi.data, "warm={warm}"); + assert_eq!( + tokio::fs::read(b.path().join("target-bucket/destination/xl.meta")) + .await + .expect("B metadata"), + b_before + ); + assert!(b.path().join("target-bucket/staged/xl.meta").exists()); + assert!(ctx_a.namespace_commit_generation() > 0); + assert_eq!(ctx_b.namespace_commit_generation(), 0); + assert!(!ctx_a.namespace_commits_pending()); + assert!(!ctx_b.namespace_commits_pending()); + assert_eq!(ctx_a.local_disk_id_map().read().await.get(&id), Some(&disk_a.endpoint().to_string())); + } + } + + #[tokio::test] + async fn target_admission_rejects_removed_quarantined_and_replaced_arcs() { + let ctx = Arc::new(InstanceContext::new()); + let root = tempfile::tempdir().expect("root"); + let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await; + let endpoint = disk.endpoint().to_string(); + for state in ["removed", "quarantined", "replaced"] { + let replacement = new_disk( + &disk.endpoint(), + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await + .expect("separate active Arc"); + let map = ctx.local_disk_map(); + let mut entries = map.write().await; + match state { + "removed" => { + entries.remove(&endpoint); + } + "quarantined" => { + entries.insert(endpoint.clone(), None); + } + _ => { + entries.insert(endpoint.clone(), Some(replacement)); + } + } + drop(entries); + assert!( + matches!(admit_local_disk(&ctx, &disk, None, "target-bucket").await, Err(DiskError::DiskNotFound)), + "{state}" + ); + assert!(!ctx.namespace_commits_pending()); + assert_eq!(ctx.namespace_commit_generation(), 0); + } + } + + #[tokio::test] + async fn target_uuid_cache_cannot_admit_a_different_format_at_the_same_path() { + let ctx = Arc::new(InstanceContext::new()); + let root = tempfile::tempdir().expect("root"); + let old_id = Uuid::new_v4(); + let old = target_disk(&ctx, root.path(), old_id).await; + assert!(record_local_disk_id_if_active(&ctx, &old, old_id).await); + let replacement_id = Uuid::new_v4(); + let replacement = target_disk(&ctx, root.path(), replacement_id).await; + assert!(!Arc::ptr_eq(&old, &replacement)); + assert!(matches!( + local_disk_candidate(&ctx, &old_id.to_string()).await, + Err(DiskError::DiskNotFound) + )); + let (candidate, verified) = local_disk_candidate(&ctx, &replacement_id.to_string()) + .await + .expect("replacement UUID"); + assert!(Arc::ptr_eq(&candidate, &replacement)); + assert_eq!(verified, Some(replacement_id)); + assert!(!ctx.namespace_commits_pending()); + } + + #[tokio::test] + async fn bootstrap_rejects_user_volumes_aliases_and_scanner_tokens_without_mutation() { + let ctx = Arc::new(InstanceContext::new()); + let root = tempfile::tempdir().expect("root"); + let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await; + let target = BootstrapLocalTarget::new(ctx.clone()); + let fi = target_file_info("destination", Uuid::new_v4(), b"body"); + let user_before = seed_target(&disk, "victim", "staged", fi.clone()).await; + let meta_before = seed_target(&disk, ".rustfs.sys/tmp", "staged", fi.clone()).await; + for invalid in [ + "victim", + ".rustfs.sys/../victim", + ".rustfs.sys/./tmp", + ".rustfs.sys/ .. /victim", + ".rustfs.sys\\..\\victim", + ".minio.sys/../victim", + ] { + for (src, dst) in [(invalid, ".rustfs.sys/tmp"), (".rustfs.sys/tmp", invalid)] { + assert!( + target + .rename_local_data(&disk.endpoint().to_string(), (src, "staged"), &fi, (dst, "destination"), None) + .await + .is_err(), + "src={src}, dst={dst}" + ); + } + assert!( + target + .undo_local_write( + &disk.endpoint().to_string(), + invalid, + "staged", + fi.clone(), + DeleteOptions { + undo_write: true, + ..Default::default() + } + ) + .await + .is_err(), + "{invalid}" + ); + } + assert!( + target + .rename_local_data( + &disk.endpoint().to_string(), + (".rustfs.sys/tmp", "staged"), + &fi, + (".rustfs.sys/tmp", "destination"), + Some(Uuid::new_v4()) + ) + .await + .is_err() + ); + assert_eq!( + tokio::fs::read(root.path().join("victim/staged/xl.meta")) + .await + .expect("user source"), + user_before + ); + assert_eq!( + tokio::fs::read(root.path().join(".rustfs.sys/tmp/staged/xl.meta")) + .await + .expect("metadata source"), + meta_before + ); + assert!(!root.path().join("victim/destination").exists()); + assert!(!root.path().join(".rustfs.sys/tmp/destination").exists()); + assert_eq!(ctx.namespace_commit_generation(), 0); + assert!(!ctx.namespace_commits_pending()); + } + + #[tokio::test] + async fn bootstrap_allows_internal_multisegment_rename_without_namespace_owner() { + for volume in [".rustfs.sys/tmp", ".rustfs.sys/multipart", ".minio.sys/config"] { + let ctx = Arc::new(InstanceContext::new()); + let root = tempfile::tempdir().expect("root"); + let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await; + let fi = target_file_info("destination", Uuid::new_v4(), b"internal-CAS-body"); + seed_target(&disk, volume, "staged", fi.clone()).await; + BootstrapLocalTarget::new(ctx.clone()) + .rename_local_data(&disk.endpoint().to_string(), (volume, "staged"), &fi, (volume, "destination"), None) + .await + .expect("legitimate bootstrap metadata write"); + let read = disk + .read_version( + volume, + volume, + "destination", + &fi.version_id.expect("version").to_string(), + &crate::disk::ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("read bootstrap result"); + assert_eq!(read.data, fi.data); + assert_eq!(ctx.namespace_commit_generation(), 0); + assert!(!ctx.namespace_commits_pending()); + } + } + + #[cfg(not(windows))] + #[tokio::test] + async fn target_rename_cancellation_retains_real_namespace_and_scanner_owners() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + let ctx = Arc::new(InstanceContext::new()); + let sibling = Arc::new(InstanceContext::new()); + let root = tempfile::tempdir().expect("root"); + let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await; + let store = super::super::tests::build_store_with_ctx(ctx.clone()); + let fi = target_file_info("destination", Uuid::new_v4(), b"physically-owned"); + seed_target(&disk, "target-bucket", "staged", fi.clone()).await; + let (token, _) = store + .acquire_scanner_publication_lease(0, crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL) + .await + .expect("real scanner token in A"); + let destination = disk + .get_object_path_for_io_if_local("target-bucket", "destination/xl.meta") + .expect("local disk") + .expect("destination IO path"); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + let _hook = hooks::install(&destination, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv(); + }); + let disk_ref = disk.endpoint().to_string(); + let mut rename = Box::pin(store.rename_local_data( + &disk_ref, + ("target-bucket", "staged"), + &fi, + ("target-bucket", "destination"), + Some(token), + )); + tokio::time::timeout(std::time::Duration::from_secs(10), async { + tokio::select! { + result = &mut rename => panic!("rename completed before physical pause: {result:?}"), + entered = entered_rx => entered.expect("physical rename entered"), + } + }) + .await + .expect("bounded physical entry"); + drop(rename); + assert!(store.scanner_data_usage_publication_blocked().await); + assert!(ctx.namespace_commits_pending()); + assert!(!sibling.namespace_commits_pending()); + assert!( + store + .rename_local_data(&disk_ref, ("target-bucket", "staged"), &fi, ("target-bucket", "another"), Some(token)) + .await + .is_err(), + "real pending rename blocks another scanner publication" + ); + assert!(store.release_scanner_publication_lease(token).await, "remove registered token"); + let gate = ctx.data_movement_operation_gate(); + assert!( + gate.clone().try_write_owned().is_err(), + "physical operation still owns the scanner read guard" + ); + drop(release_tx); + let _drained = tokio::time::timeout(std::time::Duration::from_secs(10), gate.write_owned()) + .await + .expect("physical tail must release scanner guard"); + tokio::time::timeout(std::time::Duration::from_secs(10), async { + while ctx.namespace_commits_pending() { + tokio::task::yield_now().await; + } + }) + .await + .expect("namespace owner drains"); + let read = disk + .read_version( + "target-bucket", + "target-bucket", + "destination", + &fi.version_id.expect("version").to_string(), + &crate::disk::ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("read actual late commit"); + assert_eq!(read.data, fi.data); + assert!(ctx.namespace_commit_generation() >= 2); + assert_eq!(sibling.namespace_commit_generation(), 0); + } + + #[tokio::test] + async fn target_ready_rejects_unknown_foreign_released_and_expired_scanner_tokens() { + let ctx = Arc::new(InstanceContext::new()); + let other = Arc::new(InstanceContext::new()); + let store = super::super::tests::build_store_with_ctx(ctx.clone()); + let other_store = super::super::tests::build_store_with_ctx(other); + let root = tempfile::tempdir().expect("root"); + let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await; + let fi = target_file_info("destination", Uuid::new_v4(), b"unchanged"); + let before = seed_target(&disk, "target-bucket", "staged", fi.clone()).await; + let ttl = crate::runtime::instance::SCANNER_PUBLICATION_LEASE_TTL; + let (foreign, _) = other_store.acquire_scanner_publication_lease(0, ttl).await.expect("B token"); + let (released, _) = store.acquire_scanner_publication_lease(0, ttl).await.expect("A token"); + assert!(store.release_scanner_publication_lease(released).await); + let (valid, _) = store.acquire_scanner_publication_lease(0, ttl).await.expect("new A token"); + for token in [Uuid::new_v4(), foreign, released] { + assert!( + store + .rename_local_data( + &disk.endpoint().to_string(), + ("target-bucket", "staged"), + &fi, + ("target-bucket", "destination"), + Some(token) + ) + .await + .is_err() + ); + } + tokio::time::pause(); + tokio::time::advance(ttl + std::time::Duration::from_secs(1)).await; + tokio::time::resume(); + assert!( + store + .rename_local_data( + &disk.endpoint().to_string(), + ("target-bucket", "staged"), + &fi, + ("target-bucket", "destination"), + Some(valid) + ) + .await + .is_err(), + "expired real token" + ); + let _ = other_store.release_scanner_publication_lease(foreign).await; + assert_eq!( + tokio::fs::read(root.path().join("target-bucket/staged/xl.meta")) + .await + .expect("source bytes"), + before + ); + assert!(!root.path().join("target-bucket/destination").exists()); + assert!(!ctx.namespace_commits_pending()); + } + + #[cfg(not(windows))] + #[tokio::test] + #[serial_test::serial] + async fn target_ordinary_timeout_keeps_its_physical_namespace_owner() { + use crate::disk::os::prepared_publication_test_hooks as hooks; + temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("1"))], async { + let ctx = Arc::new(InstanceContext::new()); + let store = super::super::tests::build_store_with_ctx(ctx.clone()); + let root = tempfile::tempdir().expect("root"); + let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await; + let fi = target_file_info("destination", Uuid::new_v4(), b"timed-out-physical-commit"); + seed_target(&disk, "target-bucket", "staged", fi.clone()).await; + let path = disk + .get_object_path_for_io_if_local("target-bucket", "destination/xl.meta") + .expect("local") + .expect("destination IO path"); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + let _hook = hooks::install(&path, move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv(); + }); + let disk_ref = disk.endpoint().to_string(); + let mut rename = Box::pin(store.rename_local_data( + &disk_ref, + ("target-bucket", "staged"), + &fi, + ("target-bucket", "destination"), + None, + )); + tokio::time::timeout(std::time::Duration::from_secs(10), async { + tokio::select! { + result = &mut rename => panic!("completed before physical pause: {result:?}"), + entered = entered_rx => entered.expect("physical entry"), + } + }) + .await + .expect("bounded entry"); + tokio::time::pause(); + tokio::time::advance(std::time::Duration::from_secs(2)).await; + tokio::time::resume(); + let result = tokio::time::timeout(std::time::Duration::from_secs(5), &mut rename) + .await + .expect("ordinary deadline remains enabled"); + assert!(matches!(result, Err(DiskError::Timeout)), "{result:?}"); + drop(rename); + assert!(ctx.namespace_commits_pending(), "timeout is not a physical drain"); + drop(release_tx); + tokio::time::timeout(std::time::Duration::from_secs(10), async { + while ctx.namespace_commits_pending() { + tokio::task::yield_now().await; + } + }) + .await + .expect("late physical owner drains"); + let read = disk + .read_version( + "target-bucket", + "target-bucket", + "destination", + &fi.version_id.expect("version").to_string(), + &crate::disk::ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("read actual timeout tail"); + assert_eq!(read.data, fi.data); + }) + .await; + } + #[test] fn endpoint_rpc_authority_preserves_port_and_ipv6_brackets() { let endpoint = Endpoint::try_from("https://127.0.0.1:9001/d1").expect("URL endpoint"); diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 52a7e6598..151099e8e 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -68,7 +68,7 @@ license = [] io-scheduler-debug = [] # Enable debug information in I/O scheduler tracing-chunk-debug = [] # Enable per-chunk tracing in data plane (high noise, for debugging only) full = ["metrics-gpu", "ftps", "swift", "webdav", "sftp", "pyroscope", "gcs"] -e2e-test-hooks = [] +e2e-test-hooks = ["rustfs-ecstore/e2e-test-hooks"] # Shortens Connect credentials only in debug E2E builds. connect-e2e-short-credentials = [] # Builds the dedicated rustfs-cli-e2e target with a build-time public enrollment root. diff --git a/rustfs/src/app/context/server_slot.rs b/rustfs/src/app/context/server_slot.rs index 56ccedf57..99c260144 100644 --- a/rustfs/src/app/context/server_slot.rs +++ b/rustfs/src/app/context/server_slot.rs @@ -26,13 +26,14 @@ //! server is not ready rather than that another server's global context applies. use super::global::{AppContext, get_global_app_context}; -use crate::app::storage_api::context::ECStore; +use crate::app::storage_api::context::{BootstrapLocalTarget, ECStore, InstanceContext}; use std::sync::{Arc, OnceLock}; /// Late-bound, per-server handle to the application context. #[derive(Default)] pub struct ServerContextSlot { app_context: OnceLock>, + bootstrap_target: Option, heal_topology_fingerprint: Arc>, } @@ -50,15 +51,47 @@ impl ServerContextSlot { pub fn new() -> Arc { Arc::new(Self { app_context: OnceLock::new(), + bootstrap_target: None, heal_topology_fingerprint: Arc::new(tokio::sync::OnceCell::new()), }) } + /// Bind the listener to its foundation before it can accept requests. + pub fn with_instance_context(ctx: Arc) -> Arc { + Arc::new(Self { + bootstrap_target: Some(BootstrapLocalTarget::new(ctx)), + ..Self::default() + }) + } + /// Install this server's application context (once). Returns `false` if /// the slot was already installed; the first installation wins, matching /// the process-global singleton's `get_or_init` semantics. pub fn install(&self, context: Arc) -> bool { - self.app_context.set(context).is_ok() + self.try_install(context).is_ok() + } + + /// Claim the slot before any process-global application publication. + /// Repeated installation, even of the same Arc, is an explicit conflict. + pub fn try_install(&self, context: Arc) -> std::io::Result<()> { + if self + .bootstrap_target + .as_ref() + .is_some_and(|target| !target.is_for_store(&context.object_store())) + { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "application context does not belong to this server foundation", + )); + } + self.app_context.set(context).map_err(|_| { + std::io::Error::new(std::io::ErrorKind::AlreadyExists, "server application context is already installed") + }) + } + + /// Immutable, restricted startup capability; never resolves an ambient store. + pub fn bootstrap_target(&self) -> Option { + self.bootstrap_target.clone() } /// This server's installed application context, if startup has completed. diff --git a/rustfs/src/app/context/startup.rs b/rustfs/src/app/context/startup.rs index b19b1aca0..77966fe6c 100644 --- a/rustfs/src/app/context/startup.rs +++ b/rustfs/src/app/context/startup.rs @@ -37,8 +37,8 @@ impl AppContext { // also publishes to the process default (first server wins) so legacy // free-function readers keep resolving the first server's context. let context = Arc::new(AppContext::with_default_interfaces(store, iam, kms_interface)); - publish_global_app_context(context.clone()); - let _ = server_ctx.install(context); + server_ctx.try_install(context.clone())?; + publish_global_app_context(context); Ok(()) } } diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index 7a48d5ff1..aee2a44e9 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -1261,7 +1261,7 @@ pub(crate) mod context { pub(crate) use super::EndpointServerPools; pub(crate) use super::bucket; pub(crate) use super::runtime; - pub(crate) use crate::storage::storage_api::{ECStore, EndpointServerPools}; + pub(crate) use crate::storage::storage_api::{BootstrapLocalTarget, ECStore, EndpointServerPools, InstanceContext}; #[cfg(test)] pub(crate) use crate::storage::storage_api::{Endpoint, Endpoints, PoolEndpoints}; } diff --git a/rustfs/src/server/http.rs b/rustfs/src/server/http.rs index d82d67f20..41a9e3519 100644 --- a/rustfs/src/server/http.rs +++ b/rustfs/src/server/http.rs @@ -36,7 +36,9 @@ use crate::server::{ }; use crate::storage_api::server::http as storage; use crate::storage_api::server::http::rpc::InternodeRpcService; +#[cfg(test)] use crate::storage_api::server::http::tonic_service::make_server; +use crate::storage_api::server::http::tonic_service::make_server_for_slot; use crate::storage_api::server::http::{ ServerContextSlot, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, verify_tonic_rpc_signature_with_bootstrap, @@ -1834,7 +1836,7 @@ fn process_connection( // each service in the auth interceptor. let rpc_max_message_size = rustfs_protos::internode_rpc_max_message_size(); let node_service = InterceptedService::new( - NodeServiceServer::new(make_server()) + NodeServiceServer::new(make_server_for_slot(Arc::clone(&server_ctx))) .max_decoding_message_size(rpc_max_message_size) .max_encoding_message_size(rpc_max_message_size), check_auth, diff --git a/rustfs/src/startup_embedded.rs b/rustfs/src/startup_embedded.rs index e23bf9563..7738f85ba 100644 --- a/rustfs/src/startup_embedded.rs +++ b/rustfs/src/startup_embedded.rs @@ -124,9 +124,6 @@ pub(crate) async fn run_embedded_startup(args: EmbeddedStartupArgs) -> Result Result Result<()> { + #[cfg(feature = "e2e-test-hooks")] + if let Ok(nonce) = std::env::var("RUSTFS_E2E_STARTUP_CAS_PROBE") { + let nonce = uuid::Uuid::parse_str(&nonce).map_err(Error::other)?; + // This precedes CLI parsing and observability, including `--help`. + println!( + "RUSTFS_E2E_STARTUP_CAS {}", + serde_json::json!({ + "kind": "capability", "schema": "fresh-startup-cas/v1", "nonce": nonce, + }) + ); + return Ok(()); + } + #[cfg(feature = "e2e-test-hooks")] + if let Ok(nonce) = std::env::var("RUSTFS_E2E_STARTUP_CAS_NONCE") { + let nonce = uuid::Uuid::parse_str(&nonce).map_err(Error::other)?; + let line = format!( + "RUSTFS_E2E_STARTUP_CAS {}\n", + serde_json::json!({ + "kind": "observer-ready", "nonce": nonce, "pid": std::process::id(), + }) + ); + let _ = std::io::Write::write_all(&mut std::io::stderr().lock(), line.as_bytes()); + } hotpath::tokio_runtime!(); // Log container resource detection early in startup @@ -141,10 +164,6 @@ async fn run(config: Config) -> Result<()> { // the storage path explicitly (Phase 5 follow-up, backlog#1052); a future // multi-instance server constructs its own context here instead. let instance_ctx = bootstrap_instance_ctx(); - // This server's request-path context slot (backlog#1052 S2): handed to the - // HTTP service now, installed once IAM bootstrap completes. - let server_ctx = ServerContextSlot::new(); - let StartupListenContext { readiness, server_addr, @@ -152,6 +171,7 @@ async fn run(config: Config) -> Result<()> { } = init_startup_listen_context(&config, &instance_ctx).await?; let endpoint_pools = init_startup_storage_foundation(&server_address, &config.volumes, &instance_ctx).await?; + let server_ctx = ServerContextSlot::with_instance_context(instance_ctx.clone()); let StartupHttpServers { state_manager, s3_shutdown_tx, @@ -163,6 +183,33 @@ async fn run(config: Config) -> Result<()> { shutdown_token: ctx, } = init_startup_storage_runtime(server_addr, &endpoint_pools, readiness.clone(), instance_ctx).await?; + #[cfg(feature = "e2e-test-hooks")] + if let Ok(nonce) = std::env::var("RUSTFS_E2E_STARTUP_CAS_NONCE") { + let nonce = uuid::Uuid::parse_str(&nonce).map_err(Error::other)?; + let release = std::path::PathBuf::from( + std::env::var_os("RUSTFS_E2E_STARTUP_CAS_RELEASE") + .ok_or_else(|| Error::other("startup CAS fixture requires a release path"))?, + ); + if server_ctx.installed_object_store().is_some() { + return Err(Error::other("startup CAS gate reached an installed slot")); + } + let line = format!( + "RUSTFS_E2E_STARTUP_CAS {}\n", + serde_json::json!({ + "kind": "gate", "nonce": nonce, "pid": std::process::id(), "slot_installed": false, + }) + ); + let _ = std::io::Write::write_all(&mut std::io::stderr().lock(), line.as_bytes()); + tokio::time::timeout(std::time::Duration::from_secs(180), async { + while !tokio::fs::try_exists(&release).await? { + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + Ok::<_, Error>(()) + }) + .await + .map_err(|_| Error::other("startup CAS gate release timed out"))??; + } + let capacity_tasks = crate::capacity::capacity_integration::init_capacity_management_managed().await; let service_runtime = init_startup_runtime_services( diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 25b200fb2..3c32ad41d 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -28,9 +28,9 @@ use crate::storage::storage_api::rpc_consumer::node_service::{ SCANNER_PUBLICATION_LEASE_TTL_MS, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _, StorageResult, all_local_disk_path, find_local_disk_by_ref, reload_transition_tier_config, }; -use crate::storage::storage_api::runtime_sources_consumer::{EndpointServerPools, runtime_sources}; +use crate::storage::storage_api::runtime_sources_consumer::{EndpointServerPools, ServerContextSlot, runtime_sources}; use crate::storage::storage_api::{ - sign_tonic_rpc_response_proof, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, + BootstrapLocalTarget, sign_tonic_rpc_response_proof, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, }; use bytes::Bytes; @@ -482,6 +482,97 @@ mod metrics; pub struct NodeService { local_peer: LocalPeerS3Client, context: Option>, + server_ctx: Option>, +} + +enum LocalMutationTarget { + Ready(Arc), + Bootstrap(BootstrapLocalTarget), + Unbound, +} + +#[cfg(feature = "e2e-test-hooks")] +pub(crate) mod rename_target_capture_test_hook { + use super::LocalMutationTarget; + use rustfs_protos::proto_gen::node_service::RenameDataRequest; + use std::sync::{LazyLock, Mutex}; + use tokio::sync::oneshot; + use uuid::Uuid; + + struct Hook { + id: Uuid, + disk: String, + volume: String, + path: String, + captured: oneshot::Sender, + release: oneshot::Receiver<()>, + } + + static HOOK: LazyLock>> = LazyLock::new(|| Mutex::new(None)); + + /// One exact signed rename paused after its listener target was captured. + /// Dropping the handle removes an unused hook and releases an entered one. + pub struct RenameTargetCapturePause { + id: Uuid, + captured: oneshot::Receiver, + release: Option>, + } + + impl RenameTargetCapturePause { + pub async fn wait_until_captured(&mut self) -> bool { + (&mut self.captured) + .await + .expect("matching rename must report its actual captured target") + } + } + + impl Drop for RenameTargetCapturePause { + fn drop(&mut self) { + let unused = HOOK + .lock() + .expect("rename capture hook lock") + .take_if(|hook| hook.id == self.id); + drop(unused); + if let Some(release) = self.release.take() { + let _ = release.send(()); + } + } + } + + pub fn pause_rename_after_target_capture(disk: &str, volume: &str, path: &str) -> RenameTargetCapturePause { + let id = Uuid::new_v4(); + let (captured_tx, captured) = oneshot::channel(); + let (release, release_rx) = oneshot::channel(); + let mut active = HOOK.lock().expect("rename capture hook lock"); + if active.is_some() { + drop(active); + panic!("only one rename capture hook may be active"); + } + *active = Some(Hook { + id, + disk: disk.to_owned(), + volume: volume.to_owned(), + path: path.to_owned(), + captured: captured_tx, + release: release_rx, + }); + RenameTargetCapturePause { + id, + captured, + release: Some(release), + } + } + + pub(super) async fn wait(target: &LocalMutationTarget, request: &RenameDataRequest) { + let hook = { + let mut active = HOOK.lock().expect("rename capture hook lock"); + active.take_if(|hook| hook.disk == request.disk && hook.volume == request.dst_volume && hook.path == request.dst_path) + }; + if let Some(hook) = hook { + let _ = hook.captured.send(matches!(target, LocalMutationTarget::Bootstrap(_))); + let _ = hook.release.await; + } + } } impl std::fmt::Debug for NodeService { @@ -507,7 +598,19 @@ pub fn make_server() -> NodeService { pub fn make_server_for_context(context: Option>) -> NodeService { let local_peer = LocalPeerS3Client::new(None, None); - NodeService { local_peer, context } + NodeService { + local_peer, + context, + server_ctx: None, + } +} + +pub(crate) fn make_server_for_slot(server_ctx: Arc) -> NodeService { + // Unrelated RPCs retain their existing context policy. Target mutations + // resolve exclusively through this listener slot on each request. + let mut service = make_server(); + service.server_ctx = Some(server_ctx); + service } #[derive(Clone, Debug, Default)] @@ -1074,6 +1177,24 @@ impl heal_control_service_server::HealControlService for HealControlRpcService { } impl NodeService { + fn local_mutation_target(&self) -> LocalMutationTarget { + if let Some(slot) = &self.server_ctx { + // Capture exactly once per request, not at connection acceptance. + // A captured Bootstrap request cannot upgrade across a later await. + if let Some(store) = slot.installed_object_store() { + LocalMutationTarget::Ready(store) + } else if let Some(target) = slot.bootstrap_target() { + LocalMutationTarget::Bootstrap(target) + } else { + LocalMutationTarget::Unbound + } + } else if let Some(context) = &self.context { + LocalMutationTarget::Ready(context.object_store()) + } else { + LocalMutationTarget::Unbound + } + } + fn resolve_object_store(&self) -> Option> { let context = self.context.clone().or_else(runtime_sources::current_app_context); runtime_sources::current_object_store_handle_for_context(context.as_deref()) @@ -2680,6 +2801,7 @@ mod tests { validate_admin_heal_control_start, }; use crate::storage::rpc::node_service::heal::heal_topology_fingerprint; + use crate::storage::storage_api::ecstore_disk::DiskAPI as _; use crate::storage::storage_api::rpc_consumer::node_service::{DiskError, HealBucketInfo}; use crate::storage::storage_api::set_tonic_canonical_body_digest; use crate::storage::storage_api::{ @@ -4660,6 +4782,687 @@ mod tests { assert!(rename_response.error.is_some()); } + struct TargetRpcFixture { + _root: tempfile::TempDir, + env: rustfs_test_utils::TestECStoreEnv, + instance: Arc, + context: Arc, + iam: Arc>, + } + + async fn target_rpc_fixture() -> TargetRpcFixture { + super::timeout(Duration::from_secs(90), async { + let root = tempfile::tempdir().expect("target RPC root"); + let env = rustfs_test_utils::TestECStoreEnv::builder() + .base_dir(root.path()) + .init_bucket_metadata(false) + .build() + .await; + ObjectStore::new(env.ecstore.clone()) + .save_iam_config(serde_json::json!({"version": 1}), format!("{}/format.json", *IAM_CONFIG_PREFIX)) + .await + .expect("seed real IAM format"); + let iam = rustfs_iam::build_iam_sys(env.ecstore.clone()) + .await + .expect("build fixture IAM"); + let context = Arc::new(crate::runtime_sources::AppContext::with_default_interfaces( + env.ecstore.clone(), + iam.clone(), + Arc::new(KmsServiceManager::new()), + )); + let instance = crate::storage::storage_api::bootstrap_instance_ctx(); + assert!( + super::BootstrapLocalTarget::new(instance.clone()).is_for_store(&env.ecstore), + "the standard builder must use this exact instance context" + ); + super::timeout(Duration::from_secs(10), async { + while env.ecstore.scanner_data_usage_publication_blocked().await { + tokio::task::yield_now().await; + } + }) + .await + .expect("startup namespace commits drain before test"); + TargetRpcFixture { + _root: root, + env, + instance, + context, + iam, + } + }) + .await + .expect("bounded real fixture initialization") + } + + async fn stage_target_rpc(fixture: &TargetRpcFixture) -> (super::DiskStore, rustfs_filemeta::FileInfo, Vec) { + use crate::storage::storage_api::ecstore_disk::{DiskAPI, ReadOptions}; + let set = fixture + .env + .ecstore + .all_set_disks() + .into_iter() + .next() + .expect("target erasure set"); + let disk = set.disks.read().await.iter().find_map(Clone::clone).expect("local target"); + let mut fi = rustfs_filemeta::FileInfo::new("destination", 1, 0); + fi.erasure.index = 1; + fi.version_id = Some(Uuid::new_v4()); + fi.mod_time = Some(OffsetDateTime::now_utc()); + fi.size = 17; + fi.parts = vec![rustfs_filemeta::ObjectPartInfo { + number: 1, + size: 17, + actual_size: 17, + ..Default::default() + }]; + fi.data = Some(Bytes::from_static(b"target-rpc-inline")); + fi.set_inline_data(); + disk.make_volume("target-rpc").await.expect("target volume"); + disk.write_metadata("target-rpc", "target-rpc", "staged", fi.clone()) + .await + .expect("stage real inline body"); + let read = disk + .read_version( + "target-rpc", + "target-rpc", + "staged", + &fi.version_id.expect("version").to_string(), + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("read staged body before mutation"); + assert_eq!(read.data, fi.data); + let before = tokio::fs::read(disk.path().join("target-rpc/staged/xl.meta")) + .await + .expect("staged bytes"); + (disk, fi, before) + } + + fn target_rename_request(disk: &super::DiskStore, fi: &rustfs_filemeta::FileInfo) -> Request { + let mut request = Request::new(RenameDataRequest { + disk: disk.endpoint().to_string(), + src_volume: "target-rpc".to_string(), + src_path: "staged".to_string(), + dst_volume: "target-rpc".to_string(), + dst_path: "destination".to_string(), + file_info: serde_json::to_string(fi).expect("real FileInfo JSON"), + ..Default::default() + }); + let body = rustfs_protos::canonical_rename_data_request_body(request.get_ref()).expect("canonical target body"); + set_tonic_canonical_body_digest(&mut request, &body).expect("body digest"); + // Direct-handler precondition only; this does not stand in for wire authentication. + mark_v2_authenticated(&mut request); + request + } + + #[tokio::test] + async fn target_slot_rejects_mismatched_and_repeated_install_before_global_publication() { + let fixture = target_rpc_fixture().await; + assert!( + crate::runtime_sources::current_app_context().is_none(), + "requires a separate nextest process" + ); + let wrong = super::ServerContextSlot::with_instance_context(crate::storage::storage_api::new_instance_ctx()); + let error = crate::runtime_sources::AppContext::ensure_startup_after_iam( + fixture.env.ecstore.clone(), + Arc::new(KmsServiceManager::new()), + &wrong, + fixture.iam.clone(), + ) + .expect_err("mismatched startup must fail"); + assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput); + assert!(wrong.installed_app_context().is_none()); + assert!( + crate::runtime_sources::current_app_context().is_none(), + "failed install must not publish globally" + ); + assert!(!wrong.install(fixture.context.clone()), "bool adapter cannot bypass identity checks"); + let slot = super::ServerContextSlot::with_instance_context(fixture.instance.clone()); + crate::runtime_sources::AppContext::ensure_startup_after_iam( + fixture.env.ecstore.clone(), + Arc::new(KmsServiceManager::new()), + &slot, + fixture.iam.clone(), + ) + .expect("matching startup installation"); + let installed = slot.installed_app_context().expect("installed A"); + assert!(Arc::ptr_eq( + &crate::runtime_sources::current_app_context().expect("published A"), + &installed + )); + assert_eq!( + slot.try_install(installed.clone()) + .expect_err("same Arc is still a duplicate") + .kind(), + std::io::ErrorKind::AlreadyExists + ); + assert!(!slot.install(installed.clone())); + assert!(Arc::ptr_eq(&slot.installed_app_context().expect("first winner retained"), &installed)); + } + + #[tokio::test] + async fn target_slot_captures_bootstrap_once_and_next_request_observes_ready() { + let fixture = target_rpc_fixture().await; + let (disk, fi, before) = stage_target_rpc(&fixture).await; + let slot = super::ServerContextSlot::with_instance_context(fixture.instance.clone()); + let service = super::make_server_for_slot(slot.clone()); + let captured = service.local_mutation_target(); + slot.try_install(fixture.context.clone()) + .expect("install after the request captures bootstrap"); + let super::LocalMutationTarget::Bootstrap(target) = captured else { + panic!("pre-install request must capture bootstrap"); + }; + assert!( + target + .rename_local_data( + &disk.endpoint().to_string(), + ("target-rpc", "staged"), + &fi, + ("target-rpc", "destination"), + None + ) + .await + .is_err(), + "captured request cannot acquire Ready privileges" + ); + assert_eq!( + tokio::fs::read(disk.path().join("target-rpc/staged/xl.meta")) + .await + .expect("original source"), + before + ); + assert!(!disk.path().join("target-rpc/destination").exists()); + assert!( + matches!(service.local_mutation_target(), super::LocalMutationTarget::Ready(_)), + "the same service must read the installed slot for its next request" + ); + let result = service + .rename_data(target_rename_request(&disk, &fi)) + .await + .expect("ready handler") + .into_inner(); + assert!(result.success, "{:?}", result.error); + } + + #[tokio::test] + async fn target_unbound_slot_never_mutates_a_published_global_store() { + let fixture = target_rpc_fixture().await; + let (disk, fi, before) = stage_target_rpc(&fixture).await; + let published = crate::runtime_sources::publish_test_app_context(fixture.context.clone()); + assert!(Arc::ptr_eq(&published, &fixture.context)); + let service = super::make_server_for_slot(super::ServerContextSlot::new()); + let result = service + .rename_data(target_rename_request(&disk, &fi)) + .await + .expect("handler reply") + .into_inner(); + assert!(!result.success); + assert!(result.error.is_some()); + assert_eq!( + tokio::fs::read(disk.path().join("target-rpc/staged/xl.meta")) + .await + .expect("source remains"), + before + ); + assert!(!disk.path().join("target-rpc/destination").exists()); + assert!(!fixture.env.ecstore.scanner_data_usage_publication_blocked().await); + } + + #[tokio::test] + async fn target_undo_rejects_force_delete_marker_before_mutation() { + let fixture = target_rpc_fixture().await; + let (disk, fi, before) = stage_target_rpc(&fixture).await; + let service = make_server_for_context(Some(fixture.context.clone())); + let opts = crate::storage::storage_api::ecstore_disk::DeleteOptions { + undo_write: true, + ..Default::default() + }; + let mut request = Request::new(DeleteVersionRequest { + disk: disk.endpoint().to_string(), + volume: "target-rpc".to_string(), + path: "staged".to_string(), + file_info: serde_json::to_string(&fi).expect("FileInfo"), + opts: serde_json::to_string(&opts).expect("opts"), + force_del_marker: true, + ..Default::default() + }); + let body = rustfs_protos::canonical_delete_version_request_body(request.get_ref()).expect("canonical undo body"); + set_tonic_canonical_body_digest(&mut request, &body).expect("body digest"); + mark_v2_authenticated(&mut request); + let result = service.delete_version(request).await.expect("handler reply").into_inner(); + assert!(!result.success); + assert!(result.error.is_some()); + assert_eq!( + tokio::fs::read(disk.path().join("target-rpc/staged/xl.meta")) + .await + .expect("source remains"), + before + ); + assert!(!fixture.env.ecstore.scanner_data_usage_publication_blocked().await); + } + + #[cfg(not(windows))] + #[tokio::test] + async fn target_handler_cancellation_retains_namespace_through_physical_rename() { + use crate::storage::storage_api::{ + LocalPublicationPause, LocalPublicationStage, + ecstore_disk::{DiskAPI, ReadOptions}, + }; + let fixture = target_rpc_fixture().await; + let (disk, fi, _) = stage_target_rpc(&fixture).await; + let slot = super::ServerContextSlot::with_instance_context(fixture.instance.clone()); + slot.try_install(fixture.context.clone()).expect("ready target"); + let service = super::make_server_for_slot(slot); + let mut pause = + LocalPublicationPause::install(&disk, "target-rpc", "destination/xl.meta", LocalPublicationStage::PreparedRename) + .expect("install scoped physical pause"); + let mut handler = Box::pin(service.rename_data(target_rename_request(&disk, &fi))); + super::timeout(Duration::from_secs(10), async { + tokio::select! { + result = &mut handler => panic!("handler completed before physical entry: {result:?}"), + entered = pause.entered() => entered.expect("physical executor entered"), + } + }) + .await + .expect("bounded physical entry"); + drop(handler); + assert!( + fixture.env.ecstore.scanner_data_usage_publication_blocked().await, + "dropping the actual target handler must not release its physical owner" + ); + drop(pause); + super::timeout(Duration::from_secs(10), async { + while fixture.env.ecstore.scanner_data_usage_publication_blocked().await { + tokio::task::yield_now().await; + } + }) + .await + .expect("physical owner must drain"); + let read = disk + .read_version( + "target-rpc", + "target-rpc", + "destination", + &fi.version_id.expect("version").to_string(), + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("read real late commit"); + assert_eq!(read.data, fi.data); + } + + #[cfg(not(windows))] + #[tokio::test] + async fn target_undo_handler_cancellation_retains_owner_until_backup_restoration() { + use crate::storage::storage_api::{ + LocalPublicationPause, LocalPublicationStage, + ecstore_disk::{DeleteOptions, DiskAPI, ReadOptions}, + }; + let fixture = target_rpc_fixture().await; + let (disk, fi, _) = stage_target_rpc(&fixture).await; + let mut old = fi.clone(); + old.data = Some(Bytes::from_static(b"previous-rpc-body")); + assert_eq!(old.data.as_ref().expect("old body").len(), 17); + disk.write_metadata("target-rpc", "target-rpc", "destination", old.clone()) + .await + .expect("old actual version"); + let old_bytes = tokio::fs::read(disk.path().join("target-rpc/destination/xl.meta")) + .await + .expect("old metadata bytes"); + let committed = fixture + .env + .ecstore + .rename_local_data( + &disk.endpoint().to_string(), + ("target-rpc", "staged"), + &fi, + ("target-rpc", "destination"), + None, + ) + .await + .expect("real overwrite creates rollback backup"); + let opts = DeleteOptions { + undo_write: true, + old_data_dir: Some(committed.rollback_data_dir.expect("real rollback backup")), + ..Default::default() + }; + let service = make_server_for_context(Some(fixture.context.clone())); + let mut request = Request::new(DeleteVersionRequest { + disk: disk.endpoint().to_string(), + volume: "target-rpc".to_string(), + path: "destination".to_string(), + file_info: serde_json::to_string(&fi).expect("FileInfo"), + opts: serde_json::to_string(&opts).expect("undo options"), + ..Default::default() + }); + let body = rustfs_protos::canonical_delete_version_request_body(request.get_ref()).expect("canonical undo body"); + set_tonic_canonical_body_digest(&mut request, &body).expect("body digest"); + mark_v2_authenticated(&mut request); + let mut pause = LocalPublicationPause::install(&disk, "target-rpc", "destination/xl.meta", LocalPublicationStage::Rename) + .expect("pause actual backup restoration"); + let mut handler = Box::pin(service.delete_version(request)); + super::timeout(Duration::from_secs(10), async { + tokio::select! { + result = &mut handler => panic!("undo completed before physical entry: {result:?}"), + entered = pause.entered() => entered.expect("physical restore entered"), + } + }) + .await + .expect("bounded physical restore entry"); + drop(handler); + assert!(fixture.env.ecstore.scanner_data_usage_publication_blocked().await); + drop(pause); + super::timeout(Duration::from_secs(10), async { + while fixture.env.ecstore.scanner_data_usage_publication_blocked().await { + tokio::task::yield_now().await; + } + }) + .await + .expect("restore owner drains"); + assert_eq!( + tokio::fs::read(disk.path().join("target-rpc/destination/xl.meta")) + .await + .expect("restored bytes"), + old_bytes + ); + let read = disk + .read_version( + "target-rpc", + "target-rpc", + "destination", + &fi.version_id.expect("version").to_string(), + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("restored readable version"); + assert_eq!(read.data, old.data); + } + + #[tokio::test] + async fn rename_data_same_uuid_uses_captured_instance_instead_of_global_disk() { + use crate::storage::storage_api::{ + ECStore, + ecstore_disk::{DiskAPI, RUSTFS_META_BUCKET, ReadOptions}, + init_local_disks_with_instance_ctx, new_instance_ctx, read_config_no_lock, + }; + use rustfs_filemeta::{FileInfo, ObjectPartInfo}; + use tokio_util::sync::CancellationToken; + + async fn build_store(root: &std::path::Path) -> Arc { + let mut endpoints = Vec::new(); + for index in 0..4 { + let path = root.join(format!("disk{index}")); + tokio::fs::create_dir_all(&path).await.expect("create instance disk"); + let mut endpoint = Endpoint::try_from(path.to_str().expect("UTF-8 disk path")).expect("local endpoint"); + endpoint.set_pool_index(0); + endpoint.set_set_index(0); + endpoint.set_disk_index(index); + endpoints.push(endpoint); + } + let pools = EndpointServerPools(vec![PoolEndpoints { + legacy: false, + set_count: 1, + drives_per_set: 4, + endpoints: Endpoints::from(endpoints), + cmd_line: "namespace-target-context".to_string(), + platform: "test".to_string(), + }]); + let instance = new_instance_ctx(); + init_local_disks_with_instance_ctx(&instance, pools.clone()) + .await + .expect("register this instance's real disks"); + // Match the isolated ECStore fixtures: startup still runs, while + // unrelated background recovery is cancelled for this process. + let shutdown = CancellationToken::new(); + shutdown.cancel(); + ECStore::new_with_instance_ctx("127.0.0.1:0".parse().expect("local address"), pools, shutdown, instance) + .await + .expect("initialize isolated ECStore") + } + + async fn context(store: &Arc) -> Arc { + ObjectStore::new(store.clone()) + .save_iam_config(serde_json::json!({"version": 1}), format!("{}/format.json", *IAM_CONFIG_PREFIX)) + .await + .expect("seed isolated IAM format"); + let iam = rustfs_iam::build_iam_sys(store.clone()).await.expect("build isolated IAM"); + Arc::new(crate::runtime_sources::AppContext::with_default_interfaces( + store.clone(), + iam, + Arc::new(KmsServiceManager::new()), + )) + } + + async fn internal_snapshot(root: &std::path::Path) -> std::collections::BTreeMap>> { + let mut snapshot = std::collections::BTreeMap::new(); + let mut directories = (0..4) + .map(|index| std::path::PathBuf::from(format!("disk{index}/{RUSTFS_META_BUCKET}"))) + .collect::>(); + while let Some(relative) = directories.pop() { + let mut entries = tokio::fs::read_dir(root.join(&relative)) + .await + .expect("read internal snapshot directory"); + snapshot.insert(relative.clone(), None); + while let Some(entry) = entries.next_entry().await.expect("read internal snapshot entry") { + let path = relative.join(entry.file_name()); + let file_type = entry.file_type().await.expect("read internal snapshot entry type"); + if file_type.is_dir() { + directories.push(path); + } else { + assert!(file_type.is_file(), "fixture snapshot must contain only directories and regular files"); + snapshot.insert(path, Some(tokio::fs::read(entry.path()).await.expect("read snapshot file bytes"))); + } + } + } + snapshot + } + + fn file_info(object: &str, version: Uuid, body: Bytes) -> FileInfo { + let mut fi = FileInfo::new(object, 1, 0); + fi.erasure.index = 1; + fi.name = object.to_string(); + fi.version_id = Some(version); + fi.size = i64::try_from(body.len()).expect("small fixture body"); + fi.parts = vec![ObjectPartInfo { + number: 1, + size: body.len(), + actual_size: fi.size, + ..Default::default() + }]; + fi.data = Some(body); + fi.set_inline_data(); + fi.mod_time = Some(OffsetDateTime::now_utc()); + fi + } + + // Both global publications are first-writer-wins. Run this fixture in + // its own nextest process; do not reset or replace another test's state. + assert!( + crate::runtime_sources::current_app_context().is_none(), + "requires an unpublished AppContext" + ); + super::timeout(Duration::from_secs(90), async { + let root_b = tempfile::tempdir().expect("instance B directory"); + let root_a = tempfile::tempdir().expect("instance A directory"); + let store_b = build_store(root_b.path()).await; + let context_b = context(&store_b).await; + let published = crate::runtime_sources::publish_test_app_context(context_b.clone()); + assert!(Arc::ptr_eq(&published, &context_b), "B must win the process AppContext publication"); + + // Existing formats require their committed pool metadata on restart. + // Copy the complete internal trees, including erasure part data, + // without editing disk IDs, cluster identity, epochs or pool topology. + super::timeout(Duration::from_secs(10), async { + while store_b.scanner_data_usage_publication_blocked().await { + tokio::task::yield_now().await; + } + }) + .await + .expect("B startup namespace commits must drain before its snapshot"); + let snapshot_generation = store_b.scanner_namespace_mutation_generation(); + let pool_config = read_config_no_lock(store_b.clone(), "pool.bin") + .await + .expect("read B's actually committed pool metadata"); + let pool_identity = read_config_no_lock(store_b.clone(), "pool.bin.identity") + .await + .expect("read B's actually committed pool identity"); + let snapshot = internal_snapshot(root_b.path()).await; + for (relative, contents) in &snapshot { + let target = root_a.path().join(relative); + match contents { + None => tokio::fs::create_dir_all(target).await.expect("copy internal directory"), + Some(bytes) => tokio::fs::write(target, bytes).await.expect("copy complete internal file"), + } + } + assert_eq!(internal_snapshot(root_a.path()).await, snapshot, "A must receive the complete physical snapshot"); + assert_eq!(internal_snapshot(root_b.path()).await, snapshot, "B's source snapshot must remain unchanged"); + assert!(!store_b.scanner_data_usage_publication_blocked().await); + assert_eq!(store_b.scanner_namespace_mutation_generation(), snapshot_generation); + let store_a = build_store(root_a.path()).await; + assert_eq!( + read_config_no_lock(store_a.clone(), "pool.bin").await.expect("read A's restarted pool metadata"), + pool_config, + "A must load the same committed topology without a bootstrap rewrite" + ); + assert_eq!( + read_config_no_lock(store_a.clone(), "pool.bin.identity") + .await + .expect("read A's restarted pool identity"), + pool_identity, + "A must preserve the initialized cluster identity and epoch" + ); + let service = make_server_for_context(Some(context(&store_a).await)); + assert!(Arc::ptr_eq(&service.resolve_object_store().expect("captured store"), &store_a)); + assert!(Arc::ptr_eq( + &crate::runtime_sources::current_object_store_handle().expect("global store"), + &store_b + )); + let disk_a = store_a.disk_map[&0][0].as_ref().expect("A disk zero").clone(); + let disk_b = store_b.disk_map[&0][0].as_ref().expect("B disk zero").clone(); + assert!(disk_a.is_local() && disk_b.is_local()); + assert!(!Arc::ptr_eq(&disk_a, &disk_b)); + let disk_id = disk_a.get_disk_id().await.expect("A disk ID").expect("formatted A disk"); + assert!(!disk_id.is_nil()); + assert_eq!(disk_b.get_disk_id().await.expect("B disk ID"), Some(disk_id)); + let global_disk = super::find_local_disk_by_ref(&disk_id.to_string()) + .await + .expect("global UUID lookup must resolve B before the request"); + assert!(Arc::ptr_eq(&global_disk, &disk_b)); + + let volume = "namespace-target-context"; + let object = "destination"; + let staging = "staged"; + let version = Uuid::new_v4(); + let new_body = Bytes::from_static(b"committed-through-captured-A"); + let new_fi = file_info(object, version, new_body.clone()); + let opts = ReadOptions { read_data: true, ..Default::default() }; + for (disk, old_body) in [ + (&disk_a, Bytes::from_static(b"old-body-A")), + (&disk_b, Bytes::from_static(b"old-body-B")), + ] { + disk.make_volume(volume).await.expect("create destination volume"); + disk.write_metadata(volume, volume, object, file_info(object, version, old_body.clone())) + .await + .expect("write real old object metadata and inline body"); + disk.write_metadata(volume, volume, staging, new_fi.clone()) + .await + .expect("stage identical valid metadata on both physical disks"); + let seeded = disk + .read_version(volume, volume, object, &version.to_string(), &opts) + .await + .expect("decode seeded inline object before invoking the handler"); + assert_eq!(seeded.data, Some(old_body), "the real reader must return the seeded body"); + } + let a_meta = disk_a.path().join(volume).join(object).join("xl.meta"); + let b_meta = disk_b.path().join(volume).join(object).join("xl.meta"); + let a_staging = disk_a.path().join(volume).join(staging).join("xl.meta"); + let b_staging = disk_b.path().join(volume).join(staging).join("xl.meta"); + let a_before = tokio::fs::read(&a_meta).await.expect("A old metadata bytes"); + let b_before = tokio::fs::read(&b_meta).await.expect("B old metadata bytes"); + let b_staging_before = tokio::fs::read(&b_staging).await.expect("B staged metadata bytes"); + assert!(tokio::fs::try_exists(&a_staging).await.expect("A staging exists")); + assert_ne!(a_before, b_before, "the old on-disk bodies must distinguish A from B"); + super::timeout(Duration::from_secs(10), async { + while store_a.scanner_data_usage_publication_blocked().await + || store_b.scanner_data_usage_publication_blocked().await + { + tokio::task::yield_now().await; + } + }) + .await + .expect("startup namespace commits must drain before measuring the handler"); + let generation_before = ( + store_a.scanner_namespace_mutation_generation(), + store_b.scanner_namespace_mutation_generation(), + ); + + let mut request = Request::new(RenameDataRequest { + disk: disk_id.to_string(), + src_volume: volume.to_string(), + src_path: staging.to_string(), + dst_volume: volume.to_string(), + dst_path: object.to_string(), + file_info: serde_json::to_string(&new_fi).expect("encode real FileInfo"), + file_info_bin: Vec::new().into(), + scanner_publication_lease_token: Vec::new().into(), + }); + let body = rustfs_protos::canonical_rename_data_request_body(request.get_ref()).expect("canonical rename body"); + set_tonic_canonical_body_digest(&mut request, &body).expect("body-bound handler request"); + mark_v2_authenticated(&mut request); + let response = super::timeout(Duration::from_secs(10), service.rename_data(request)) + .await + .expect("real rename handler must finish within ten seconds") + .expect("rename handler response") + .into_inner(); + assert!(response.success, "the valid staged rename must execute: {:?}", response.error); + + let a_after = disk_a + .read_version(volume, volume, object, &version.to_string(), &opts) + .await + .expect("read A's physical object after rename"); + let b_after = disk_b + .read_version(volume, volume, object, &version.to_string(), &opts) + .await + .expect("read B's physical object after rename"); + let a_bytes_after = tokio::fs::read(&a_meta).await.expect("A metadata after rename"); + let b_bytes_after = tokio::fs::read(&b_meta).await.expect("B metadata after rename"); + let b_staging_after = tokio::fs::read(&b_staging).await.ok(); + let generation_after = ( + store_a.scanner_namespace_mutation_generation(), + store_b.scanner_namespace_mutation_generation(), + ); + let pending_after = ( + store_a.scanner_data_usage_publication_blocked().await, + store_b.scanner_data_usage_publication_blocked().await, + ); + for disk in store_a.disk_map.values().chain(store_b.disk_map.values()).flatten().flatten() { + disk.close().await.expect("close real fixture disk before assertions and directory cleanup"); + } + assert_eq!( + a_after.data, + Some(new_body), + "RenameData must commit to captured A, even when global B owns the same UUID; B body={:?}, generations={generation_before:?}->{generation_after:?}, pending={pending_after:?}", + b_after.data + ); + assert_ne!(a_bytes_after, a_before, "A metadata must actually be replaced"); + assert_eq!(b_after.data, Some(Bytes::from_static(b"old-body-B")), "B body must remain unchanged"); + assert_eq!(b_bytes_after, b_before, "B metadata must remain byte-for-byte unchanged"); + assert_eq!(b_staging_after, Some(b_staging_before), "B staging must not be consumed"); + assert_eq!(pending_after, (false, false), "both stores must reach a stable terminal state"); + }) + .await + .expect("two-instance handler fixture must finish within ninety seconds"); + } + #[tokio::test] async fn test_make_volumes_invalid_disk() { let service = create_test_node_service(); diff --git a/rustfs/src/storage/rpc/node_service/disk.rs b/rustfs/src/storage/rpc/node_service/disk.rs index 19fb86c80..fb237b31c 100644 --- a/rustfs/src/storage/rpc/node_service/disk.rs +++ b/rustfs/src/storage/rpc/node_service/disk.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::NodeService; +use super::{LocalMutationTarget, NodeService}; use crate::storage::storage_api::rpc_consumer::node_service::{ BatchReadVersionReq, BatchReadVersionResp, DeleteOptions, DiskError, DiskInfoOptions, FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions, StorageDiskRpcExt as _, UpdateMetadataOpts, validate_batch_read_version_item_count, @@ -39,6 +39,69 @@ use tonic::{Request, Response, Status}; use tracing::debug; use uuid::Uuid; +#[cfg(feature = "e2e-test-hooks")] +fn startup_cas_rename_observation( + target: &LocalMutationTarget, + request: &RenameDataRequest, + file_info: &FileInfo, +) -> Option { + use sha2::{Digest, Sha256}; + if request.dst_volume != ".rustfs.sys" || !matches!(request.dst_path.as_str(), "pool.bin" | "pool.bin.identity") { + return None; + } + let nonce = uuid::Uuid::parse_str(&std::env::var("RUSTFS_E2E_STARTUP_CAS_NONCE").ok()?).ok()?; + let body = rustfs_protos::canonical_rename_data_request_body(request).ok()?; + Some(serde_json::json!({ + "kind": "receiver", "nonce": nonce, "pid": std::process::id(), + "target": match target { LocalMutationTarget::Ready(_) => "ready", LocalMutationTarget::Bootstrap(_) => "bootstrap", LocalMutationTarget::Unbound => "unbound" }, + "disk": request.disk, "src_volume": request.src_volume, "src_path": request.src_path, + "dst_volume": request.dst_volume, "dst_path": request.dst_path, + "body_sha256": rustfs_utils::crypto::hex(Sha256::digest(body)), + "etag": file_info.metadata.get("etag"), + "mod_time": file_info.mod_time.map(|time| time.unix_timestamp_nanos().to_string()), + })) +} + +impl LocalMutationTarget { + async fn rename_local_data( + &self, + disk_ref: &str, + source: (&str, &str), + fi: &FileInfo, + destination: (&str, &str), + scanner_token: Option, + ) -> Result { + match self { + Self::Ready(store) => { + store + .rename_local_data(disk_ref, source, fi, destination, scanner_token) + .await + } + Self::Bootstrap(target) => { + target + .rename_local_data(disk_ref, source, fi, destination, scanner_token) + .await + } + Self::Unbound => Err(DiskError::other("target disk instance is unavailable")), + } + } + + async fn undo_local_write( + &self, + disk_ref: &str, + volume: &str, + path: &str, + fi: FileInfo, + opts: DeleteOptions, + ) -> Result<(), DiskError> { + match self { + Self::Ready(store) => store.undo_local_write(disk_ref, volume, path, fi, opts).await, + Self::Bootstrap(target) => target.undo_local_write(disk_ref, volume, path, fi, opts).await, + Self::Unbound => Err(DiskError::other("target disk instance is unavailable")), + } + } +} + /// Initial capacity hint (bytes) for typical small msgpack requests and responses. const MSGPACK_ENCODE_CAPACITY_HINT: usize = 512; const FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT: usize = 1024; @@ -670,55 +733,59 @@ impl NodeService { "delete_version", )?; let request = request.into_inner(); - if let Some(disk) = self.find_disk(&request.disk).await { - let file_info = match decode_msgpack_or_json::(&request.file_info_bin, &request.file_info, "FileInfo") { - Ok(file_info) => file_info, - Err(err) => { - return Ok(Response::new(DeleteVersionResponse { - success: false, - raw_file_info: "".to_string(), - error: Some(DiskError::other(format!("decode FileInfo failed: {err}")).into()), - })); - } - }; - let opts = match decode_msgpack_or_json::(&request.opts_bin, &request.opts, "DeleteOptions") { - Ok(opts) => opts, - Err(err) => { - return Ok(Response::new(DeleteVersionResponse { - success: false, - raw_file_info: "".to_string(), - error: Some(DiskError::other(format!("decode DeleteOptions failed: {err}")).into()), - })); - } - }; - match disk - .delete_version(&request.volume, &request.path, file_info, request.force_del_marker, opts) + let file_info = match decode_msgpack_or_json::(&request.file_info_bin, &request.file_info, "FileInfo") { + Ok(file_info) => file_info, + Err(err) => { + return Ok(Response::new(DeleteVersionResponse { + success: false, + raw_file_info: "".to_string(), + error: Some(DiskError::other(format!("decode FileInfo failed: {err}")).into()), + })); + } + }; + let opts = match decode_msgpack_or_json::(&request.opts_bin, &request.opts, "DeleteOptions") { + Ok(opts) => opts, + Err(err) => { + return Ok(Response::new(DeleteVersionResponse { + success: false, + raw_file_info: "".to_string(), + error: Some(DiskError::other(format!("decode DeleteOptions failed: {err}")).into()), + })); + } + }; + let result = if opts.undo_write { + if request.force_del_marker { + Err(DiskError::other("undo_write cannot force a delete marker")) + } else { + let target = self.local_mutation_target(); + target + .undo_local_write(&request.disk, &request.volume, &request.path, file_info, opts) + .await + } + } else if let Some(disk) = self.find_disk(&request.disk).await { + disk.delete_version(&request.volume, &request.path, file_info, request.force_del_marker, opts) .await - { - Ok(raw_file_info) => match serde_json::to_string(&raw_file_info) { - Ok(raw_file_info) => Ok(Response::new(DeleteVersionResponse { - success: true, - raw_file_info, - error: None, - })), - Err(err) => Ok(Response::new(DeleteVersionResponse { - success: false, - raw_file_info: "".to_string(), - error: Some(DiskError::other(format!("encode data failed: {err}")).into()), - })), - }, + } else { + Err(DiskError::other("cannot find disk")) + }; + match result { + Ok(raw_file_info) => match serde_json::to_string(&raw_file_info) { + Ok(raw_file_info) => Ok(Response::new(DeleteVersionResponse { + success: true, + raw_file_info, + error: None, + })), Err(err) => Ok(Response::new(DeleteVersionResponse { success: false, raw_file_info: "".to_string(), - error: Some(err.into()), + error: Some(DiskError::other(format!("encode data failed: {err}")).into()), })), - } - } else { - Ok(Response::new(DeleteVersionResponse { + }, + Err(err) => Ok(Response::new(DeleteVersionResponse { success: false, raw_file_info: "".to_string(), - error: Some(DiskError::other("cannot find disk".to_string()).into()), - })) + error: Some(err.into()), + })), } } @@ -1206,98 +1273,70 @@ impl NodeService { "rename_data", )?; let request = request.into_inner(); - if let Some(disk) = self.find_disk(&request.disk).await { - let decoded_file_info = match decode_rename_data_request_file_info(&request.file_info_bin, &request.file_info) { - Ok(file_info) => file_info, - Err(err) => { - return Ok(Response::new(RenameDataResponse { - success: false, - rename_data_resp: String::new(), - rename_data_resp_bin: Vec::new().into(), - error: Some(DiskError::other(format!("decode FileInfo failed: {err}")).into()), - })); - } - }; - let scanner_publication_lease_token = if request.scanner_publication_lease_token.is_empty() { - None - } else { - let token = Uuid::from_slice(&request.scanner_publication_lease_token) - .map_err(|_| Status::invalid_argument("scanner publication lease token must be a UUID"))?; - if token.is_nil() { - return Err(Status::invalid_argument("scanner publication lease token must not be nil")); - } - Some(token) - }; - // The target owns this read guard. It must span the complete - // disk rename, not merely the preflight, so a movement transition - // cannot restart after validation and before rename linearization. - let scanner_publication_lease_guard: Option> = - if let Some(token) = scanner_publication_lease_token { - let Some(store) = self.resolve_object_store() else { - return Ok(Response::new(RenameDataResponse { - success: false, - rename_data_resp: String::new(), - rename_data_resp_bin: Vec::new().into(), - error: Some(DiskError::other("scanner publication lease owner is unavailable").into()), - })); - }; - match store.acquire_scanner_publication_lease_guard(token).await { - Ok(guard) => Some(Arc::new(guard)), - Err(err) => { - return Ok(Response::new(RenameDataResponse { - success: false, - rename_data_resp: String::new(), - rename_data_resp_bin: Vec::new().into(), - error: Some(DiskError::other(err.to_string()).into()), - })); - } - } - } else { - None - }; - let request_decoded_from_msgpack = decoded_file_info.from_msgpack; - match disk - .rename_data_borrowed_with_fence_and_guard( - &request.src_volume, - &request.src_path, - &decoded_file_info.value, - &request.dst_volume, - &request.dst_path, - scanner_publication_lease_token, - scanner_publication_lease_guard, - ) - .await - { - Ok(rename_data_resp) => { - match encode_rename_data_response_payloads(&rename_data_resp, request_decoded_from_msgpack) { - Ok((rename_data_resp, rename_data_resp_bin)) => Ok(Response::new(RenameDataResponse { - success: true, - rename_data_resp, - rename_data_resp_bin: rename_data_resp_bin.into(), - error: None, - })), - Err(err) => Ok(Response::new(RenameDataResponse { - success: false, - rename_data_resp: String::new(), - rename_data_resp_bin: Vec::new().into(), - error: Some(err.into()), - })), - } - } + let target = self.local_mutation_target(); + #[cfg(feature = "e2e-test-hooks")] + super::rename_target_capture_test_hook::wait(&target, &request).await; + let decoded_file_info = match decode_rename_data_request_file_info(&request.file_info_bin, &request.file_info) { + Ok(file_info) => file_info, + Err(err) => { + return Ok(Response::new(RenameDataResponse { + success: false, + rename_data_resp: String::new(), + rename_data_resp_bin: Vec::new().into(), + error: Some(DiskError::other(format!("decode FileInfo failed: {err}")).into()), + })); + } + }; + let scanner_publication_lease_token = if request.scanner_publication_lease_token.is_empty() { + None + } else { + let token = Uuid::from_slice(&request.scanner_publication_lease_token) + .map_err(|_| Status::invalid_argument("scanner publication lease token must be a UUID"))?; + if token.is_nil() { + return Err(Status::invalid_argument("scanner publication lease token must not be nil")); + } + Some(token) + }; + let request_decoded_from_msgpack = decoded_file_info.from_msgpack; + #[cfg(feature = "e2e-test-hooks")] + let observation = startup_cas_rename_observation(&target, &request, &decoded_file_info.value); + let result = target + .rename_local_data( + &request.disk, + (&request.src_volume, &request.src_path), + &decoded_file_info.value, + (&request.dst_volume, &request.dst_path), + scanner_publication_lease_token, + ) + .await; + #[cfg(feature = "e2e-test-hooks")] + if let Some(mut observation) = observation { + observation["ok"] = serde_json::json!(result.is_ok()); + observation["error"] = serde_json::json!(result.as_ref().err().map(ToString::to_string)); + let line = format!("RUSTFS_E2E_STARTUP_CAS {observation}\n"); + let _ = std::io::Write::write_all(&mut std::io::stderr().lock(), line.as_bytes()); + } + match result { + Ok(rename_data_resp) => match encode_rename_data_response_payloads(&rename_data_resp, request_decoded_from_msgpack) { + Ok((rename_data_resp, rename_data_resp_bin)) => Ok(Response::new(RenameDataResponse { + success: true, + rename_data_resp, + rename_data_resp_bin: rename_data_resp_bin.into(), + error: None, + })), Err(err) => Ok(Response::new(RenameDataResponse { success: false, rename_data_resp: String::new(), rename_data_resp_bin: Vec::new().into(), error: Some(err.into()), })), - } - } else { - Ok(Response::new(RenameDataResponse { + }, + Err(err) => Ok(Response::new(RenameDataResponse { success: false, rename_data_resp: String::new(), rename_data_resp_bin: Vec::new().into(), - error: Some(DiskError::other("cannot find disk".to_string()).into()), - })) + error: Some(err.into()), + })), } } diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index e35cf812d..39cdd312e 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -377,10 +377,12 @@ pub(crate) mod timeout_wrapper_consumer { } pub(crate) mod tonic_service_consumer { + #[cfg(test)] + pub(crate) use super::super::tonic_service::make_server; #[cfg(test)] pub(crate) use super::super::tonic_service::{heal_topology_fingerprint, make_heal_control_server_for_source}; pub(crate) use super::super::tonic_service::{ - make_heal_control_server_with_cache, make_scanner_control_server, make_server, make_tier_mutation_control_server, + make_heal_control_server_with_cache, make_scanner_control_server, make_server_for_slot, make_tier_mutation_control_server, }; } @@ -600,8 +602,8 @@ pub(crate) mod ecstore_storage { #[cfg(test)] pub(crate) use rustfs_ecstore::api::storage::init_local_disks; pub(crate) use rustfs_ecstore::api::storage::{ - ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk, all_local_disk_path, - find_local_disk_by_ref, init_local_disks_with_instance_ctx, init_lock_clients, + BootstrapLocalTarget, ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk, + all_local_disk_path, find_local_disk_by_ref, init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map_with_instance_ctx, }; } @@ -677,6 +679,9 @@ type EcstoreReplicationStats = ecstore_bucket::replication::ReplicationStats; pub(crate) type DynReplicationPool = StorageReplicationPoolHandle; pub(crate) type DynReader = ecstore_rio::DynReader; pub(crate) type ECStore = ecstore_storage::ECStore; +pub(crate) type BootstrapLocalTarget = ecstore_storage::BootstrapLocalTarget; +#[cfg(all(test, not(windows)))] +pub(crate) use rustfs_ecstore::api::disk::{LocalPublicationPause, LocalPublicationStage}; pub(crate) type Endpoint = ecstore_disk::endpoint::Endpoint; #[cfg(test)] pub(crate) type Endpoints = ecstore_layout::Endpoints; diff --git a/rustfs/src/storage/tonic_service.rs b/rustfs/src/storage/tonic_service.rs index c7d2ee28a..018da7bf2 100644 --- a/rustfs/src/storage/tonic_service.rs +++ b/rustfs/src/storage/tonic_service.rs @@ -13,8 +13,14 @@ // limitations under the License. pub(crate) use crate::storage::rpc::node_service::make_heal_control_server_with_cache; -pub(crate) use crate::storage::rpc::node_service::make_scanner_control_server; #[cfg(test)] pub(crate) use crate::storage::rpc::node_service::{heal::heal_topology_fingerprint, make_heal_control_server_for_source}; +pub(crate) use crate::storage::rpc::node_service::{make_scanner_control_server, make_server_for_slot}; pub use crate::storage::rpc::{make_heal_control_server, make_server, make_tier_mutation_control_server}; pub type NodeService = crate::storage::rpc::NodeService; + +#[cfg(feature = "e2e-test-hooks")] +#[doc(hidden)] +pub use crate::storage::rpc::node_service::rename_target_capture_test_hook::{ + RenameTargetCapturePause, pause_rename_after_target_capture, +}; diff --git a/rustfs/src/storage_api.rs b/rustfs/src/storage_api.rs index 9abdbd5fd..2b1f38d3b 100644 --- a/rustfs/src/storage_api.rs +++ b/rustfs/src/storage_api.rs @@ -171,12 +171,15 @@ pub(crate) mod server { } pub(crate) mod tonic_service { + #[cfg(test)] + pub(crate) use crate::storage::storage_api::tonic_service_consumer::make_server; #[cfg(test)] pub(crate) use crate::storage::storage_api::tonic_service_consumer::{ heal_topology_fingerprint, make_heal_control_server_for_source, }; pub(crate) use crate::storage::storage_api::tonic_service_consumer::{ - make_heal_control_server_with_cache, make_scanner_control_server, make_server, make_tier_mutation_control_server, + make_heal_control_server_with_cache, make_scanner_control_server, make_server_for_slot, + make_tier_mutation_control_server, }; } } diff --git a/rustfs/tests/embedded_multi_instance_test.rs b/rustfs/tests/embedded_multi_instance_test.rs index 1bf4f3971..844f37df0 100644 --- a/rustfs/tests/embedded_multi_instance_test.rs +++ b/rustfs/tests/embedded_multi_instance_test.rs @@ -540,3 +540,530 @@ async fn second_embedded_server_fails_closed_until_its_context_slot_is_installed server_b.shutdown().await; server_a.shutdown().await; } + +#[cfg(feature = "e2e-test-hooks")] +mod signed_target_rpc { + use super::{common, find_available_port, pause_embedded_startup_after_http_bind, sha256_hex}; + use bytes::Bytes; + use futures::FutureExt; + use hyper_util::rt::TokioIo; + use rustfs::app::context::resolve_object_store_handle; + use rustfs::embedded::RustFSServerBuilder; + use rustfs_ecstore::api::disk::{DiskAPI, DiskError, DiskOption, DiskStore, Endpoint, ReadOptions, new_disk}; + use rustfs_ecstore::api::rpc::{gen_tonic_signature_headers, normalize_tonic_rpc_audience}; + use rustfs_filemeta::{FileInfo, ObjectPartInfo}; + use rustfs_protos::proto_gen::node_service::{RenameDataRequest, RenameDataResponse, node_service_client::NodeServiceClient}; + use std::net::SocketAddr; + use std::path::Path; + use std::sync::{ + Arc, Mutex, + atomic::{AtomicUsize, Ordering}, + }; + use std::time::Duration; + use time::OffsetDateTime; + use tokio::net::TcpStream; + use tokio::time::timeout; + use tonic::transport::Channel; + use uuid::Uuid; + + const WAIT: Duration = Duration::from_secs(30); + const INTERNAL_VOLUME: &str = ".rustfs.sys/tmp"; + const USER_VOLUME: &str = "target-transport"; + + struct SingleConnection { + client: NodeServiceClient, + local: SocketAddr, + peer: SocketAddr, + attempts: Arc, + } + + impl SingleConnection { + async fn connect(address: SocketAddr) -> Self { + let socket = timeout(WAIT, TcpStream::connect(address)) + .await + .expect("bounded real TCP connection") + .expect("connect to the production listener"); + let local = socket.local_addr().expect("client socket identity"); + let peer = socket.peer_addr().expect("listener socket identity"); + let socket = Arc::new(Mutex::new(Some(socket))); + let attempts = Arc::new(AtomicUsize::new(0)); + let connector_attempts = attempts.clone(); + let channel = timeout( + WAIT, + tonic::transport::Endpoint::from_shared(format!("http://{address}")) + .expect("local endpoint") + .timeout(WAIT) + .connect_with_connector(tower::service_fn(move |_: http::Uri| { + connector_attempts.fetch_add(1, Ordering::SeqCst); + // A channel may reconnect implicitly. This fixture has exactly one + // already-connected socket and fails every subsequent dial attempt. + let socket = socket.lock().expect("single socket lock").take(); + async move { + socket.map(TokioIo::new).ok_or_else(|| { + std::io::Error::new(std::io::ErrorKind::ConnectionAborted, "implicit reconnect forbidden") + }) + } + })), + ) + .await + .expect("bounded HTTP/2 handshake") + .expect("HTTP/2 over the original TCP connection"); + Self { + client: NodeServiceClient::new(channel), + local, + peer, + attempts, + } + } + + fn assert_original_connection(&self) { + assert_eq!(self.attempts.load(Ordering::SeqCst), 1, "the channel must not redial"); + } + + async fn rename(&mut self, request: tonic::Request) -> RenameDataResponse { + let response = timeout(WAIT, self.client.rename_data(request)) + .await + .expect("bounded signed RenameData") + .expect("production authentication and RPC routing") + .into_inner(); + self.assert_original_connection(); + response + } + } + + async fn local_fixture_disk(root: &Path) -> DiskStore { + let mut endpoint = Endpoint::try_from(root.to_str().expect("UTF-8 fixture root")).expect("local disk endpoint"); + endpoint.set_pool_index(0); + endpoint.set_set_index(0); + endpoint.set_disk_index(0); + new_disk(&endpoint, &DiskOption::default()) + .await + .expect("open real fixture disk") + } + + async fn stage(disk: &DiskStore, volume: &str, path: &str, body: &'static [u8]) -> FileInfo { + match disk.make_volume(volume).await { + Ok(()) | Err(DiskError::VolumeExists) => {} + Err(err) => panic!("create fixture volume: {err}"), + } + let mut fi = FileInfo::new(path, 1, 0); + fi.erasure.index = 1; + fi.version_id = Some(Uuid::new_v4()); + fi.mod_time = Some(OffsetDateTime::now_utc()); + fi.size = i64::try_from(body.len()).expect("small fixture"); + fi.parts = vec![ObjectPartInfo { + number: 1, + size: body.len(), + actual_size: fi.size, + ..Default::default() + }]; + fi.data = Some(Bytes::from_static(body)); + fi.set_inline_data(); + disk.write_metadata(volume, volume, path, fi.clone()) + .await + .expect("stage real xl.meta"); + assert_body(disk, volume, path, &fi).await; + fi + } + + async fn assert_body(disk: &DiskStore, volume: &str, path: &str, fi: &FileInfo) { + let read = disk + .read_version( + volume, + volume, + path, + &fi.version_id.expect("version").to_string(), + &ReadOptions { + read_data: true, + ..Default::default() + }, + ) + .await + .expect("decode actual inline object bytes"); + assert_eq!(read.data, fi.data); + } + + fn signed_rename( + disk: &DiskStore, + volume: &str, + source: &str, + destination: &str, + fi: &FileInfo, + ) -> tonic::Request { + let payload = RenameDataRequest { + disk: disk.endpoint().to_string(), + src_volume: volume.to_owned(), + src_path: source.to_owned(), + dst_volume: volume.to_owned(), + dst_path: destination.to_owned(), + file_info: serde_json::to_string(fi).expect("real FileInfo JSON"), + ..Default::default() + }; + let canonical = rustfs_protos::canonical_rename_data_request_body(&payload).expect("canonical mutation body"); + // The current production interceptor uses the process RPC identity. Keep + // that authentication contract while testing listener-local disk routing. + let identity = rustfs_common::try_get_global_local_node_name().expect("startup published the RPC identity"); + let audience = normalize_tonic_rpc_audience(&identity).expect("RPC audience"); + let headers = + gen_tonic_signature_headers(&audience, "node_service.NodeService", "RenameData", Some(&sha256_hex(&canonical))) + .expect("production v2 signing with the configured shared secret"); + assert_eq!(headers.get("x-rustfs-rpc-auth-version").expect("v2 metadata"), "2"); + let mut request = tonic::Request::new(payload); + *request.metadata_mut() = tonic::metadata::MetadataMap::from_headers(headers); + request + } + + #[test] + fn signed_target_rpc_uses_listener_instance_across_install_and_reconnect() { + common::run_embedded_test(|| async { + timeout(WAIT * 6, signed_target_rpc_body()) + .await + .expect("bounded listener/startup/transport fixture"); + }); + } + + async fn signed_target_rpc_body() { + // B installs the process default first; A must remain a different target + // both before and after its own application context is installed. + let root_b = tempfile::tempdir().expect("B root"); + let server_b = timeout( + WAIT, + RustFSServerBuilder::new() + .address(format!("127.0.0.1:{}", find_available_port().expect("B port"))) + .volume(root_b.path().to_str().expect("B path")) + .access_key("target-transport-access") + .secret_key("target-transport-secret") + .build(), + ) + .await + .expect("bounded B startup") + .expect("start global B"); + let global_b = resolve_object_store_handle().expect("B installed process AppContext"); + let disk_b = local_fixture_disk(root_b.path()).await; + let global_endpoints = global_b.instance_endpoints().expect("B instance topology"); + let global_paths: Vec<_> = global_endpoints + .0 + .iter() + .flat_map(|pool| pool.endpoints.as_ref().iter()) + .map(ToString::to_string) + .collect(); + assert_eq!( + global_paths, + vec![disk_b.endpoint().to_string()], + "the ambient store must really own B's disk" + ); + let sentinel = stage(&disk_b, USER_VOLUME, "sentinel", b"global-B-must-survive").await; + let sentinel_path = root_b.path().join(USER_VOLUME).join("sentinel/xl.meta"); + let sentinel_bytes = tokio::fs::read(&sentinel_path).await.expect("B's committed bytes"); + + let root_a = tempfile::tempdir().expect("A root"); + let port_a = find_available_port().expect("A port"); + let address_a: SocketAddr = format!("127.0.0.1:{port_a}").parse().expect("A address"); + let mut barrier = pause_embedded_startup_after_http_bind(port_a); + let startup_a = RustFSServerBuilder::new() + .address(address_a.to_string()) + .volume(root_a.path().to_str().expect("A path")) + .access_key("target-transport-access") + .secret_key("target-transport-secret") + .build(); + tokio::pin!(startup_a); + timeout(WAIT, async { + tokio::select! { + () = barrier.wait_until_http_bound() => {} + result = startup_a.as_mut() => { + let _unexpected_server = result.expect("A startup before barrier"); + panic!("A must pause after bind and before ECStore/AppContext"); + } + } + }) + .await + .expect("bounded A HTTP-bind barrier"); + + // Catch assertion failures only to release the real startup barrier and + // obtain a shutdown-capable server handle before resuming the failure. + let pre_ready = std::panic::AssertUnwindSafe(async { + assert!(Arc::ptr_eq(&global_b, &resolve_object_store_handle().expect("global B remains live"))); + let disk_a = local_fixture_disk(root_a.path()).await; + let internal = stage(&disk_a, INTERNAL_VOLUME, "transport-staged", b"pre-ready-internal-body").await; + let user = stage(&disk_a, USER_VOLUME, "staged", b"listener-A-user-body").await; + let user_before = tokio::fs::read(root_a.path().join(USER_VOLUME).join("staged/xl.meta")) + .await + .expect("A staged user bytes"); + let mut connection = SingleConnection::connect(address_a).await; + let mut invalid_signature = signed_rename(&disk_a, INTERNAL_VOLUME, "transport-staged", "bad-signature", &internal); + invalid_signature + .metadata_mut() + .insert("x-rustfs-rpc-signature-v2", "00".parse().expect("invalid MAC header")); + let status = timeout(WAIT, connection.client.rename_data(invalid_signature)) + .await + .expect("bounded invalid-signature response") + .expect_err("production interceptor must reject a bad signature"); + assert_eq!(status.code(), tonic::Code::Unauthenticated); + assert!(!root_a.path().join(INTERNAL_VOLUME).join("bad-signature/xl.meta").exists()); + assert_body(&disk_a, INTERNAL_VOLUME, "transport-staged", &internal).await; + + let committed = connection + .rename(signed_rename( + &disk_a, + INTERNAL_VOLUME, + "transport-staged", + "transport-published", + &internal, + )) + .await; + assert!( + committed.success, + "Bootstrap must commit internal metadata through the bound A registry: {:?}", + committed.error + ); + assert_body(&disk_a, INTERNAL_VOLUME, "transport-published", &internal).await; + + let denied = connection + .rename(signed_rename(&disk_a, USER_VOLUME, "staged", "destination", &user)) + .await; + assert!(!denied.success, "Bootstrap must reject a real user mutation"); + let error: DiskError = denied.error.expect("typed bootstrap rejection").into(); + assert_eq!(error, DiskError::FileAccessDenied); + assert_eq!( + tokio::fs::read(root_a.path().join(USER_VOLUME).join("staged/xl.meta")) + .await + .expect("unchanged A source"), + user_before + ); + assert!(!root_a.path().join(USER_VOLUME).join("destination/xl.meta").exists()); + + assert_eq!(tokio::fs::read(&sentinel_path).await.expect("unchanged B bytes"), sentinel_bytes); + assert_body(&disk_b, USER_VOLUME, "sentinel", &sentinel).await; + connection.assert_original_connection(); + (connection, disk_a, user) + }) + .catch_unwind() + .await; + + barrier.release(); + let server_a = timeout(WAIT, startup_a.as_mut()) + .await + .expect("bounded A context installation") + .expect("A startup after real internal metadata commit"); + let (mut connection, disk_a, user) = match pre_ready { + Ok(fixture) => fixture, + Err(panic) => { + timeout(WAIT, server_a.shutdown()).await.expect("bounded A failure cleanup"); + timeout(WAIT, server_b.shutdown()).await.expect("bounded B failure cleanup"); + std::panic::resume_unwind(panic); + } + }; + assert!(Arc::ptr_eq( + &global_b, + &resolve_object_store_handle().expect("A install preserves global B") + )); + assert_eq!(connection.peer, server_a.address()); + assert_body(&disk_a, USER_VOLUME, "staged", &user).await; + let committed = connection + .rename(signed_rename(&disk_a, USER_VOLUME, "staged", "destination", &user)) + .await; + assert!( + committed.success, + "the same accepted connection must observe Ready for its next request: {:?}", + committed.error + ); + assert_body(&disk_a, USER_VOLUME, "destination", &user).await; + + // Keep the first connection open so the OS cannot recycle its 4-tuple. + let mut reconnected = SingleConnection::connect(address_a).await; + assert_ne!(reconnected.local, connection.local); + assert_eq!(reconnected.peer, connection.peer); + let committed = reconnected + .rename(signed_rename(&disk_a, USER_VOLUME, "destination", "reconnected", &user)) + .await; + assert!(committed.success, "new connections must retain listener A: {:?}", committed.error); + assert_body(&disk_a, USER_VOLUME, "reconnected", &user).await; + assert_eq!(tokio::fs::read(&sentinel_path).await.expect("B remains unchanged"), sentinel_bytes); + assert_body(&disk_b, USER_VOLUME, "sentinel", &sentinel).await; + assert!(!root_b.path().join(USER_VOLUME).join("reconnected/xl.meta").exists()); + connection.assert_original_connection(); + reconnected.assert_original_connection(); + drop(reconnected); + drop(connection); + drop(disk_a); + drop(disk_b); + timeout(WAIT, server_a.shutdown()).await.expect("bounded A shutdown"); + timeout(WAIT, server_b.shutdown()).await.expect("bounded B shutdown"); + } + + #[test] + fn signed_bootstrap_request_does_not_upgrade_after_context_installation() { + common::run_embedded_test(|| async { + timeout(WAIT * 6, signed_delayed_bootstrap_body()) + .await + .expect("bounded delayed Bootstrap fixture"); + }); + } + + async fn signed_delayed_bootstrap_body() { + use rustfs::storage::tonic_service::pause_rename_after_target_capture; + + let root_b = tempfile::tempdir().expect("B root"); + let server_b = timeout( + WAIT, + RustFSServerBuilder::new() + .address(format!("127.0.0.1:{}", find_available_port().expect("B port"))) + .volume(root_b.path().to_str().expect("B path")) + .access_key("delayed-bootstrap-access") + .secret_key("delayed-bootstrap-secret") + .build(), + ) + .await + .expect("bounded B startup") + .expect("start global B"); + let global_b = resolve_object_store_handle().expect("B's published context"); + let disk_b = local_fixture_disk(root_b.path()).await; + let endpoints = global_b.instance_endpoints().expect("B instance topology"); + let paths: Vec<_> = endpoints + .0 + .iter() + .flat_map(|pool| pool.endpoints.as_ref().iter()) + .map(ToString::to_string) + .collect(); + assert_eq!(paths, [disk_b.endpoint().to_string()], "the ambient store owns B"); + stage(&disk_b, USER_VOLUME, "delayed-sentinel", b"B-is-not-the-listener-target").await; + let sentinel_path = root_b.path().join(USER_VOLUME).join("delayed-sentinel/xl.meta"); + let sentinel_before = tokio::fs::read(&sentinel_path).await.expect("B sentinel bytes"); + + let root_a = tempfile::tempdir().expect("A root"); + let port_a = find_available_port().expect("A port"); + let address_a = format!("127.0.0.1:{port_a}").parse().expect("A address"); + let mut startup_barrier = Some(pause_embedded_startup_after_http_bind(port_a)); + let startup_a = RustFSServerBuilder::new() + .address(format!("127.0.0.1:{port_a}")) + .volume(root_a.path().to_str().expect("A path")) + .access_key("delayed-bootstrap-access") + .secret_key("delayed-bootstrap-secret") + .build(); + tokio::pin!(startup_a); + timeout(WAIT, async { + tokio::select! { + () = startup_barrier.as_mut().expect("startup barrier").wait_until_http_bound() => {} + startup = startup_a.as_mut() => { + let _unexpected_server = startup.expect("A initial startup"); + panic!("A must reach its pre-AppContext barrier"); + } + } + }) + .await + .expect("bounded A listener startup"); + + let disk_a = local_fixture_disk(root_a.path()).await; + let delayed_info = stage(&disk_a, USER_VOLUME, "delayed-source", b"captured-Bootstrap-must-not-publish").await; + let control_info = stage(&disk_a, USER_VOLUME, "control-source", b"new-Ready-request-can-publish").await; + let source_path = root_a.path().join(USER_VOLUME).join("delayed-source/xl.meta"); + let destination_path = root_a.path().join(USER_VOLUME).join("delayed-destination/xl.meta"); + let source_before = tokio::fs::read(&source_path).await.expect("delayed source bytes"); + let mut connection = SingleConnection::connect(address_a).await; + let mut server_a = None; + let mut startup_finished = false; + + let (observations, delayed_result, source_after, destination_exists, sentinel_after) = { + let mut capture = + pause_rename_after_target_capture(&disk_a.endpoint().to_string(), USER_VOLUME, "delayed-destination"); + let mut delayed_client = connection.client.clone(); + let delayed = delayed_client.rename_data(signed_rename( + &disk_a, + USER_VOLUME, + "delayed-source", + "delayed-destination", + &delayed_info, + )); + tokio::pin!(delayed); + let mut early_response = None; + + // Bound all work while the request is parked to less than the + // existing channel's 30-second deadline; no timeout is disabled. + let observations = std::panic::AssertUnwindSafe(timeout(Duration::from_secs(20), async { + let was_bootstrap = tokio::select! { + observed = capture.wait_until_captured() => observed, + response = delayed.as_mut() => { + early_response = Some(response); + panic!("signed request finished before the capture pause: {early_response:?}"); + }, + }; + assert!(was_bootstrap, "the actual authenticated handler captured Bootstrap"); + assert!(Arc::ptr_eq(&global_b, &resolve_object_store_handle().expect("global B"))); + assert_eq!(tokio::fs::read(&source_path).await.expect("source before install"), source_before); + assert!(!destination_path.exists()); + + startup_barrier.take().expect("unreleased startup barrier").release(); + let started = startup_a.as_mut().await; + startup_finished = true; + server_a = Some(started.expect("normal A context installation")); + assert!(Arc::ptr_eq(&global_b, &resolve_object_store_handle().expect("global remains B"))); + assert_eq!(connection.peer, server_a.as_ref().expect("A handle").address()); + + // A separate source prevents this control from consuming the + // delayed request's data and masking an erroneous second lookup. + let ready = connection + .rename(signed_rename(&disk_a, USER_VOLUME, "control-source", "ready-control", &control_info)) + .await; + assert!(ready.success, "a fresh signed user request must actually use Ready: {:?}", ready.error); + assert_body(&disk_a, USER_VOLUME, "ready-control", &control_info).await; + assert_body(&disk_a, USER_VOLUME, "delayed-source", &delayed_info).await; + assert!(!destination_path.exists(), "the original request remains parked"); + connection.assert_original_connection(); + })) + .catch_unwind() + .await; + + // Release on every assertion/timeout path, then drain the original + // RPC before shutting down the server and its connection. + drop(capture); + if let Some(barrier) = startup_barrier.take() { + barrier.release(); + } + if !startup_finished { + let started = timeout(WAIT, startup_a.as_mut()).await; + if let Ok(Ok(started)) = started { + server_a = Some(started); + } + } + let delayed_result = match early_response { + Some(response) => Ok(response), + None => timeout(WAIT, delayed.as_mut()).await, + }; + let source_after = tokio::fs::read(&source_path).await; + let destination_exists = tokio::fs::try_exists(&destination_path).await; + let sentinel_after = tokio::fs::read(&sentinel_path).await; + (observations, delayed_result, source_after, destination_exists, sentinel_after) + }; + let connection_attempts = connection.attempts.load(Ordering::SeqCst); + drop(connection); + let shutdown_a = if let Some(server) = server_a { + Some(timeout(WAIT, server.shutdown()).await) + } else { + None + }; + let shutdown_b = timeout(WAIT, server_b.shutdown()).await; + if let Some(result) = shutdown_a { + result.expect("bounded A shutdown"); + } + shutdown_b.expect("bounded B shutdown"); + + assert_eq!(connection_attempts, 1, "the original channel must not redial"); + match observations { + Err(panic) => std::panic::resume_unwind(panic), + Ok(result) => result.expect("complete capture/install/Ready-control within the parked request deadline"), + } + let response = delayed_result + .expect("bounded original request drain") + .expect("the original signed request must return an application result") + .into_inner(); + assert!( + !response.success, + "a captured Bootstrap request must not upgrade to Ready after its await" + ); + let error: DiskError = response.error.expect("typed Bootstrap rejection").into(); + assert_eq!(error, DiskError::FileAccessDenied); + assert_eq!(source_after.expect("original source remains readable"), source_before); + assert!(!destination_exists.expect("read original destination state")); + assert_eq!(sentinel_after.expect("global B sentinel survives"), sentinel_before); + } +}