chore(ecstore): sync heal lock fix with main

This commit is contained in:
overtrue
2026-09-05 17:10:25 +08:00
55 changed files with 7489 additions and 1204 deletions
+1
View File
@@ -40,6 +40,7 @@ script-tests: ## Run shell script tests
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test $(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test $(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test $(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py $(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test $(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
+1
View File
@@ -129,6 +129,7 @@ jobs:
run: | run: |
python3 ./scripts/check_test_wiring.py --self-test python3 ./scripts/check_test_wiring.py --self-test
python3 ./scripts/check_scheduled_validation_freshness.py --self-test python3 ./scripts/check_scheduled_validation_freshness.py --self-test
python3 ./scripts/test_security_workflow.py
python3 ./scripts/check_test_wiring.py python3 ./scripts/check_test_wiring.py
- name: Check no planning docs committed - name: Check no planning docs committed
+1
View File
@@ -167,6 +167,7 @@ jobs:
run: | run: |
python3 ./scripts/check_test_wiring.py --self-test python3 ./scripts/check_test_wiring.py --self-test
python3 ./scripts/check_scheduled_validation_freshness.py --self-test python3 ./scripts/check_scheduled_validation_freshness.py --self-test
python3 ./scripts/test_security_workflow.py
python3 ./scripts/check_test_wiring.py python3 ./scripts/check_test_wiring.py
- name: Check no planning docs committed - name: Check no planning docs committed
+55 -28
View File
@@ -74,10 +74,23 @@ env:
jobs: jobs:
security-test: security-test:
runs-on: smoke-testing runs-on: smoke-testing
continue-on-error: true
timeout-minutes: 360 timeout-minutes: 360
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps: steps:
- name: Checkout repository (for the OIDC live gate script)
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Initialize security evidence
id: evidence
run: |
set -euo pipefail
umask 077
SECURITY_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-security-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
mkdir -- "${SECURITY_ARTIFACTS_DIR}"
printf 'SECURITY_ARTIFACTS_DIR=%s\n' "${SECURITY_ARTIFACTS_DIR}" >> "${GITHUB_ENV}"
# auto-testing is private: clone it with the dedicated PF token (not # auto-testing is private: clone it with the dedicated PF token (not
# GITHUB_TOKEN) and retry transient GitHub/network failures. # GITHUB_TOKEN) and retry transient GitHub/network failures.
- name: Checkout auto-testing scripts (with retry) - name: Checkout auto-testing scripts (with retry)
@@ -98,11 +111,6 @@ jobs:
echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2 echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
exit 1 exit 1
- name: Checkout repository (for the OIDC live gate script)
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Show environment - name: Show environment
run: | run: |
uname -a uname -a
@@ -135,7 +143,8 @@ jobs:
id: test id: test
continue-on-error: true continue-on-error: true
env: env:
REPORT_FILE: /tmp/rustfs-security-report.md REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md
TMPDIR: ${{ env.SECURITY_ARTIFACTS_DIR }}
RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/scripts/test/oidc_keycloak_live.sh RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/scripts/test/oidc_keycloak_live.sh
run: | run: |
set -euo pipefail set -euo pipefail
@@ -159,29 +168,48 @@ jobs:
else else
ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}") ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
fi fi
./auto-testing/rustfs-security-test.sh "${ARGS[@]}" GITHUB_STEP_SUMMARY=/dev/null ./auto-testing/rustfs-security-test.sh "${ARGS[@]}"
- name: Generate report - name: Generate report
if: always() id: report
if: ${{ always() && steps.evidence.outcome == 'success' }}
env:
TEST_OUTCOME: ${{ steps.test.outcome }}
run: | run: |
set -euo pipefail set -euo pipefail
if [ ! -f /tmp/rustfs-security-report.md ]; then RESULT=failure
{ if [ "${TEST_OUTCOME}" = "success" ] && [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then
echo "# RustFS security test report" RESULT=success
echo ""
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
echo "- Trigger: ${{ github.event_name }}"
echo "- Test Step Outcome: failure (suite did not produce a report)"
} > /tmp/rustfs-security-report.md
fi fi
cat /tmp/rustfs-security-report.md >> "${GITHUB_STEP_SUMMARY}" {
echo "# RustFS security test report"
echo ""
echo "- Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
echo "- Workflow Commit: ${GITHUB_SHA}"
echo "- Trigger: ${GITHUB_EVENT_NAME}"
echo "- Test Step Outcome: ${RESULT}"
echo "- Suite Step Outcome: ${TEST_OUTCOME}"
echo ""
# The dashboard prioritizes case rows over the step outcome.
# Keep partial case results in the artifact when the suite fails.
if [ "${RESULT}" = "success" ]; then
cat "${SECURITY_ARTIFACTS_DIR}/suite-report.md"
elif [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then
echo "The suite did not complete successfully. See suite-report.md in this run's artifact for diagnostics."
else
echo "The suite did not produce a non-empty report."
fi
} > "${SECURITY_ARTIFACTS_DIR}/report.md"
cat "${SECURITY_ARTIFACTS_DIR}/report.md" >> "${GITHUB_STEP_SUMMARY}"
[ "${RESULT}" = "success" ]
- name: Upload functional report to dashboard - name: Upload functional report to dashboard
if: always() if: ${{ always() && steps.evidence.outcome == 'success' }}
continue-on-error: true continue-on-error: true
env: env:
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
REPORT_FILE: /tmp/rustfs-security-report.md REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md
SUITE: security SUITE: security
run: | run: |
set -euo pipefail set -euo pipefail
@@ -210,8 +238,9 @@ jobs:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
SUITE: 'security' SUITE: 'security'
SUITE_LABEL: 'Security' SUITE_LABEL: 'Security'
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
REPORT_FILE: '/tmp/rustfs-security-report.md' REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md
LOG_FILE: '' LOG_FILE: ''
run: | run: |
set -euo pipefail set -euo pipefail
@@ -245,7 +274,7 @@ jobs:
echo "" echo ""
echo "## Report (errors and symptoms)" echo "## Report (errors and symptoms)"
echo "" echo ""
if [ -s "${REPORT_FILE}" ]; then if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
redact < "${REPORT_FILE}" redact < "${REPORT_FILE}"
elif [ -s "${LOG_FILE:-}" ]; then elif [ -s "${LOG_FILE:-}" ]; then
echo "(report file missing; log tail below)" echo "(report file missing; log tail below)"
@@ -263,14 +292,12 @@ jobs:
echo "filed backlog issue for suite ${SUITE}" echo "filed backlog issue for suite ${SUITE}"
- name: Upload report and logs - name: Upload report and logs
if: always() if: ${{ always() && steps.evidence.outcome == 'success' }}
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with: with:
name: rustfs-security-test-${{ github.run_id }} name: rustfs-security-test-${{ github.run_id }}-${{ github.run_attempt }}
path: | path: ${{ env.SECURITY_ARTIFACTS_DIR }}/
/tmp/rustfs-security-report.md if-no-files-found: error
/tmp/rustfs-security.*/*
if-no-files-found: ignore
retention-days: 3 retention-days: 3
- name: Cleanup environment (after) - name: Cleanup environment (after)
@@ -42,6 +42,7 @@ jobs:
- name: Check latest scheduled runs - name: Check latest scheduled runs
env: env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
RUSTFS_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
run: | run: |
set +e set +e
python3 scripts/check_scheduled_validation_freshness.py \ python3 scripts/check_scheduled_validation_freshness.py \
@@ -6743,6 +6743,99 @@ async fn test_site_replication_replicates_object_with_bucket_versioning_real_dua
Ok(()) Ok(())
} }
#[tokio::test]
async fn test_site_replication_replays_bucket_created_during_peer_outage_real_dual_node() -> TestResult {
init_logging();
// Keep compilation outside the scenario timeout. Recovery itself waits
// for the production 30-second lightweight retry tick.
let _rustfs_binary = rustfs_binary_path();
match timeout(Duration::from_secs(150), async {
let mut site_env = replication_fast_env();
site_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
let mut site_a_env = RustFSTestEnvironment::new().await?;
site_a_env.start_rustfs_server_with_env(vec![], &site_env).await?;
let mut site_b_env = RustFSTestEnvironment::new().await?;
site_b_env.start_rustfs_server_without_cleanup_with_env(&site_env).await?;
let site_a_client = site_a_env.create_s3_client();
let site_b_client = site_b_env.create_s3_client();
let bucket = "site-repl-peer-outage";
let key = "after-recovery.txt";
let payload = b"site replication recovered the missed bucket".to_vec();
let add_status = site_replication_add(
&site_a_env,
&[
PeerSite {
name: "outage-site-a".to_string(),
endpoint: site_a_env.url.clone(),
access_key: site_a_env.access_key.clone(),
secret_key: site_a_env.secret_key.clone(),
..Default::default()
},
PeerSite {
name: "outage-site-b".to_string(),
endpoint: site_b_env.url.clone(),
access_key: site_b_env.access_key.clone(),
secret_key: site_b_env.secret_key.clone(),
..Default::default()
},
],
)
.await?;
assert!(add_status.success, "unexpected site add result: {add_status:?}");
wait_for_site_replication_enabled(&site_a_env, 2).await?;
wait_for_site_replication_enabled(&site_b_env, 2).await?;
site_b_env.stop_server();
site_a_client.create_bucket().bucket(bucket).send().await?;
site_a_client.head_bucket().bucket(bucket).send().await?;
let queued = site_replication_info(&site_a_env)
.await?
.retry_stats
.ok_or("peer outage did not persist a site replication retry event")?;
assert!(queued.pending + queued.failed > 0, "peer outage retry queue was unexpectedly empty");
site_b_env.restart_server_preserving_data(vec![], &site_env).await?;
let recovery_deadline = tokio::time::Instant::now() + Duration::from_secs(75);
loop {
let bucket_recovered = site_b_client.head_bucket().bucket(bucket).send().await.is_ok();
let queue_empty = site_replication_info(&site_a_env).await?.retry_stats.is_none();
if bucket_recovered && queue_empty {
break;
}
if tokio::time::Instant::now() >= recovery_deadline {
return Err(format!(
"site replication retry did not settle after peer recovery; bucket_recovered={bucket_recovered}, queue_empty={queue_empty}"
)
.into());
}
sleep(Duration::from_millis(250)).await;
}
site_a_client
.put_object()
.bucket(bucket)
.key(key)
.body(ByteStream::from(payload.clone()))
.send()
.await?;
assert_eq!(wait_for_object_on_target(&site_b_client, bucket, key).await?, payload);
Ok(())
})
.await
{
Ok(result) => result,
Err(_) => Err("site replication peer-outage recovery timed out after 150 seconds".into()),
}
}
/// Re-applying a site's own replication config must not disable the peer's reverse direction. /// Re-applying a site's own replication config must not disable the peer's reverse direction.
/// ///
/// `PutBucketReplication` broadcasts the config to every peer — the console's replication /// `PutBucketReplication` broadcasts the config to every peer — the console's replication
+10 -9
View File
@@ -196,15 +196,16 @@ pub mod bucket {
pub use crate::bucket::metadata_sys::ConfigWriteLockProbe; pub use crate::bucket::metadata_sys::ConfigWriteLockProbe;
pub use crate::bucket::metadata_sys::{ pub use crate::bucket::metadata_sys::{
BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock, BucketMetadataMutationGuard, BucketMetadataSys, ObjectLockConfigState, acquire_bucket_metadata_transaction_lock,
acquire_bucket_metadata_transaction_lock_for_incarnation, capture_bucket_metadata_incarnation, delete, acquire_bucket_metadata_transaction_lock_for_incarnation, acquire_scanner_bucket_incarnation_fence,
delete_if_incarnation, delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy, capture_bucket_metadata_incarnation, delete, delete_if_incarnation, delete_under_transaction_lock, get,
get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config, get_accelerate_config, get_bucket_policy, get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk,
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config, get_cors_config, get_durability_config, get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config,
get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config, get_public_access_block_config, get_notification_config, get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config,
get_quota_config, get_replication_config, get_request_payment_config, get_sse_config, get_tagging_config, get_public_access_block_config, get_quota_config, get_replication_config, get_request_payment_config, get_sse_config,
get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, get_tagging_config, get_versioning_config, get_website_config, init_bucket_metadata_sys, list_bucket_targets,
remove_bucket_metadata, set_bucket_metadata, update, update_bucket_targets_under_transaction_lock, reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata, update,
update_config_with, update_if_incarnation, update_quota_if_incarnation, update_under_transaction_lock, update_bucket_targets_under_transaction_lock, update_config_with, update_if_incarnation, update_quota_if_incarnation,
update_under_transaction_lock,
}; };
} }
+114 -7
View File
@@ -655,6 +655,12 @@ pub struct BucketMetadataMutationGuard {
} }
impl BucketMetadataMutationGuard { impl BucketMetadataMutationGuard {
/// Returns the storage-verified identity while both incarnation fences remain valid.
pub fn checked_bucket_incarnation(&self) -> Result<(&str, Uuid)> {
self.ensure_valid(&self.bucket)?;
Ok((&self.bucket, self.incarnation_id))
}
fn ensure_valid(&self, bucket: &str) -> Result<()> { fn ensure_valid(&self, bucket: &str) -> Result<()> {
if self.bucket != bucket { if self.bucket != bucket {
return Err(Error::other("bucket metadata mutation guard does not match bucket")); return Err(Error::other("bucket metadata mutation guard does not match bucket"));
@@ -674,6 +680,29 @@ async fn acquire_config_write_guard_for_incarnation(
sys: Arc<RwLock<BucketMetadataSys>>, sys: Arc<RwLock<BucketMetadataSys>>,
bucket: &str, bucket: &str,
expected_incarnation_id: Option<Uuid>, expected_incarnation_id: Option<Uuid>,
) -> Result<BucketMetadataMutationGuard> {
acquire_config_write_guard_with_migration(sys, bucket, expected_incarnation_id, true).await
}
/// Scanner probes must not create an incarnation to make a capability available.
pub async fn acquire_scanner_bucket_incarnation_fence(
bucket: &str,
expected_incarnation_id: Uuid,
expected_owner_id: Uuid,
) -> Result<BucketMetadataMutationGuard> {
super::utils::check_valid_bucket_name(bucket)?;
let sys = get_bucket_metadata_sys()?;
if expected_owner_id.is_nil() || sys.read().await.api.id != expected_owner_id || expected_incarnation_id.is_nil() {
return Err(Error::other("scanner bucket incarnation owner does not match"));
}
acquire_config_write_guard_with_migration(sys, bucket, Some(expected_incarnation_id), false).await
}
async fn acquire_config_write_guard_with_migration(
sys: Arc<RwLock<BucketMetadataSys>>,
bucket: &str,
expected_incarnation_id: Option<Uuid>,
migrate: bool,
) -> Result<BucketMetadataMutationGuard> { ) -> Result<BucketMetadataMutationGuard> {
let metadata_sys = sys.read().await.clone(); let metadata_sys = sys.read().await.clone();
let lifecycle_guard = metadata_sys.api.acquire_bucket_lifecycle_read_lock(bucket).await?; let lifecycle_guard = metadata_sys.api.acquire_bucket_lifecycle_read_lock(bucket).await?;
@@ -681,13 +710,15 @@ async fn acquire_config_write_guard_for_incarnation(
// Legacy buckets are migrated while the lifecycle fence prevents a // Legacy buckets are migrated while the lifecycle fence prevents a
// same-name replacement. The second read under the write transaction is // same-name replacement. The second read under the write transaction is
// the CAS source of truth for the actual rewrite. // the CAS source of truth for the actual rewrite.
await_bucket_namespace_operation( if migrate {
Some(&lifecycle_guard), await_bucket_namespace_operation(
bucket, Some(&lifecycle_guard),
"bucket config incarnation migration", bucket,
metadata_sys.get_bucket_incarnation_id(bucket), "bucket config incarnation migration",
) metadata_sys.get_bucket_incarnation_id(bucket),
.await?; )
.await?;
}
let transaction_guard = await_bucket_namespace_operation( let transaction_guard = await_bucket_namespace_operation(
Some(&lifecycle_guard), Some(&lifecycle_guard),
bucket, bucket,
@@ -3176,6 +3207,82 @@ mod tests {
); );
} }
#[tokio::test]
async fn scoped_dirty_usage_incarnation_probe_does_not_migrate_legacy_metadata() {
let (dirs, store) = isolated_store_over_temp_disks().await;
let sys = Arc::new(RwLock::new(BucketMetadataSys::new(store.clone())));
let bucket = "scoped-ack-legacy";
for dir in &dirs {
std::fs::create_dir_all(dir.path().join(bucket)).expect("create legacy bucket");
}
let mut metadata = BucketMetadata::new(bucket);
metadata.bucket_incarnation_id = Uuid::nil();
sys.read()
.await
.persist_and_set(metadata)
.await
.expect("persist legacy metadata");
assert!(
acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(Uuid::new_v4()), false)
.await
.is_err()
);
assert!(load_bucket_incarnation(store, bucket).await.expect("read sidecar").is_none());
assert!(
sys.read()
.await
.get_config_from_disk(bucket)
.await
.expect("read metadata")
.bucket_incarnation_id
.is_nil()
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn scoped_dirty_usage_incarnation_rejects_deleted_and_recreated_bucket() {
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let sys = bucket_metadata_sys_of(&store.ctx).expect("metadata owner");
let bucket = "scoped-ack-recreated";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket");
let old = store.bucket_incarnation_id_from_disk(bucket).await.expect("old incarnation");
let guard = acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(old), false)
.await
.expect("trusted incarnation fence");
assert_eq!(guard.checked_bucket_incarnation().expect("valid fences"), (bucket, old));
drop(guard);
store
.delete_bucket(bucket, &DeleteBucketOptions::default())
.await
.expect("delete bucket");
assert!(
acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(old), false)
.await
.is_err()
);
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("recreate bucket");
let new = store.bucket_incarnation_id_from_disk(bucket).await.expect("new incarnation");
assert_ne!(old, new);
assert!(
acquire_config_write_guard_with_migration(sys.clone(), bucket, Some(old), false)
.await
.is_err()
);
assert!(
acquire_config_write_guard_with_migration(sys, bucket, Some(new), false)
.await
.is_ok()
);
}
#[tokio::test] #[tokio::test]
async fn old_node_metadata_rewrite_cannot_replace_bucket_incarnation_sidecar() { async fn old_node_metadata_rewrite_cannot_replace_bucket_incarnation_sidecar() {
let (dirs, ecstore) = isolated_store_over_temp_disks().await; let (dirs, ecstore) = isolated_store_over_temp_disks().await;
+19
View File
@@ -30,6 +30,7 @@ use rustfs_protos::{
ChannelClass, create_new_channel, get_channel_for_class, ChannelClass, create_new_channel, get_channel_for_class,
proto_gen::node_service::{ proto_gen::node_service::{
heal_control_service_client::HealControlServiceClient, node_service_client::NodeServiceClient, heal_control_service_client::HealControlServiceClient, node_service_client::NodeServiceClient,
scanner_control_service_client::ScannerControlServiceClient,
tier_mutation_control_service_client::TierMutationControlServiceClient, tier_mutation_control_service_client::TierMutationControlServiceClient,
}, },
}; };
@@ -60,6 +61,24 @@ pub async fn node_service_time_out_client(
node_service_time_out_client_for_class(addr, interceptor, ChannelClass::Control).await node_service_time_out_client_for_class(addr, interceptor, ChannelClass::Control).await
} }
pub(crate) async fn scanner_control_time_out_client(
addr: &str,
interceptor: TonicInterceptor,
) -> crate::error::Result<ScannerControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
let interceptor = interceptor.with_rpc_audience(addr)?;
let channel = match runtime_sources::cached_node_channel(addr).await {
Some(channel) => channel,
None => create_new_channel(addr)
.await
.map_err(|err| crate::error::Error::other(err.to_string()))?,
};
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
let limit = rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize;
Ok(ScannerControlServiceClient::with_interceptor(channel, interceptor)
.max_decoding_message_size(limit)
.max_encoding_message_size(limit))
}
pub async fn heal_control_time_out_client( pub async fn heal_control_time_out_client(
addr: &str, addr: &str,
interceptor: TonicInterceptor, interceptor: TonicInterceptor,
@@ -2050,6 +2050,53 @@ impl PeerRestClient {
.await .await
} }
/// Probe only: scoped ACK production requires a durable per-bucket proof.
pub async fn scanner_scoped_dirty_usage_capability(
&self,
owner_id: String,
instance_id: String,
entries: Vec<rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry>,
) -> Result<bool> {
use rustfs_protos::scoped_dirty_usage::*;
let payload = rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageAckRequest {
challenge: Uuid::new_v4().as_bytes().to_vec().into(),
protocol_version: SCOPED_DIRTY_USAGE_PROTOCOL_VERSION,
owner_id,
instance_id,
scope: SCOPED_DIRTY_USAGE_BUCKET_SCOPE,
probe_only: true,
entries,
};
let canonical = canonical_scoped_dirty_usage_request(&payload).map_err(|err| Error::other(err.to_string()))?;
self.finalize_result(
async {
let mut client = super::client::scanner_control_time_out_client(
&self.grid_host,
TonicInterceptor::Signature(gen_tonic_signature_interceptor()),
)
.await?;
let mut request = Request::new(payload.clone());
set_tonic_canonical_body_digest(&mut request, &canonical)?;
let response = client.scanner_scoped_dirty_usage_ack(request).await?.into_inner();
let body = canonical_scoped_dirty_usage_response(&canonical, &response)
.map_err(|_| Error::other("scoped dirty usage capability response is too large"))?;
verify_tonic_rpc_response_proof(&body, response.response_proof.as_ref())?;
if response.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION
|| response.owner_id != payload.owner_id
|| response.instance_id != payload.instance_id
|| response.max_entries != SCOPED_DIRTY_USAGE_MAX_ENTRIES
|| response.max_request_bytes != SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES
|| response.cleared != 0
{
return Err(Error::other("scoped dirty usage capability response does not match request"));
}
Ok(response.supported)
}
.await,
)
.await
}
pub async fn acknowledge_scanner_dirty_usage(&self, instance_id: String, generation: u64) -> Result<ScannerPeerActivity> { pub async fn acknowledge_scanner_dirty_usage(&self, instance_id: String, generation: u64) -> Result<ScannerPeerActivity> {
let result = self let result = self
.scanner_activity_request_with_protocol(instance_id.clone(), generation, SCANNER_ACTIVITY_PROTOCOL_VERSION) .scanner_activity_request_with_protocol(instance_id.clone(), generation, SCANNER_ACTIVITY_PROTOCOL_VERSION)
File diff suppressed because it is too large Load Diff
+21 -369
View File
@@ -59,27 +59,34 @@ use super::super::{
send_heal_request_with_admission, should_prevent_write, to_object_err, try_read_inline_data_shards_direct, warn, send_heal_request_with_admission, should_prevent_write, to_object_err, try_read_inline_data_shards_direct, warn,
}; };
#[cfg(test)] #[cfg(test)]
pub(in crate::set_disk) use super::metadata_quorum::MetadataEarlyStopDecision;
pub(in crate::set_disk) use super::metadata_quorum::{
MetadataQuorumAccumulator, is_metadata_fanout_ignored_error, metadata_early_stop_candidate_matches,
};
#[cfg(test)]
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE; use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
#[cfg(test)] #[cfg(test)]
use crate::diagnostics::get::GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH; use crate::diagnostics::get::GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_IDENTITY_MISMATCH;
#[cfg(test)] #[cfg(test)]
use crate::diagnostics::get::GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD; use crate::diagnostics::get::GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_PAYLOAD;
use crate::diagnostics::get::{ use crate::diagnostics::get::{
GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_DELETED, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_MISSING_SHARD, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_REMOTE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE,
GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_SIZE, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED, GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_TRANSFORMED, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM,
GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND,
GET_METADATA_RESPONSE_ERROR, GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT,
GET_METADATA_RESPONSE_VALID, GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_DIRECT_MEMORY,
GET_OBJECT_PATH_INTERNAL_META, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_READER_SETUP_DROP_PENDING,
GET_STAGE_READER_SETUP_SCHEDULE, GET_STAGE_READER_SETUP_WAIT_QUORUM, GET_STAGE_READER_TASK_BITROT_READER_INIT,
GET_STAGE_READER_TASK_FILE_OPEN, GET_STAGE_READER_TASK_READER_CONSTRUCTION, get_stage_timer_if_enabled,
record_get_stage_duration_if_enabled,
};
#[cfg(test)]
use crate::diagnostics::get::{
GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, GET_METADATA_EARLY_STOP_REASON_ERROR, GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, GET_METADATA_EARLY_STOP_REASON_ERROR,
GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM,
GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM,
GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND,
GET_METADATA_RESPONSE_CORRUPT, GET_METADATA_RESPONSE_DISK_NOT_FOUND, GET_METADATA_RESPONSE_ERROR,
GET_METADATA_RESPONSE_IGNORED, GET_METADATA_RESPONSE_NOT_FOUND, GET_METADATA_RESPONSE_TIMEOUT, GET_METADATA_RESPONSE_VALID,
GET_METADATA_RESPONSE_VERSION_NOT_FOUND, GET_OBJECT_PATH_DIRECT_MEMORY, GET_OBJECT_PATH_INTERNAL_META,
GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_READER_SETUP_DROP_PENDING, GET_STAGE_READER_SETUP_SCHEDULE,
GET_STAGE_READER_SETUP_WAIT_QUORUM, GET_STAGE_READER_TASK_BITROT_READER_INIT, GET_STAGE_READER_TASK_FILE_OPEN,
GET_STAGE_READER_TASK_READER_CONSTRUCTION, get_stage_timer_if_enabled, record_get_stage_duration_if_enabled,
}; };
#[cfg(test)] #[cfg(test)]
use crate::disk::CHECK_PART_FILE_NOT_FOUND; use crate::disk::CHECK_PART_FILE_NOT_FOUND;
@@ -690,324 +697,6 @@ impl MetadataFanoutDiagnostics {
} }
} }
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(in crate::set_disk) struct MetadataEarlyStopDecision {
pub(in crate::set_disk) reason: &'static str,
}
#[derive(Clone, Debug)]
pub(in crate::set_disk) struct MetadataQuorumAccumulator {
pub(in crate::set_disk) total_disks: usize,
pub(in crate::set_disk) default_parity_count: usize,
pub(in crate::set_disk) allow_early_stop: bool,
pub(in crate::set_disk) valid_responses: usize,
pub(in crate::set_disk) not_found_responses: usize,
pub(in crate::set_disk) version_not_found_responses: usize,
pub(in crate::set_disk) ignored_errors: usize,
pub(in crate::set_disk) hard_errors: usize,
pub(in crate::set_disk) candidate: Option<FileInfo>,
pub(in crate::set_disk) candidate_votes: usize,
// Bitset of shard indexes whose metadata matches the candidate. Erasure
// layouts are capped at 16 shards, so this stays allocation-free on the
// GET metadata hot path.
candidate_shard_mask: u16,
pub(in crate::set_disk) conflicting_metadata: bool,
pub(in crate::set_disk) delete_marker_seen: bool,
pub(in crate::set_disk) delete_marker_candidates: Vec<(FileInfo, usize)>,
pub(in crate::set_disk) delete_marker_votes: usize,
pub(in crate::set_disk) requested_version_id: String,
pub(in crate::set_disk) matching_version_votes: usize,
}
impl MetadataQuorumAccumulator {
pub(in crate::set_disk) fn new(total_disks: usize, default_parity_count: usize, allow_early_stop: bool) -> Self {
Self {
total_disks,
default_parity_count,
allow_early_stop,
valid_responses: 0,
not_found_responses: 0,
version_not_found_responses: 0,
ignored_errors: 0,
hard_errors: 0,
candidate: None,
candidate_votes: 0,
candidate_shard_mask: 0,
conflicting_metadata: false,
delete_marker_seen: false,
delete_marker_candidates: Vec::new(),
delete_marker_votes: 0,
requested_version_id: String::new(),
matching_version_votes: 0,
}
}
pub(in crate::set_disk) fn with_requested_version_id(mut self, version_id: &str) -> Self {
self.requested_version_id = version_id.to_string();
self
}
pub(in crate::set_disk) fn observe_file_info(&mut self, file_info: &FileInfo) {
self.observe_file_info_with_index(None, file_info);
}
pub(in crate::set_disk) fn observe_file_info_at(&mut self, disk_index: usize, file_info: &FileInfo) {
self.observe_file_info_with_index(Some(disk_index), file_info);
}
fn observe_file_info_with_index(&mut self, disk_index: Option<usize>, file_info: &FileInfo) {
if !file_info_is_valid_for_metadata(file_info) {
self.hard_errors = self.hard_errors.saturating_add(1);
return;
}
self.valid_responses = self.valid_responses.saturating_add(1);
// Track version match for versioned requests
if !self.requested_version_id.is_empty()
&& let Some(ref vid) = file_info.version_id
&& vid.to_string() == self.requested_version_id
{
self.matching_version_votes = self.matching_version_votes.saturating_add(1);
}
if file_info.is_canonical_delete_marker() {
self.delete_marker_seen = true;
if let Some((_, votes)) = self
.delete_marker_candidates
.iter_mut()
.find(|(candidate, _)| metadata_early_stop_candidate_matches(candidate, file_info))
{
*votes = votes.saturating_add(1);
} else {
self.delete_marker_candidates.push((file_info.clone(), 1));
}
self.delete_marker_votes = self
.delete_marker_candidates
.iter()
.map(|(_, votes)| *votes)
.max()
.unwrap_or_default();
self.conflicting_metadata |= self.delete_marker_candidates.len() > 1;
return;
}
match &self.candidate {
Some(candidate) if metadata_early_stop_candidate_matches(candidate, file_info) => {
self.candidate_votes = self.candidate_votes.saturating_add(1);
if let Some(disk_index) = disk_index
&& let Some(bit) = Self::candidate_shard_bit(candidate, file_info, disk_index)
{
self.candidate_shard_mask |= bit;
}
}
Some(_) => {
self.conflicting_metadata = true;
}
None => {
self.candidate = Some(file_info.clone());
self.candidate_votes = 1;
if let Some(disk_index) = disk_index
&& let Some(bit) = Self::candidate_shard_bit(file_info, file_info, disk_index)
{
self.candidate_shard_mask |= bit;
}
}
}
}
fn candidate_shard_bit(candidate: &FileInfo, file_info: &FileInfo, disk_index: usize) -> Option<u16> {
let &erasure_index = candidate.erasure.distribution.get(disk_index)?;
if erasure_index == 0 || erasure_index > u16::BITS as usize || file_info.erasure.index != erasure_index {
return None;
}
Some(1u16 << (erasure_index - 1))
}
pub(in crate::set_disk) fn candidate_has_read_reserve(&self) -> bool {
self.candidate_read_reserve_target()
.is_some_and(|required| self.candidate_shard_mask.count_ones() as usize >= required)
}
pub(in crate::set_disk) fn candidate_read_reserve_target(&self) -> Option<usize> {
let candidate = self.candidate.as_ref()?;
Some(
candidate
.erasure
.data_blocks
.saturating_add(usize::from(candidate.erasure.parity_blocks > 0)),
)
}
pub(in crate::set_disk) fn observe_error(&mut self, err: &DiskError) {
match err {
DiskError::FileNotFound | DiskError::VolumeNotFound => {
self.not_found_responses = self.not_found_responses.saturating_add(1);
}
DiskError::FileVersionNotFound => {
self.version_not_found_responses = self.version_not_found_responses.saturating_add(1);
}
_ if is_metadata_fanout_ignored_error(err) => {
self.ignored_errors = self.ignored_errors.saturating_add(1);
}
_ => {
self.hard_errors = self.hard_errors.saturating_add(1);
}
}
}
pub(in crate::set_disk) fn early_stop_decision(&self) -> Option<MetadataEarlyStopDecision> {
if !self.allow_early_stop {
return None;
}
if self.delete_marker_votes >= self.default_write_quorum() {
return Some(MetadataEarlyStopDecision {
reason: GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER,
});
}
if self.conflicting_metadata
|| self.delete_marker_seen
|| self.not_found_responses > 0
|| self.version_not_found_responses > 0
|| self.hard_errors > 0
{
return None;
}
if self
.candidate
.as_ref()
.and_then(|candidate| self.candidate_latest_quorum(candidate))
.is_some_and(|latest_quorum| self.candidate_votes >= latest_quorum)
{
return Some(MetadataEarlyStopDecision {
reason: GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM,
});
}
None
}
/// Check if a versioned request can early-stop because the requested
/// version_id has reached quorum across disks.
pub(in crate::set_disk) fn version_early_stop_decision(&self) -> Option<MetadataEarlyStopDecision> {
if !self.allow_early_stop {
return None;
}
if self.requested_version_id.is_empty() {
return None;
}
if self.conflicting_metadata
|| self.delete_marker_seen
|| self.not_found_responses > 0
|| self.version_not_found_responses > 0
|| self.hard_errors > 0
{
return None;
}
if self.matching_version_votes >= self.read_quorum_for_version() {
return Some(MetadataEarlyStopDecision {
reason: GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM,
});
}
None
}
pub(in crate::set_disk) fn can_still_reach_early_stop_with_pending(&self, pending: usize) -> bool {
if !self.allow_early_stop {
return false;
}
if self.delete_marker_votes.saturating_add(pending) >= self.default_write_quorum() {
return true;
}
if self.conflicting_metadata
|| self.delete_marker_seen
|| self.not_found_responses > 0
|| self.version_not_found_responses > 0
|| self.hard_errors > 0
{
return false;
}
if !self.requested_version_id.is_empty()
&& self.matching_version_votes.saturating_add(pending) >= self.read_quorum_for_version()
{
return true;
}
match &self.candidate {
Some(candidate) => self
.candidate_latest_quorum(candidate)
.is_some_and(|latest_quorum| self.candidate_votes.saturating_add(pending) >= latest_quorum),
None => pending >= self.default_write_quorum(),
}
}
/// Compute the read quorum threshold for version-aware early-stop.
/// Uses `total_disks / 2` (like `missing_response_quorum`) when
/// `default_parity_count` is set, otherwise requires all disks.
pub(in crate::set_disk) fn read_quorum_for_version(&self) -> usize {
self.missing_response_quorum()
}
pub(in crate::set_disk) fn final_miss_reason(&self) -> &'static str {
if !self.allow_early_stop {
return GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST;
}
if self.conflicting_metadata {
return GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA;
}
if self.delete_marker_seen {
return GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER;
}
let missing_response_quorum = self.missing_response_quorum();
if self.version_not_found_responses >= missing_response_quorum {
return GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND;
}
if self.not_found_responses >= missing_response_quorum {
return GET_METADATA_EARLY_STOP_REASON_NOT_FOUND;
}
if self.hard_errors > 0 {
return GET_METADATA_EARLY_STOP_REASON_ERROR;
}
if self.ignored_errors > 0 {
return GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM;
}
GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM
}
pub(in crate::set_disk) fn candidate_latest_quorum(&self, candidate: &FileInfo) -> Option<usize> {
if self.default_parity_count == 0 {
return Some(self.total_disks);
}
if candidate.is_canonical_delete_marker() || candidate.size == 0 || candidate.erasure.parity_blocks >= self.total_disks {
return None;
}
let data_blocks = candidate.erasure.data_blocks;
Some(if data_blocks == candidate.erasure.parity_blocks {
data_blocks.saturating_add(1)
} else {
data_blocks
})
}
pub(crate) fn default_write_quorum(&self) -> usize {
if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
return self.total_disks;
}
let data_blocks = self.total_disks.saturating_sub(self.default_parity_count);
if data_blocks == self.default_parity_count {
data_blocks.saturating_add(1)
} else {
data_blocks
}
}
pub(in crate::set_disk) fn missing_response_quorum(&self) -> usize {
if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
self.total_disks
} else {
self.total_disks / 2
}
}
}
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub(in crate::set_disk) enum MetadataCacheLookup { pub(in crate::set_disk) enum MetadataCacheLookup {
Hit(Arc<GetObjectMetadataCacheEntry>), Hit(Arc<GetObjectMetadataCacheEntry>),
@@ -1015,39 +704,6 @@ pub(in crate::set_disk) enum MetadataCacheLookup {
RejectedInsufficientQuorum, RejectedInsufficientQuorum,
} }
pub(in crate::set_disk) fn metadata_early_stop_candidate_matches(left: &FileInfo, right: &FileInfo) -> bool {
left.volume == right.volume
&& left.name == right.name
&& left.version_id == right.version_id
&& left.is_latest == right.is_latest
&& left.deleted == right.deleted
&& left.mark_deleted == right.mark_deleted
&& left.transition_status == right.transition_status
&& left.transitioned_objname == right.transitioned_objname
&& left.transition_tier == right.transition_tier
&& left.transition_version_id == right.transition_version_id
&& left.transition_version == right.transition_version
&& left.transition_version_state == right.transition_version_state
&& left.expire_restored == right.expire_restored
&& left.size == right.size
&& left.mod_time == right.mod_time
&& left.mode == right.mode
&& left.written_by_version == right.written_by_version
&& left.metadata == right.metadata
&& left.replication_state_internal == right.replication_state_internal
&& left.parts == right.parts
&& left.checksum == right.checksum
&& left.versioned == right.versioned
&& left.num_versions == right.num_versions
&& left.successor_mod_time == right.successor_mod_time
&& left.data_dir == right.data_dir
&& left.erasure.algorithm == right.erasure.algorithm
&& left.erasure.data_blocks == right.erasure.data_blocks
&& left.erasure.parity_blocks == right.erasure.parity_blocks
&& left.erasure.block_size == right.erasure.block_size
&& left.erasure.distribution == right.erasure.distribution
}
pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason( pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason(
bucket: &str, bucket: &str,
object: &str, object: &str,
@@ -1247,10 +903,6 @@ pub(in crate::set_disk) fn classify_metadata_response_error(err: &DiskError) ->
} }
} }
pub(in crate::set_disk) fn is_metadata_fanout_ignored_error(err: &DiskError) -> bool {
OBJECT_OP_IGNORED_ERRS.iter().any(|ignored| ignored == err)
}
pub(in crate::set_disk) fn is_confirmed_missing_part_error(err: Option<&str>) -> bool { pub(in crate::set_disk) fn is_confirmed_missing_part_error(err: Option<&str>) -> bool {
let Some(err) = err else { let Some(err) = err else {
return false; return false;
@@ -0,0 +1,385 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Pure metadata quorum and early-stop decisions for `SetDisks` reads.
//!
//! Disk scheduling, coalescing, cancellation, and late shard materialization
//! remain with their existing owners; this module only classifies observations.
use crate::diagnostics::get::{
GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER,
GET_METADATA_EARLY_STOP_REASON_ERROR, GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM,
GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST,
GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM,
GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND,
};
use crate::disk::error::DiskError;
use crate::disk::error_reduce::OBJECT_OP_IGNORED_ERRS;
use crate::set_disk::file_info_is_valid_for_metadata;
use rustfs_filemeta::FileInfo;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(in crate::set_disk) struct MetadataEarlyStopDecision {
pub(in crate::set_disk) reason: &'static str,
}
#[derive(Clone, Debug)]
pub(in crate::set_disk) struct MetadataQuorumAccumulator {
pub(in crate::set_disk) total_disks: usize,
pub(in crate::set_disk) default_parity_count: usize,
pub(in crate::set_disk) allow_early_stop: bool,
pub(in crate::set_disk) valid_responses: usize,
pub(in crate::set_disk) not_found_responses: usize,
pub(in crate::set_disk) version_not_found_responses: usize,
pub(in crate::set_disk) ignored_errors: usize,
pub(in crate::set_disk) hard_errors: usize,
pub(in crate::set_disk) candidate: Option<FileInfo>,
pub(in crate::set_disk) candidate_votes: usize,
// Bitset of shard indexes whose metadata matches the candidate. Erasure
// layouts are capped at 16 shards, so this stays allocation-free on the
// GET metadata hot path.
candidate_shard_mask: u16,
pub(in crate::set_disk) conflicting_metadata: bool,
pub(in crate::set_disk) delete_marker_seen: bool,
pub(in crate::set_disk) delete_marker_candidates: Vec<(FileInfo, usize)>,
pub(in crate::set_disk) delete_marker_votes: usize,
pub(in crate::set_disk) requested_version_id: String,
pub(in crate::set_disk) matching_version_votes: usize,
}
impl MetadataQuorumAccumulator {
pub(in crate::set_disk) fn new(total_disks: usize, default_parity_count: usize, allow_early_stop: bool) -> Self {
Self {
total_disks,
default_parity_count,
allow_early_stop,
valid_responses: 0,
not_found_responses: 0,
version_not_found_responses: 0,
ignored_errors: 0,
hard_errors: 0,
candidate: None,
candidate_votes: 0,
candidate_shard_mask: 0,
conflicting_metadata: false,
delete_marker_seen: false,
delete_marker_candidates: Vec::new(),
delete_marker_votes: 0,
requested_version_id: String::new(),
matching_version_votes: 0,
}
}
pub(in crate::set_disk) fn with_requested_version_id(mut self, version_id: &str) -> Self {
self.requested_version_id = version_id.to_string();
self
}
pub(in crate::set_disk) fn observe_file_info(&mut self, file_info: &FileInfo) {
self.observe_file_info_with_index(None, file_info);
}
pub(in crate::set_disk) fn observe_file_info_at(&mut self, disk_index: usize, file_info: &FileInfo) {
self.observe_file_info_with_index(Some(disk_index), file_info);
}
fn observe_file_info_with_index(&mut self, disk_index: Option<usize>, file_info: &FileInfo) {
if !file_info_is_valid_for_metadata(file_info) {
self.hard_errors = self.hard_errors.saturating_add(1);
return;
}
self.valid_responses = self.valid_responses.saturating_add(1);
// Track version match for versioned requests
if !self.requested_version_id.is_empty()
&& let Some(ref vid) = file_info.version_id
&& vid.to_string() == self.requested_version_id
{
self.matching_version_votes = self.matching_version_votes.saturating_add(1);
}
if file_info.is_canonical_delete_marker() {
self.delete_marker_seen = true;
if let Some((_, votes)) = self
.delete_marker_candidates
.iter_mut()
.find(|(candidate, _)| metadata_early_stop_candidate_matches(candidate, file_info))
{
*votes = votes.saturating_add(1);
} else {
self.delete_marker_candidates.push((file_info.clone(), 1));
}
self.delete_marker_votes = self
.delete_marker_candidates
.iter()
.map(|(_, votes)| *votes)
.max()
.unwrap_or_default();
self.conflicting_metadata |= self.delete_marker_candidates.len() > 1;
return;
}
match &self.candidate {
Some(candidate) if metadata_early_stop_candidate_matches(candidate, file_info) => {
self.candidate_votes = self.candidate_votes.saturating_add(1);
if let Some(disk_index) = disk_index
&& let Some(bit) = Self::candidate_shard_bit(candidate, file_info, disk_index)
{
self.candidate_shard_mask |= bit;
}
}
Some(_) => {
self.conflicting_metadata = true;
}
None => {
self.candidate = Some(file_info.clone());
self.candidate_votes = 1;
if let Some(disk_index) = disk_index
&& let Some(bit) = Self::candidate_shard_bit(file_info, file_info, disk_index)
{
self.candidate_shard_mask |= bit;
}
}
}
}
fn candidate_shard_bit(candidate: &FileInfo, file_info: &FileInfo, disk_index: usize) -> Option<u16> {
let &erasure_index = candidate.erasure.distribution.get(disk_index)?;
if erasure_index == 0 || erasure_index > u16::BITS as usize || file_info.erasure.index != erasure_index {
return None;
}
Some(1u16 << (erasure_index - 1))
}
pub(in crate::set_disk) fn candidate_has_read_reserve(&self) -> bool {
self.candidate_read_reserve_target()
.is_some_and(|required| self.candidate_shard_mask.count_ones() as usize >= required)
}
pub(in crate::set_disk) fn candidate_read_reserve_target(&self) -> Option<usize> {
let candidate = self.candidate.as_ref()?;
Some(
candidate
.erasure
.data_blocks
.saturating_add(usize::from(candidate.erasure.parity_blocks > 0)),
)
}
pub(in crate::set_disk) fn observe_error(&mut self, err: &DiskError) {
match err {
DiskError::FileNotFound | DiskError::VolumeNotFound => {
self.not_found_responses = self.not_found_responses.saturating_add(1);
}
DiskError::FileVersionNotFound => {
self.version_not_found_responses = self.version_not_found_responses.saturating_add(1);
}
_ if is_metadata_fanout_ignored_error(err) => {
self.ignored_errors = self.ignored_errors.saturating_add(1);
}
_ => {
self.hard_errors = self.hard_errors.saturating_add(1);
}
}
}
pub(in crate::set_disk) fn early_stop_decision(&self) -> Option<MetadataEarlyStopDecision> {
if !self.allow_early_stop {
return None;
}
if self.delete_marker_votes >= self.default_write_quorum() {
return Some(MetadataEarlyStopDecision {
reason: GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER,
});
}
if self.conflicting_metadata
|| self.delete_marker_seen
|| self.not_found_responses > 0
|| self.version_not_found_responses > 0
|| self.hard_errors > 0
{
return None;
}
if self
.candidate
.as_ref()
.and_then(|candidate| self.candidate_latest_quorum(candidate))
.is_some_and(|latest_quorum| self.candidate_votes >= latest_quorum)
{
return Some(MetadataEarlyStopDecision {
reason: GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM,
});
}
None
}
/// Check if a versioned request can early-stop because the requested
/// version_id has reached quorum across disks.
pub(in crate::set_disk) fn version_early_stop_decision(&self) -> Option<MetadataEarlyStopDecision> {
if !self.allow_early_stop {
return None;
}
if self.requested_version_id.is_empty() {
return None;
}
if self.conflicting_metadata
|| self.delete_marker_seen
|| self.not_found_responses > 0
|| self.version_not_found_responses > 0
|| self.hard_errors > 0
{
return None;
}
if self.matching_version_votes >= self.read_quorum_for_version() {
return Some(MetadataEarlyStopDecision {
reason: GET_METADATA_EARLY_STOP_REASON_VERSION_MATCH_QUORUM,
});
}
None
}
pub(in crate::set_disk) fn can_still_reach_early_stop_with_pending(&self, pending: usize) -> bool {
if !self.allow_early_stop {
return false;
}
if self.delete_marker_votes.saturating_add(pending) >= self.default_write_quorum() {
return true;
}
if self.conflicting_metadata
|| self.delete_marker_seen
|| self.not_found_responses > 0
|| self.version_not_found_responses > 0
|| self.hard_errors > 0
{
return false;
}
if !self.requested_version_id.is_empty()
&& self.matching_version_votes.saturating_add(pending) >= self.read_quorum_for_version()
{
return true;
}
match &self.candidate {
Some(candidate) => self
.candidate_latest_quorum(candidate)
.is_some_and(|latest_quorum| self.candidate_votes.saturating_add(pending) >= latest_quorum),
None => pending >= self.default_write_quorum(),
}
}
/// Compute the read quorum threshold for version-aware early-stop.
/// Uses `total_disks / 2` (like `missing_response_quorum`) when
/// `default_parity_count` is set, otherwise requires all disks.
pub(in crate::set_disk) fn read_quorum_for_version(&self) -> usize {
self.missing_response_quorum()
}
pub(in crate::set_disk) fn final_miss_reason(&self) -> &'static str {
if !self.allow_early_stop {
return GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST;
}
if self.conflicting_metadata {
return GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA;
}
if self.delete_marker_seen {
return GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER;
}
let missing_response_quorum = self.missing_response_quorum();
if self.version_not_found_responses >= missing_response_quorum {
return GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND;
}
if self.not_found_responses >= missing_response_quorum {
return GET_METADATA_EARLY_STOP_REASON_NOT_FOUND;
}
if self.hard_errors > 0 {
return GET_METADATA_EARLY_STOP_REASON_ERROR;
}
if self.ignored_errors > 0 {
return GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM;
}
GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM
}
pub(in crate::set_disk) fn candidate_latest_quorum(&self, candidate: &FileInfo) -> Option<usize> {
if self.default_parity_count == 0 {
return Some(self.total_disks);
}
if candidate.is_canonical_delete_marker() || candidate.size == 0 || candidate.erasure.parity_blocks >= self.total_disks {
return None;
}
let data_blocks = candidate.erasure.data_blocks;
Some(if data_blocks == candidate.erasure.parity_blocks {
data_blocks.saturating_add(1)
} else {
data_blocks
})
}
pub(crate) fn default_write_quorum(&self) -> usize {
if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
return self.total_disks;
}
let data_blocks = self.total_disks.saturating_sub(self.default_parity_count);
if data_blocks == self.default_parity_count {
data_blocks.saturating_add(1)
} else {
data_blocks
}
}
pub(in crate::set_disk) fn missing_response_quorum(&self) -> usize {
if self.default_parity_count == 0 || self.default_parity_count >= self.total_disks {
self.total_disks
} else {
self.total_disks / 2
}
}
}
pub(in crate::set_disk) fn metadata_early_stop_candidate_matches(left: &FileInfo, right: &FileInfo) -> bool {
left.volume == right.volume
&& left.name == right.name
&& left.version_id == right.version_id
&& left.is_latest == right.is_latest
&& left.deleted == right.deleted
&& left.mark_deleted == right.mark_deleted
&& left.transition_status == right.transition_status
&& left.transitioned_objname == right.transitioned_objname
&& left.transition_tier == right.transition_tier
&& left.transition_version_id == right.transition_version_id
&& left.transition_version == right.transition_version
&& left.transition_version_state == right.transition_version_state
&& left.expire_restored == right.expire_restored
&& left.size == right.size
&& left.mod_time == right.mod_time
&& left.mode == right.mode
&& left.written_by_version == right.written_by_version
&& left.metadata == right.metadata
&& left.replication_state_internal == right.replication_state_internal
&& left.parts == right.parts
&& left.checksum == right.checksum
&& left.versioned == right.versioned
&& left.num_versions == right.num_versions
&& left.successor_mod_time == right.successor_mod_time
&& left.data_dir == right.data_dir
&& left.erasure.algorithm == right.erasure.algorithm
&& left.erasure.data_blocks == right.erasure.data_blocks
&& left.erasure.parity_blocks == right.erasure.parity_blocks
&& left.erasure.block_size == right.erasure.block_size
&& left.erasure.distribution == right.erasure.distribution
}
pub(in crate::set_disk) fn is_metadata_fanout_ignored_error(err: &DiskError) -> bool {
OBJECT_OP_IGNORED_ERRS.iter().any(|ignored| ignored == err)
}
+1
View File
@@ -18,3 +18,4 @@
//! duplicating read/write/erasure logic. //! duplicating read/write/erasure logic.
pub(crate) mod io_primitives; pub(crate) mod io_primitives;
mod metadata_quorum;
+60 -12
View File
@@ -45,6 +45,11 @@ use tracing::{debug, error, info, warn};
use super::{DiskError, Endpoint, HealDiskExt as _, local_disk_map_read}; use super::{DiskError, Endpoint, HealDiskExt as _, local_disk_map_read};
const KEEP_HEAL_TASK_STATUS_DURATION: Duration = Duration::from_secs(10 * 60); const KEEP_HEAL_TASK_STATUS_DURATION: Duration = Duration::from_secs(10 * 60);
// Each cache includes alias tokens in its count and byte budget. Eviction
// removes every token sharing a snapshot; neither cache retains repair state.
const MAX_COMPLETED_HEAL_TOKENS: usize = 1024;
const MAX_COMPLETED_HEAL_BYTES: usize = 64 * 1024 * 1024;
const MAX_COMPLETED_HEAL_RESULT_BYTES: usize = 1024 * 1024;
const DISPLACED_HEAL_REASON: &str = "reason=displaced; retry_hint=submit_again"; const DISPLACED_HEAL_REASON: &str = "reason=displaced; retry_hint=submit_again";
const LOG_COMPONENT_HEAL: &str = "heal"; const LOG_COMPONENT_HEAL: &str = "heal";
const LOG_SUBSYSTEM_DISK_SCANNER: &str = "disk_scanner"; const LOG_SUBSYSTEM_DISK_SCANNER: &str = "disk_scanner";
@@ -180,6 +185,8 @@ fn record_displaced_terminal(
request: &HealRequest, request: &HealRequest,
) -> Arc<CompletedHealStatus> { ) -> Arc<CompletedHealStatus> {
let terminal = Arc::new(CompletedHealStatus { let terminal = Arc::new(CompletedHealStatus {
progress: None,
retained_bytes: std::sync::OnceLock::new(),
heal_type: request.heal_type.clone(), heal_type: request.heal_type.clone(),
status: HealTaskStatus::Failed { status: HealTaskStatus::Failed {
error: format!("heal task displaced by a higher-priority request ({DISPLACED_HEAL_REASON})"), error: format!("heal task displaced by a higher-priority request ({DISPLACED_HEAL_REASON})"),
@@ -193,6 +200,7 @@ fn record_displaced_terminal(
let mut terminals = lock_displaced_terminals(registry); let mut terminals = lock_displaced_terminals(registry);
prune_completed_heal_statuses(&mut terminals); prune_completed_heal_statuses(&mut terminals);
terminals.insert(request.id.clone(), Arc::clone(&terminal)); terminals.insert(request.id.clone(), Arc::clone(&terminal));
prune_completed_heal_statuses(&mut terminals);
terminal terminal
} }
@@ -209,9 +217,15 @@ async fn remove_displaced_task_aliases(
.collect::<Vec<_>>(); .collect::<Vec<_>>();
let mut displaced_terminals = lock_displaced_terminals(terminals); let mut displaced_terminals = lock_displaced_terminals(terminals);
prune_completed_heal_statuses(&mut displaced_terminals); prune_completed_heal_statuses(&mut displaced_terminals);
for alias_id in alias_ids { if displaced_terminals
displaced_terminals.insert(alias_id, Arc::clone(terminal)); .get(task_id)
.is_some_and(|current| Arc::ptr_eq(current, terminal))
{
for alias_id in alias_ids {
displaced_terminals.insert(alias_id, Arc::clone(terminal));
}
} }
prune_completed_heal_statuses(&mut displaced_terminals);
aliases.retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id); aliases.retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id);
} }
@@ -222,6 +236,36 @@ async fn remove_task_aliases_for_task(registry: &Arc<Mutex<HashMap<String, HealT
.retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id); .retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id);
} }
// Callers hold active ownership until publication. Lock order is active ->
// retrying (when needed) -> aliases -> completed; queries release aliases
// before looking up active state. Publishing aliases before removing their
// mapping keeps both an already-resolved token and a new lookup valid.
async fn publish_completed_heal(
completed_heals: &Mutex<HashMap<String, Arc<CompletedHealStatus>>>,
task_aliases: &Mutex<HashMap<String, HealTaskAlias>>,
task_id: &str,
completed: CompletedHealStatus,
terminal: bool,
) {
let completed = Arc::new(completed);
completed.retained_bytes();
let mut aliases = task_aliases.lock().await;
let mut retained = completed_heals.lock().await;
if let Some(previous) = retained.get(task_id).cloned() {
for entry in retained.values_mut().filter(|entry| Arc::ptr_eq(entry, &previous)) {
*entry = Arc::clone(&completed);
}
}
retained.insert(task_id.to_owned(), Arc::clone(&completed));
if terminal {
for (alias_id, _) in aliases.iter().filter(|(_, alias)| alias.task_id == task_id) {
retained.insert(alias_id.clone(), Arc::clone(&completed));
}
aliases.retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id);
}
prune_completed_heal_statuses(&mut retained);
}
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct HealTaskReport { pub struct HealTaskReport {
pub status: HealTaskStatus, pub status: HealTaskStatus,
@@ -268,7 +312,7 @@ fn completed_task_report(completed: &CompletedHealStatus, since: Option<u64>) ->
let result_items = match since { let result_items = match since {
None => completed.seqed_items.iter().map(|(_, item)| item.clone()).collect(), None => completed.seqed_items.iter().map(|(_, item)| item.clone()).collect(),
Some(cursor) => { Some(cursor) => {
if cursor + 1 < completed.min_seq { if cursor.saturating_add(1) < completed.min_seq {
lagged = true; lagged = true;
} }
completed completed
@@ -283,7 +327,7 @@ fn completed_task_report(completed: &CompletedHealStatus, since: Option<u64>) ->
status: completed.status.clone(), status: completed.status.clone(),
result_items, result_items,
result_items_truncated: completed.result_items_truncated || lagged, result_items_truncated: completed.result_items_truncated || lagged,
progress: None, progress: completed.progress.clone(),
next_seq: completed.next_seq, next_seq: completed.next_seq,
min_seq: completed.min_seq, min_seq: completed.min_seq,
} }
@@ -1847,14 +1891,14 @@ impl HealManager {
pub async fn get_task_progress(&self, task_id: &str) -> Result<HealProgress> { pub async fn get_task_progress(&self, task_id: &str) -> Result<HealProgress> {
let canonical_task_id = self.canonical_task_id(task_id).await; let canonical_task_id = self.canonical_task_id(task_id).await;
let active_heals = self.active_heals.lock().await; let progress = match self.lookup_task_state(&canonical_task_id, None).await {
if let Some(task) = active_heals.get(&canonical_task_id) { TaskStateLookup::Active(task) => Some(task.get_progress().await),
Ok(task.get_progress().await) TaskStateLookup::Completed(completed) => completed.progress.clone(),
} else { _ => None,
Err(Error::TaskNotFound { };
task_id: task_id.to_string(), progress.ok_or_else(|| Error::TaskNotFound {
}) task_id: task_id.to_string(),
} })
} }
/// Cancel task /// Cancel task
@@ -1864,6 +1908,8 @@ impl HealManager {
let mut active_heals = self.active_heals.lock().await; let mut active_heals = self.active_heals.lock().await;
if let Some(task) = active_heals.get(&canonical_task_id) { if let Some(task) = active_heals.get(&canonical_task_id) {
task.cancel().await?; task.cancel().await?;
let completed = CompletedHealStatus::snapshot(task, HealTaskStatus::Cancelled).await;
publish_completed_heal(&self.completed_heals, &self.task_aliases, &canonical_task_id, completed, true).await;
active_heals.remove(&canonical_task_id); active_heals.remove(&canonical_task_id);
publish_active_heal_count(&active_heals); publish_active_heal_count(&active_heals);
info!( info!(
@@ -1940,6 +1986,8 @@ impl HealManager {
for task_id in &task_ids { for task_id in &task_ids {
if let Some(task) = active_heals.get(task_id) { if let Some(task) = active_heals.get(task_id) {
task.cancel().await?; task.cancel().await?;
let completed = CompletedHealStatus::snapshot(task, HealTaskStatus::Cancelled).await;
publish_completed_heal(&self.completed_heals, &self.task_aliases, task_id, completed, true).await;
} }
active_heals.remove(task_id); active_heals.remove(task_id);
cancelled += 1; cancelled += 1;
+129
View File
@@ -82,6 +82,8 @@ pub(super) enum QueuePushOutcome {
pub(super) struct CompletedHealStatus { pub(super) struct CompletedHealStatus {
pub(super) heal_type: HealType, pub(super) heal_type: HealType,
pub(super) status: HealTaskStatus, pub(super) status: HealTaskStatus,
pub(super) progress: Option<HealProgress>,
pub(super) retained_bytes: std::sync::OnceLock<usize>,
pub(super) result_items_truncated: bool, pub(super) result_items_truncated: bool,
pub(super) completed_at: SystemTime, pub(super) completed_at: SystemTime,
/// Sequence-stamped retained window, archived with the completion so /// Sequence-stamped retained window, archived with the completion so
@@ -92,6 +94,133 @@ pub(super) struct CompletedHealStatus {
pub(super) min_seq: u64, pub(super) min_seq: u64,
} }
impl CompletedHealStatus {
// Account for owned capacities, including nested drive arrays. Aliases
// conservatively charge the shared allocation again, keeping both token
// count and retained payload bounded without a second ownership index.
pub(super) fn retained_bytes(&self) -> usize {
*self.retained_bytes.get_or_init(|| self.measure_retained_bytes())
}
fn measure_retained_bytes(&self) -> usize {
let mut bytes = size_of::<Self>();
let mut add = |amount: usize| bytes = bytes.saturating_add(amount);
match &self.heal_type {
HealType::Cluster => {}
HealType::Bucket { bucket } => add(bucket.capacity()),
HealType::Object {
bucket,
object,
version_id,
}
| HealType::ECDecode {
bucket,
object,
version_id,
} => {
add(bucket.capacity());
add(object.capacity());
add(version_id.as_ref().map_or(0, String::capacity));
}
HealType::Prefix { bucket, prefix } => {
add(bucket.capacity());
add(prefix.capacity());
}
HealType::Metadata { bucket, object } => {
add(bucket.capacity());
add(object.capacity());
}
HealType::ErasureSet { buckets, set_disk_id } => {
add(buckets.capacity().saturating_mul(size_of::<String>()));
for bucket in buckets {
add(bucket.capacity());
}
add(set_disk_id.capacity());
}
}
if let HealTaskStatus::Failed { error } | HealTaskStatus::Retrying { error, .. } = &self.status {
add(error.capacity());
}
add(self
.progress
.as_ref()
.and_then(|progress| progress.current_object.as_ref())
.map_or(0, String::capacity));
add(self.seqed_items.capacity().saturating_mul(size_of::<(u64, HealResultItem)>()));
for (_, item) in &self.seqed_items {
add(Self::result_item_heap_bytes(item));
}
bytes
}
fn result_item_heap_bytes(item: &HealResultItem) -> usize {
let mut bytes = 0usize;
let mut add = |amount: usize| bytes = bytes.saturating_add(amount);
for value in [
&item.heal_item_type,
&item.bucket,
&item.object,
&item.version_id,
&item.detail,
] {
add(value.capacity());
}
for infos in [&item.before, &item.after] {
add(infos
.drives
.capacity()
.saturating_mul(size_of::<rustfs_madmin::heal_commands::HealDriveInfo>()));
for drive in &infos.drives {
add(drive.uuid.capacity());
add(drive.endpoint.capacity());
add(drive.state.capacity());
}
}
bytes
}
pub(super) fn bound_result_window(&mut self) {
let mut bytes = 0usize;
let retained = self
.seqed_items
.iter()
.rev()
.take_while(|(_, item)| {
bytes = bytes
.saturating_add(size_of::<(u64, HealResultItem)>())
.saturating_add(Self::result_item_heap_bytes(item));
bytes <= MAX_COMPLETED_HEAL_RESULT_BYTES
})
.count();
let truncated = retained < self.seqed_items.len();
if truncated {
self.seqed_items.drain(..self.seqed_items.len() - retained);
self.seqed_items.shrink_to_fit();
self.min_seq = self.seqed_items.first().map_or(self.next_seq, |(seq, _)| *seq);
self.result_items_truncated = true;
self.retained_bytes.take();
}
}
pub(super) async fn snapshot(task: &HealTask, status: HealTaskStatus) -> Self {
let seqed_items = task.get_seqed_result_items().await;
let (next_seq, min_seq) = task.result_seq_cursors();
let mut snapshot = Self {
heal_type: task.heal_type.clone(),
status,
progress: Some(task.get_progress().await),
retained_bytes: std::sync::OnceLock::new(),
result_items_truncated: task.result_items_truncated(),
completed_at: SystemTime::now(),
seqed_items,
next_seq,
min_seq,
};
snapshot.bound_result_window();
snapshot
}
}
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub(super) struct HealTaskAlias { pub(super) struct HealTaskAlias {
pub(super) task_id: String, pub(super) task_id: String,
+75 -39
View File
@@ -264,7 +264,7 @@ impl HealManager {
error: error.clone(), error: error.clone(),
retry_attempt: request.retry_attempts, retry_attempt: request.retry_attempts,
}); });
let retry_request_for_queue = retry_request; let mut retry_request_for_queue = retry_request;
let retry_cancel_token = retry_request_for_queue.as_ref().map(|_| CancellationToken::new()); let retry_cancel_token = retry_request_for_queue.as_ref().map(|_| CancellationToken::new());
if retry_request_for_queue.is_none() { if retry_request_for_queue.is_none() {
replacement_recovery_anchors_clone replacement_recovery_anchors_clone
@@ -272,7 +272,35 @@ impl HealManager {
.unwrap_or_else(|poisoned| poisoned.into_inner()) .unwrap_or_else(|poisoned| poisoned.into_inner())
.remove(&task_id); .remove(&task_id);
} }
let mut completed_status = match retry_request_for_status {
Some(status) => status,
None => task.get_status().await,
};
let mut completed_status_entry = CompletedHealStatus::snapshot(&task, completed_status.clone()).await;
let completed_progress = task.get_progress().await;
#[cfg(test)]
tests::pause_completed_retention_before_publish(&task_id, &completed_status).await;
let mut active_heals_guard = active_heals_clone.lock().await; let mut active_heals_guard = active_heals_clone.lock().await;
let owns_completion = active_heals_guard.contains_key(&task_id);
let cancelled_completion = if owns_completion {
false
} else {
// Cancellation can win while a finished worker waits
// for active ownership. It must not resurrect a retry
// or replace an acknowledged cancellation with success.
retry_request_for_queue = None;
completed_heals_clone
.lock()
.await
.get(&task_id)
.is_some_and(|completed| completed.status == HealTaskStatus::Cancelled)
};
if cancelled_completion {
completed_status = HealTaskStatus::Cancelled;
completed_status_entry.status = HealTaskStatus::Cancelled;
}
let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. });
let successful_completion = matches!(completed_status, HealTaskStatus::Completed);
// Keep retry ownership continuous: status snapshots acquire // Keep retry ownership continuous: status snapshots acquire
// these locks in the same active -> retrying order. // these locks in the same active -> retrying order.
let mut retrying_heals_guard = if let (Some((request, _, error)), Some(cancel_token)) = let mut retrying_heals_guard = if let (Some((request, _, error)), Some(cancel_token)) =
@@ -295,6 +323,16 @@ impl HealManager {
} else { } else {
None None
}; };
if owns_completion || cancelled_completion {
publish_completed_heal(
&completed_heals_clone,
&task_aliases_clone,
&task_id,
completed_status_entry,
terminal_completion,
)
.await;
}
let completed_task = active_heals_guard.remove(&task_id); let completed_task = active_heals_guard.remove(&task_id);
if let Some(completed_task) = completed_task.as_ref() { if let Some(completed_task) = completed_task.as_ref() {
publish_active_heal_count(&active_heals_guard); publish_active_heal_count(&active_heals_guard);
@@ -304,33 +342,10 @@ impl HealManager {
drop(retrying_heals_guard.take()); drop(retrying_heals_guard.take());
drop(active_heals_guard); drop(active_heals_guard);
if let Some(completed_task) = completed_task { #[cfg(test)]
let completed_status = if let Some(status) = retry_request_for_status { tests::pause_completed_retention_handoff(&task_id).await;
status
} else { if completed_task.is_some() {
completed_task.get_status().await
};
let terminal_completion = !matches!(completed_status, HealTaskStatus::Retrying { .. });
let successful_completion = matches!(completed_status, HealTaskStatus::Completed);
let completed_progress = completed_task.get_progress().await;
// Single snapshot of the retained window: the task is
// finished and already off the active map, so there is
// no concurrent writer to race with.
let seqed_items = completed_task.get_seqed_result_items().await;
let (next_seq, min_seq) = completed_task.result_seq_cursors();
let completed_status_entry = CompletedHealStatus {
heal_type: completed_task.heal_type.clone(),
status: completed_status.clone(),
result_items_truncated: completed_task.result_items_truncated(),
completed_at: SystemTime::now(),
seqed_items,
next_seq,
min_seq,
};
let mut completed_heals_guard = completed_heals_clone.lock().await;
prune_completed_heal_statuses(&mut completed_heals_guard);
completed_heals_guard.insert(task_id.clone(), Arc::new(completed_status_entry));
drop(completed_heals_guard);
// update statistics // update statistics
let mut stats = statistics_clone.write().await; let mut stats = statistics_clone.write().await;
match completed_status { match completed_status {
@@ -352,10 +367,6 @@ impl HealManager {
} else { } else {
release_mrf_repair_notice_targets(notice_targets); release_mrf_repair_notice_targets(notice_targets);
} }
task_aliases_clone
.lock()
.await
.retain(|alias_id, alias| alias_id != &task_id && alias.task_id != task_id);
} }
} }
@@ -718,17 +729,42 @@ pub(super) fn heal_request_set_key_for_task(task: &HealTask) -> Option<String> {
} }
pub(super) fn prune_completed_heal_statuses(completed_heals: &mut HashMap<String, Arc<CompletedHealStatus>>) { pub(super) fn prune_completed_heal_statuses(completed_heals: &mut HashMap<String, Arc<CompletedHealStatus>>) {
let Ok(now) = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) else { prune_completed_heal_statuses_at(completed_heals, SystemTime::now());
return; }
};
pub(super) fn prune_completed_heal_statuses_at(completed_heals: &mut HashMap<String, Arc<CompletedHealStatus>>, now: SystemTime) {
completed_heals.retain(|_, completed| { completed_heals.retain(|_, completed| {
completed now.duration_since(completed.completed_at)
.completed_at .map(|age| age <= KEEP_HEAL_TASK_STATUS_DURATION)
.duration_since(SystemTime::UNIX_EPOCH)
.map(|completed_at| now.saturating_sub(completed_at) <= KEEP_HEAL_TASK_STATUS_DURATION)
.unwrap_or(false) .unwrap_or(false)
}); });
let entry_bytes = |key: &String, value: &Arc<CompletedHealStatus>| {
key.capacity()
.saturating_add(size_of::<(String, Arc<CompletedHealStatus>)>())
.saturating_add(value.retained_bytes())
};
let mut bytes = completed_heals
.iter()
.fold(0usize, |total, (key, value)| total.saturating_add(entry_bytes(key, value)));
while completed_heals.len() > MAX_COMPLETED_HEAL_TOKENS || bytes > MAX_COMPLETED_HEAL_BYTES {
let Some(oldest) = completed_heals
.iter()
.min_by(|(left_id, left), (right_id, right)| {
left.completed_at.cmp(&right.completed_at).then_with(|| left_id.cmp(right_id))
})
.map(|(_, value)| Arc::clone(value))
else {
break;
};
completed_heals.retain(|key, value| {
if Arc::ptr_eq(value, &oldest) {
bytes = bytes.saturating_sub(entry_bytes(key, value));
false
} else {
true
}
});
}
} }
pub(super) fn can_schedule_request( pub(super) fn can_schedule_request(
+348 -3
View File
@@ -101,6 +101,326 @@ async fn process_manager_queue_once(manager: &HealManager) {
struct MockStorage; struct MockStorage;
fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus {
CompletedHealStatus {
heal_type: HealType::Cluster,
status: HealTaskStatus::Completed,
progress: Some(HealProgress {
objects_scanned: 9,
objects_healed: 8,
objects_failed: 1,
..Default::default()
}),
retained_bytes: std::sync::OnceLock::new(),
result_items_truncated: false,
completed_at,
seqed_items: vec![(3, HealResultItem::default()), (4, HealResultItem::default())],
next_seq: 5,
min_seq: 3,
}
}
#[test]
fn completed_retention_cursor_boundaries_preserve_progress() {
let completed = completed_retention_fixture(SystemTime::now());
for (cursor, count, lagged) in [
(0, 2, true),
(1, 2, true),
(2, 2, false),
(3, 1, false),
(4, 0, false),
(5, 0, false),
(u64::MAX, 0, false),
] {
let report = completed_task_report(&completed, Some(cursor));
assert_eq!(report.result_items.len(), count, "cursor={cursor}");
assert_eq!(report.result_items_truncated, lagged, "cursor={cursor}");
assert_eq!(report.progress, completed.progress);
assert_eq!((report.next_seq, report.min_seq), (5, 3));
}
assert_eq!(completed_task_report(&completed, None).result_items.len(), 2);
}
#[tokio::test]
async fn completed_retention_displaced_alias_does_not_resurrect_evicted_snapshot() {
let manager = HealManager::new(Arc::new(MockStorage), None);
let request = HealRequest::bucket("bucket".to_string());
manager.insert_task_alias("alias", &request.id).await;
let terminal = record_displaced_terminal(&manager.displaced_terminals, &request);
lock_displaced_terminals(&manager.displaced_terminals).remove(&request.id);
remove_displaced_task_aliases(&manager.task_aliases, &manager.displaced_terminals, &request.id, &terminal).await;
for token in [&request.id, &"alias".to_string()] {
assert!(matches!(manager.get_task_report(token).await, Err(Error::TaskNotFound { .. })));
}
assert!(manager.task_aliases.lock().await.is_empty());
assert!(lock_displaced_terminals(&manager.displaced_terminals).is_empty());
}
#[test]
fn completed_retention_count_ttl_and_alias_eviction_are_bounded() {
let now = SystemTime::now();
let mut entries = HashMap::new();
let oldest = Arc::new(completed_retention_fixture(now - KEEP_HEAL_TASK_STATUS_DURATION));
entries.insert("oldest".to_string(), Arc::clone(&oldest));
entries.insert("oldest-alias".to_string(), Arc::clone(&oldest));
for index in 2..MAX_COMPLETED_HEAL_TOKENS {
entries.insert(format!("task-{index}"), Arc::new(completed_retention_fixture(now)));
}
prune_completed_heal_statuses_at(&mut entries, now);
assert_eq!(entries.len(), MAX_COMPLETED_HEAL_TOKENS);
entries.insert("cap-plus-one".to_string(), Arc::new(completed_retention_fixture(now)));
prune_completed_heal_statuses_at(&mut entries, now);
assert_eq!(entries.len(), MAX_COMPLETED_HEAL_TOKENS - 1);
assert!(!entries.contains_key("oldest"));
assert!(!entries.contains_key("oldest-alias"));
entries.clear();
entries.insert("ttl-boundary".to_string(), oldest);
entries.insert(
"expired".to_string(),
Arc::new(completed_retention_fixture(
now - KEEP_HEAL_TASK_STATUS_DURATION - Duration::from_nanos(1),
)),
);
entries.insert("future".to_string(), Arc::new(completed_retention_fixture(now + Duration::from_nanos(1))));
prune_completed_heal_statuses_at(&mut entries, now);
assert_eq!(entries.len(), 1);
assert!(entries.contains_key("ttl-boundary"));
prune_completed_heal_statuses_at(&mut entries, now + Duration::from_nanos(1));
assert!(entries.is_empty());
}
#[test]
fn completed_retention_total_byte_cap_and_cap_plus_one() {
let now = SystemTime::now();
let key = "large".to_string();
let mut entry = completed_retention_fixture(now);
let base_bytes = entry.retained_bytes() + key.capacity() + size_of::<(String, Arc<CompletedHealStatus>)>();
entry.retained_bytes.take();
entry.status = HealTaskStatus::Failed {
error: "x".repeat(MAX_COMPLETED_HEAL_BYTES - base_bytes),
};
assert_eq!(
entry.retained_bytes() + key.capacity() + size_of::<(String, Arc<CompletedHealStatus>)>(),
MAX_COMPLETED_HEAL_BYTES
);
let mut entries = HashMap::from([(key, Arc::new(entry))]);
prune_completed_heal_statuses_at(&mut entries, now);
assert_eq!(entries.len(), 1, "exact byte cap remains retained");
let mut over = Arc::try_unwrap(entries.remove("large").expect("entry retained")).expect("entry not shared");
over.retained_bytes.take();
if let HealTaskStatus::Failed { error } = &mut over.status {
*error = "x".repeat(error.len() + 1);
}
entries.insert("large".to_string(), Arc::new(over));
prune_completed_heal_statuses_at(&mut entries, now);
assert!(entries.is_empty(), "oversized metadata cannot escape total byte bound");
}
#[tokio::test]
async fn completed_retention_large_window_keeps_cursors_and_progress() {
let task = HealTask::from_request(HealRequest::bucket("bucket".to_string()), Arc::new(MockStorage));
let mut snapshot = completed_retention_fixture(SystemTime::now());
snapshot.seqed_items[0].1.detail = "x".repeat(MAX_COMPLETED_HEAL_RESULT_BYTES);
snapshot.bound_result_window();
assert_eq!(snapshot.seqed_items.len(), 1);
assert_eq!((snapshot.min_seq, snapshot.next_seq), (4, 5));
assert!(snapshot.result_items_truncated);
assert!(snapshot.retained_bytes() < MAX_COMPLETED_HEAL_RESULT_BYTES);
let report = completed_task_report(&snapshot, Some(0));
assert_eq!(report.progress.expect("progress retained").objects_scanned, 9);
assert!(report.result_items_truncated);
let active_max = task.get_result_items_since(Some(u64::MAX)).await;
assert!(active_max.items.is_empty());
assert!(!active_max.lagged);
}
#[test]
fn completed_retention_result_byte_cap_and_cap_plus_one() {
for extra in [0, 1] {
let mut snapshot = completed_retention_fixture(SystemTime::now());
snapshot.seqed_items = vec![(
4,
HealResultItem {
detail: "x".repeat(MAX_COMPLETED_HEAL_RESULT_BYTES - size_of::<(u64, HealResultItem)>() + extra),
..Default::default()
},
)];
snapshot.min_seq = 4;
snapshot.bound_result_window();
assert_eq!(snapshot.seqed_items.len(), 1 - extra);
assert_eq!(snapshot.result_items_truncated, extra == 1);
assert_eq!(snapshot.min_seq, if extra == 0 { 4 } else { 5 });
assert_eq!(snapshot.next_seq, 5);
assert_eq!(snapshot.progress.as_ref().expect("progress retained").objects_scanned, 9);
}
}
#[derive(Default)]
struct CompletedRetentionHook {
started: Notify,
execute: Notify,
handoff: Notify,
finish: Notify,
pause_before_publish: bool,
before_publish: Notify,
publish: Notify,
prepared_status: Mutex<Option<HealTaskStatus>>,
}
static COMPLETED_RETENTION_HOOKS: LazyLock<Mutex<HashMap<String, Arc<CompletedRetentionHook>>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
pub(super) async fn pause_completed_retention_handoff(task_id: &str) {
let hook = COMPLETED_RETENTION_HOOKS.lock().await.get(task_id).cloned();
if let Some(hook) = hook {
hook.handoff.notify_one();
hook.finish.notified().await;
}
}
pub(super) async fn pause_completed_retention_before_publish(task_id: &str, status: &HealTaskStatus) {
let hook = COMPLETED_RETENTION_HOOKS.lock().await.get(task_id).cloned();
if let Some(hook) = hook.filter(|hook| hook.pause_before_publish) {
*hook.prepared_status.lock().await = Some(status.clone());
hook.before_publish.notify_one();
hook.publish.notified().await;
}
}
#[tokio::test]
async fn completed_retention_cancel_wins_over_a_prepared_retry_snapshot() {
let bucket = "completed-retention-retry-cancel";
let manager = HealManager::new(Arc::new(MockStorage), None);
let request = HealRequest::object(bucket.to_string(), "object".to_string(), None);
let task_id = request.id.clone();
let duplicate = HealRequest::object(bucket.to_string(), "object".to_string(), None);
let alias = duplicate.id.clone();
let hook = Arc::new(CompletedRetentionHook {
pause_before_publish: true,
..Default::default()
});
{
let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await;
hooks.insert(bucket.to_string(), Arc::clone(&hook));
hooks.insert(task_id.clone(), Arc::clone(&hook));
}
manager.submit_heal_request(request).await.expect("admit original");
manager.submit_heal_request(duplicate).await.expect("admit alias");
process_manager_queue_once(&manager).await;
tokio::time::timeout(Duration::from_secs(5), hook.started.notified())
.await
.expect("scheduler starts");
let task = manager.active_heals.lock().await.get(&task_id).cloned().expect("active task");
task.progress.write().await.update_object_progress(1, 1, 0, 0, 4096);
hook.execute.notify_one();
tokio::time::timeout(Duration::from_secs(5), hook.before_publish.notified())
.await
.expect("retry snapshot prepared");
manager.cancel_task(&alias).await.expect("cancel wins active ownership");
assert!(matches!(*hook.prepared_status.lock().await, Some(HealTaskStatus::Retrying { .. })));
hook.publish.notify_one();
tokio::time::timeout(Duration::from_secs(5), hook.handoff.notified())
.await
.expect("scheduler finishes handoff");
for token in [&task_id, &alias] {
let report = manager.get_task_report(token).await.expect("cancelled token retained");
assert_eq!(report.status, HealTaskStatus::Cancelled);
assert_eq!(report.progress.expect("frozen progress").objects_scanned, 1);
}
assert!(!manager.retrying_heals.lock().await.contains_key(&task_id));
assert!(!manager.heal_queue.lock().await.contains_request_id(&task_id));
hook.finish.notify_one();
COMPLETED_RETENTION_HOOKS
.lock()
.await
.retain(|key, _| key != bucket && key != &task_id);
}
#[tokio::test]
async fn completed_retention_scheduler_preserves_progress_aliases_and_atomic_handoff() {
for outcome in ["success", "failed", "cancelled"] {
let bucket = format!("completed-retention-{outcome}");
let hook = Arc::new(CompletedRetentionHook::default());
let manager = Arc::new(HealManager::new(Arc::new(MockStorage), None));
let request = HealRequest::object(bucket.clone(), "object".to_string(), None);
let task_id = request.id.clone();
let duplicate = HealRequest::object(bucket.clone(), "object".to_string(), None);
let alias = duplicate.id.clone();
{
let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await;
hooks.insert(bucket.clone(), Arc::clone(&hook));
hooks.insert(task_id.clone(), Arc::clone(&hook));
}
manager.submit_heal_request(request).await.expect("admit original");
manager.submit_heal_request(duplicate).await.expect("admit alias");
process_manager_queue_once(&manager).await;
tokio::time::timeout(Duration::from_secs(5), hook.started.notified())
.await
.expect("scheduler reaches storage");
let task = manager
.active_heals
.lock()
.await
.get(&task_id)
.cloned()
.expect("task is active");
task.progress.write().await.update_object_progress(1, 1, 0, 0, 4096);
let before = manager.get_task_report(&alias).await.expect("alias resolves active progress");
assert_eq!(before.progress.as_ref().expect("active progress").objects_scanned, 1);
let poll_manager = Arc::clone(&manager);
let poll_alias = alias.clone();
let stop = CancellationToken::new();
let poll_stop = stop.clone();
let polling = tokio::spawn(async move {
while !poll_stop.is_cancelled() {
let report = poll_manager
.get_task_report(&poll_alias)
.await
.expect("handoff must never return NotFound");
assert!(report.progress.expect("progress never disappears").objects_scanned >= 1);
tokio::task::yield_now().await;
}
});
if outcome == "cancelled" {
manager.cancel_task(&alias).await.expect("cancel active task by alias");
} else {
hook.execute.notify_one();
}
tokio::time::timeout(Duration::from_secs(5), hook.handoff.notified())
.await
.expect("scheduler archives terminal");
assert!(!manager.active_heals.lock().await.contains_key(&task_id));
let expected = task.get_progress().await;
for token in [&task_id, &alias] {
assert_eq!(manager.get_task_progress(token).await.expect("terminal progress query"), expected);
let report = manager
.get_task_report_for_path_since(&format!("{bucket}/object"), token, Some(u64::MAX))
.await
.expect("terminal token remains queryable at handoff");
assert_eq!(report.progress.as_ref(), Some(&expected));
assert!(report.result_items.is_empty());
match outcome {
"success" => assert_eq!(report.status, HealTaskStatus::Completed),
"failed" => assert!(matches!(report.status, HealTaskStatus::Failed { .. })),
_ => assert_eq!(report.status, HealTaskStatus::Cancelled),
}
}
let retained = manager.completed_heals.lock().await;
assert!(Arc::ptr_eq(&retained[&task_id], &retained[&alias]));
drop(retained);
stop.cancel();
polling.await.expect("concurrent polling succeeds");
// Archived progress must not alias a mutable live progress object.
task.progress.write().await.objects_scanned = 999;
assert_eq!(manager.get_task_report(&alias).await.expect("frozen report").progress, Some(expected));
hook.finish.notify_one();
COMPLETED_RETENTION_HOOKS
.lock()
.await
.retain(|key, _| key != &bucket && key != &task_id);
}
}
#[async_trait::async_trait] #[async_trait::async_trait]
impl HealStorageAPI for MockStorage { impl HealStorageAPI for MockStorage {
async fn get_object_meta(&self, _bucket: &str, _object: &str) -> Result<Option<HealObjectInfo>> { async fn get_object_meta(&self, _bucket: &str, _object: &str) -> Result<Option<HealObjectInfo>> {
@@ -123,6 +443,12 @@ impl HealStorageAPI for MockStorage {
} }
async fn object_exists(&self, bucket: &str, _object: &str) -> Result<bool> { async fn object_exists(&self, bucket: &str, _object: &str) -> Result<bool> {
let hook = COMPLETED_RETENTION_HOOKS.lock().await.get(bucket).cloned();
if let Some(hook) = hook {
hook.started.notify_one();
hook.execute.notified().await;
return Ok(true);
}
Ok(bucket == "retry-transition") Ok(bucket == "retry-transition")
} }
@@ -133,13 +459,18 @@ impl HealStorageAPI for MockStorage {
_version_id: Option<&str>, _version_id: Option<&str>,
_opts: &HealOpts, _opts: &HealOpts,
) -> Result<(HealResultItem, Option<Error>)> { ) -> Result<(HealResultItem, Option<Error>)> {
if bucket == "completed-retention-failed" {
return Err(Error::TaskExecutionFailed {
message: "retention fixture failure".to_string(),
});
}
if let Some(hook) = manager_recovery_test_hook() { if let Some(hook) = manager_recovery_test_hook() {
*hook *hook
.heal_object_calls .heal_object_calls
.lock() .lock()
.expect("manager recovery object call lock should not poison") += 1; .expect("manager recovery object call lock should not poison") += 1;
} }
if bucket == "retry-transition" { if matches!(bucket, "retry-transition" | "completed-retention-retry-cancel") {
return Ok(( return Ok((
HealResultItem::default(), HealResultItem::default(),
Some(Error::Storage(EcstoreError::InsufficientReadQuorum( Some(Error::Storage(EcstoreError::InsufficientReadQuorum(
@@ -1145,7 +1476,13 @@ async fn test_active_duplicate_token_can_query_and_cancel_original_task() {
.expect("duplicate token should cancel merged active task"); .expect("duplicate token should cancel merged active task");
assert!(manager.active_heals.lock().await.get(&active_task_id).is_none()); assert!(manager.active_heals.lock().await.get(&active_task_id).is_none());
assert!(matches!(manager.get_task_status(&active_task_id).await, Err(Error::TaskNotFound { .. }))); assert_eq!(
manager
.get_task_status(&active_task_id)
.await
.expect("cancelled task remains queryable"),
HealTaskStatus::Cancelled
);
} }
#[tokio::test] #[tokio::test]
@@ -1638,6 +1975,8 @@ async fn insert_retrying_request(manager: &HealManager, request: HealRequest) ->
manager.completed_heals.lock().await.insert( manager.completed_heals.lock().await.insert(
task_id, task_id,
Arc::new(CompletedHealStatus { Arc::new(CompletedHealStatus {
progress: None,
retained_bytes: std::sync::OnceLock::new(),
heal_type: request.heal_type, heal_type: request.heal_type,
status: HealTaskStatus::Retrying { status: HealTaskStatus::Retrying {
error: "Lock acquisition timeout".to_string(), error: "Lock acquisition timeout".to_string(),
@@ -2053,7 +2392,7 @@ async fn admin_force_start_cancels_overlapping_active_task_first() {
"the overlapping admin task must be cancelled (removed from the active table) before the new one starts" "the overlapping admin task must be cancelled (removed from the active table) before the new one starts"
); );
assert!( assert!(
matches!(manager.get_task_status(&old_id).await, Err(Error::TaskNotFound { .. })), matches!(manager.get_task_status(&old_id).await, Ok(HealTaskStatus::Cancelled)),
"a cancelled task must no longer resolve as an active heal" "a cancelled task must no longer resolve as an active heal"
); );
} }
@@ -2360,6 +2699,8 @@ async fn test_retrying_completion_outranks_the_queue_for_the_same_id() {
manager.completed_heals.lock().await.insert( manager.completed_heals.lock().await.insert(
task_id.clone(), task_id.clone(),
Arc::new(CompletedHealStatus { Arc::new(CompletedHealStatus {
progress: None,
retained_bytes: std::sync::OnceLock::new(),
heal_type: request.heal_type.clone(), heal_type: request.heal_type.clone(),
status: HealTaskStatus::Retrying { status: HealTaskStatus::Retrying {
error: "transient disk failure".to_string(), error: "transient disk failure".to_string(),
@@ -2395,6 +2736,8 @@ async fn test_get_task_status_reads_recent_completed_status() {
manager.completed_heals.lock().await.insert( manager.completed_heals.lock().await.insert(
"completed-token".to_string(), "completed-token".to_string(),
Arc::new(CompletedHealStatus { Arc::new(CompletedHealStatus {
progress: None,
retained_bytes: std::sync::OnceLock::new(),
heal_type: HealType::Bucket { heal_type: HealType::Bucket {
bucket: "bucket".to_string(), bucket: "bucket".to_string(),
}, },
@@ -2424,6 +2767,8 @@ async fn test_get_task_report_for_path_reads_completed_items() {
manager.completed_heals.lock().await.insert( manager.completed_heals.lock().await.insert(
"completed-token".to_string(), "completed-token".to_string(),
Arc::new(CompletedHealStatus { Arc::new(CompletedHealStatus {
progress: None,
retained_bytes: std::sync::OnceLock::new(),
heal_type: HealType::Object { heal_type: HealType::Object {
bucket: "bucket".to_string(), bucket: "bucket".to_string(),
object: "object".to_string(), object: "object".to_string(),
+1 -1
View File
@@ -999,7 +999,7 @@ impl HealTask {
let items = match since { let items = match since {
None => result_items.iter().map(|(_, item)| item.clone()).collect::<Vec<_>>(), None => result_items.iter().map(|(_, item)| item.clone()).collect::<Vec<_>>(),
Some(cursor) => { Some(cursor) => {
if cursor + 1 < min_seq { if cursor.saturating_add(1) < min_seq {
lagged = true; lagged = true;
} }
result_items result_items
+72
View File
@@ -182,6 +182,9 @@ pub struct BackgroundHealStatus {
pub heal_active_tasks: u64, pub heal_active_tasks: u64,
#[serde(default)] #[serde(default)]
pub cluster_status_complete: bool, pub cluster_status_complete: bool,
/// Missing on older servers; absent coverage or counts mean unknown.
#[serde(default)]
pub coverage: Option<BackgroundHealCoverage>,
#[serde(default)] #[serde(default)]
pub progress: Option<serde_json::Value>, pub progress: Option<serde_json::Value>,
/// Remaining wire fields (flattened `BackgroundHealInfo` plus the /// Remaining wire fields (flattened `BackgroundHealInfo` plus the
@@ -190,6 +193,22 @@ pub struct BackgroundHealStatus {
pub extra: serde_json::Map<String, serde_json::Value>, pub extra: serde_json::Map<String, serde_json::Value>,
} }
/// Node coverage of a background heal status snapshot. Counters describe only
/// nodes with usable snapshots; unknown peers may still be running heal work.
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct BackgroundHealCoverage {
#[serde(default)]
pub expected: Option<usize>,
#[serde(default)]
pub responded: Option<usize>,
#[serde(default)]
pub unknown: Option<usize>,
/// Stable reason codes; unknown future codes are preserved verbatim.
#[serde(default)]
pub reasons: Vec<String>,
}
/// `GET /v3/scanner/status` response, typed at the fields operators branch /// `GET /v3/scanner/status` response, typed at the fields operators branch
/// on; everything else passes through verbatim. /// on; everything else passes through verbatim.
#[derive(Debug, Clone, Deserialize)] #[derive(Debug, Clone, Deserialize)]
@@ -630,9 +649,39 @@ mod tests {
assert_eq!(status.state, "active"); assert_eq!(status.state, "active");
assert_eq!(status.heal_queue_length, 3); assert_eq!(status.heal_queue_length, 3);
assert!(status.cluster_status_complete); assert!(status.cluster_status_complete);
assert!(status.coverage.is_none(), "legacy payloads have unknown coverage");
assert!(status.extra.contains_key("healOperations"), "unknown nested payloads must pass through"); assert!(status.extra.contains_key("healOperations"), "unknown nested payloads must pass through");
} }
#[test]
fn background_heal_status_missing_coverage_fields_remain_unknown() {
for raw in [json!({"state": "degraded"}), json!({"state": "degraded", "coverage": {}})] {
let status: BackgroundHealStatus = serde_json::from_value(raw).expect("partial legacy payload decodes");
assert!(!status.cluster_status_complete);
if let Some(coverage) = status.coverage {
assert_eq!(coverage.expected, None);
assert_eq!(coverage.responded, None);
assert_eq!(coverage.unknown, None);
}
}
}
#[test]
fn background_heal_status_preserves_future_fields_and_reasons() {
let raw = json!({
"state": "degraded", "clusterStatusComplete": false,
"coverage": {"expected": 3, "responded": 1, "unknown": 2, "reasons": ["future_reason"], "futureCoverage": true},
"futureStatus": {"value": 7}
});
let status: BackgroundHealStatus = serde_json::from_value(raw).expect("future additive fields decode");
assert_eq!(status.extra["futureStatus"]["value"], 7);
let coverage = status.coverage.expect("coverage supplied");
assert_eq!(coverage.expected, Some(3));
assert_eq!(coverage.responded, Some(1));
assert_eq!(coverage.unknown, Some(2));
assert_eq!(coverage.reasons, ["future_reason"]);
}
#[test] #[test]
fn scanner_status_defaults_freshness_to_unknown() { fn scanner_status_defaults_freshness_to_unknown() {
let raw = json!({"enabled": true, "freshness": {"state": "stale"}, "metrics": {}}); let raw = json!({"enabled": true, "freshness": {"state": "stale"}, "metrics": {}});
@@ -721,6 +770,7 @@ mod tests {
let status = client.background_heal_status().await.expect("status decodes"); let status = client.background_heal_status().await.expect("status decodes");
assert_eq!(status.state, "idle"); assert_eq!(status.state, "idle");
assert!(status.coverage.is_none(), "older HTTP responses retain unknown coverage");
let request = server.recorded(); let request = server.recorded();
// The server registers this route POST-only; a GET here answers 405. // The server registers this route POST-only; a GET here answers 405.
assert_eq!(request.method, "POST"); assert_eq!(request.method, "POST");
@@ -728,6 +778,28 @@ mod tests {
assert_eq!(request.query, ""); assert_eq!(request.query, "");
} }
#[tokio::test]
async fn background_heal_status_decodes_partial_coverage_over_http() {
let body = r#"{"state":"degraded","healQueueLength":0,"healActiveTasks":0,"clusterStatusComplete":false,"coverage":{"expected":3,"responded":1,"unknown":2,"reasons":["notification_system_unavailable"]},"futureStatus":true}"#;
let server = TestServer::spawn(body, 200).await;
let client = AdminClient::new(&format!("http://{}", server.addr), "ak", "sk").expect("client builds");
let status = client
.background_heal_status()
.await
.expect("partial status is a successful response");
assert_eq!(status.state, "degraded");
assert!(!status.cluster_status_complete);
assert_eq!(status.extra["futureStatus"], true);
let coverage = status.coverage.expect("partial coverage supplied");
assert_eq!(coverage.expected, Some(3));
assert_eq!(coverage.responded, Some(1));
assert_eq!(coverage.unknown, Some(2));
assert_eq!(coverage.reasons, ["notification_system_unavailable"]);
let request = server.recorded();
assert_eq!(request.method, "POST");
assert_eq!(request.query, "", "reading status must not send heal control parameters");
}
#[tokio::test] #[tokio::test]
async fn http_error_status_maps_to_a_typed_error_with_body() { async fn http_error_status_maps_to_a_typed_error_with_body() {
let server = TestServer::spawn(r#"{"code":"AccessDenied","message":"denied"}"#, 403).await; let server = TestServer::spawn(r#"{"code":"AccessDenied","message":"denied"}"#, 403).await;
@@ -1283,6 +1283,54 @@ pub struct ScannerDirtyUsageSnapshotResponse {
#[prost(bytes = "bytes", tag = "7")] #[prost(bytes = "bytes", tag = "7")]
pub response_proof: ::prost::bytes::Bytes, pub response_proof: ::prost::bytes::Bytes,
} }
/// Receiver-only protocol. Producers must retain whole-cycle ACK until they
/// have a durable per-bucket publication proof.
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ScannerScopedDirtyUsageEntry {
#[prost(string, tag = "1")]
pub bucket: ::prost::alloc::string::String,
#[prost(bytes = "bytes", tag = "2")]
pub bucket_incarnation: ::prost::bytes::Bytes,
#[prost(uint64, tag = "3")]
pub generation: u64,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ScannerScopedDirtyUsageAckRequest {
#[prost(bytes = "bytes", tag = "1")]
pub challenge: ::prost::bytes::Bytes,
#[prost(uint32, tag = "2")]
pub protocol_version: u32,
#[prost(string, tag = "3")]
pub owner_id: ::prost::alloc::string::String,
#[prost(string, tag = "4")]
pub instance_id: ::prost::alloc::string::String,
/// Only scope 1 (a complete bucket) is supported; zero is invalid.
#[prost(uint32, tag = "5")]
pub scope: u32,
#[prost(bool, tag = "6")]
pub probe_only: bool,
#[prost(message, repeated, tag = "7")]
pub entries: ::prost::alloc::vec::Vec<ScannerScopedDirtyUsageEntry>,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ScannerScopedDirtyUsageAckResponse {
#[prost(uint32, tag = "1")]
pub protocol_version: u32,
#[prost(string, tag = "2")]
pub owner_id: ::prost::alloc::string::String,
#[prost(string, tag = "3")]
pub instance_id: ::prost::alloc::string::String,
#[prost(bool, tag = "4")]
pub supported: bool,
#[prost(uint32, tag = "5")]
pub max_entries: u32,
#[prost(uint32, tag = "6")]
pub max_request_bytes: u32,
#[prost(uint64, tag = "7")]
pub cleared: u64,
#[prost(bytes = "bytes", tag = "8")]
pub response_proof: ::prost::bytes::Bytes,
}
/// A short-lived storage-owned read admission used only around a final /// A short-lived storage-owned read admission used only around a final
/// scanner metadata publication. It is intentionally separate from the /// scanner metadata publication. It is intentionally separate from the
/// ScannerActivity observation wire so v6/v7 rolling compatibility remains /// ScannerActivity observation wire so v6/v7 rolling compatibility remains
@@ -6282,6 +6330,244 @@ pub mod node_service_server {
} }
} }
/// Generated client implementations. /// Generated client implementations.
pub mod scanner_control_service_client {
#![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]
use tonic::codegen::http::Uri;
use tonic::codegen::*;
#[derive(Debug, Clone)]
pub struct ScannerControlServiceClient<T> {
inner: tonic::client::Grpc<T>,
}
impl ScannerControlServiceClient<tonic::transport::Channel> {
/// Attempt to create a new client by connecting to a given endpoint.
pub async fn connect<D>(dst: D) -> Result<Self, tonic::transport::Error>
where
D: TryInto<tonic::transport::Endpoint>,
D::Error: Into<StdError>,
{
let conn = tonic::transport::Endpoint::new(dst)?.connect().await?;
Ok(Self::new(conn))
}
}
impl<T> ScannerControlServiceClient<T>
where
T: tonic::client::GrpcService<tonic::body::Body>,
T::Error: Into<StdError>,
T::ResponseBody: Body<Data = Bytes> + std::marker::Send + 'static,
<T::ResponseBody as Body>::Error: Into<StdError> + std::marker::Send,
{
pub fn new(inner: T) -> Self {
let inner = tonic::client::Grpc::new(inner);
Self { inner }
}
pub fn with_origin(inner: T, origin: Uri) -> Self {
let inner = tonic::client::Grpc::with_origin(inner, origin);
Self { inner }
}
pub fn with_interceptor<F>(inner: T, interceptor: F) -> ScannerControlServiceClient<InterceptedService<T, F>>
where
F: tonic::service::Interceptor,
T::ResponseBody: Default,
T: tonic::codegen::Service<
http::Request<tonic::body::Body>,
Response = http::Response<<T as tonic::client::GrpcService<tonic::body::Body>>::ResponseBody>,
>,
<T as tonic::codegen::Service<http::Request<tonic::body::Body>>>::Error:
Into<StdError> + std::marker::Send + std::marker::Sync,
{
ScannerControlServiceClient::new(InterceptedService::new(inner, interceptor))
}
/// Compress requests with the given encoding.
///
/// This requires the server to support it otherwise it might respond with an
/// error.
#[must_use]
pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
self.inner = self.inner.send_compressed(encoding);
self
}
/// Enable decompressing responses.
#[must_use]
pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
self.inner = self.inner.accept_compressed(encoding);
self
}
/// Limits the maximum size of a decoded message.
///
/// Default: `4MB`
#[must_use]
pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
self.inner = self.inner.max_decoding_message_size(limit);
self
}
/// Limits the maximum size of an encoded message.
///
/// Default: `usize::MAX`
#[must_use]
pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
self.inner = self.inner.max_encoding_message_size(limit);
self
}
pub async fn scanner_scoped_dirty_usage_ack(
&mut self,
request: impl tonic::IntoRequest<super::ScannerScopedDirtyUsageAckRequest>,
) -> std::result::Result<tonic::Response<super::ScannerScopedDirtyUsageAckResponse>, tonic::Status> {
self.inner
.ready()
.await
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static("/node_service.ScannerControlService/ScannerScopedDirtyUsageAck");
let mut req = request.into_request();
req.extensions_mut()
.insert(GrpcMethod::new("node_service.ScannerControlService", "ScannerScopedDirtyUsageAck"));
self.inner.unary(req, path, codec).await
}
}
}
/// Generated server implementations.
pub mod scanner_control_service_server {
#![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]
use tonic::codegen::*;
/// Generated trait containing gRPC methods that should be implemented for use with ScannerControlServiceServer.
#[async_trait]
pub trait ScannerControlService: std::marker::Send + std::marker::Sync + 'static {
async fn scanner_scoped_dirty_usage_ack(
&self,
request: tonic::Request<super::ScannerScopedDirtyUsageAckRequest>,
) -> std::result::Result<tonic::Response<super::ScannerScopedDirtyUsageAckResponse>, tonic::Status>;
}
#[derive(Debug)]
pub struct ScannerControlServiceServer<T> {
inner: Arc<T>,
accept_compression_encodings: EnabledCompressionEncodings,
send_compression_encodings: EnabledCompressionEncodings,
max_decoding_message_size: Option<usize>,
max_encoding_message_size: Option<usize>,
}
impl<T> ScannerControlServiceServer<T> {
pub fn new(inner: T) -> Self {
Self::from_arc(Arc::new(inner))
}
pub fn from_arc(inner: Arc<T>) -> Self {
Self {
inner,
accept_compression_encodings: Default::default(),
send_compression_encodings: Default::default(),
max_decoding_message_size: None,
max_encoding_message_size: None,
}
}
pub fn with_interceptor<F>(inner: T, interceptor: F) -> InterceptedService<Self, F>
where
F: tonic::service::Interceptor,
{
InterceptedService::new(Self::new(inner), interceptor)
}
/// Enable decompressing requests with the given encoding.
#[must_use]
pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
self.accept_compression_encodings.enable(encoding);
self
}
/// Compress responses with the given encoding, if the client supports it.
#[must_use]
pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
self.send_compression_encodings.enable(encoding);
self
}
/// Limits the maximum size of a decoded message.
///
/// Default: `4MB`
#[must_use]
pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
self.max_decoding_message_size = Some(limit);
self
}
/// Limits the maximum size of an encoded message.
///
/// Default: `usize::MAX`
#[must_use]
pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
self.max_encoding_message_size = Some(limit);
self
}
}
impl<T, B> tonic::codegen::Service<http::Request<B>> for ScannerControlServiceServer<T>
where
T: ScannerControlService,
B: Body + std::marker::Send + 'static,
B::Error: Into<StdError> + std::marker::Send + 'static,
{
type Response = http::Response<tonic::body::Body>;
type Error = std::convert::Infallible;
type Future = BoxFuture<Self::Response, Self::Error>;
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>> {
Poll::Ready(Ok(()))
}
fn call(&mut self, req: http::Request<B>) -> Self::Future {
match req.uri().path() {
"/node_service.ScannerControlService/ScannerScopedDirtyUsageAck" => {
#[allow(non_camel_case_types)]
struct ScannerScopedDirtyUsageAckSvc<T: ScannerControlService>(pub Arc<T>);
impl<T: ScannerControlService> tonic::server::UnaryService<super::ScannerScopedDirtyUsageAckRequest>
for ScannerScopedDirtyUsageAckSvc<T>
{
type Response = super::ScannerScopedDirtyUsageAckResponse;
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
fn call(&mut self, request: tonic::Request<super::ScannerScopedDirtyUsageAckRequest>) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
<T as ScannerControlService>::scanner_scoped_dirty_usage_ack(&inner, request).await
};
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let method = ScannerScopedDirtyUsageAckSvc(inner);
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
let res = grpc.unary(method, req).await;
Ok(res)
};
Box::pin(fut)
}
_ => Box::pin(async move {
let mut response = http::Response::new(tonic::body::Body::default());
let headers = response.headers_mut();
headers.insert(tonic::Status::GRPC_STATUS, (tonic::Code::Unimplemented as i32).into());
headers.insert(http::header::CONTENT_TYPE, tonic::metadata::GRPC_CONTENT_TYPE);
Ok(response)
}),
}
}
}
impl<T> Clone for ScannerControlServiceServer<T> {
fn clone(&self) -> Self {
let inner = self.inner.clone();
Self {
inner,
accept_compression_encodings: self.accept_compression_encodings,
send_compression_encodings: self.send_compression_encodings,
max_decoding_message_size: self.max_decoding_message_size,
max_encoding_message_size: self.max_encoding_message_size,
}
}
}
/// Generated gRPC service name
pub const SERVICE_NAME: &str = "node_service.ScannerControlService";
impl<T> tonic::server::NamedService for ScannerControlServiceServer<T> {
const NAME: &'static str = SERVICE_NAME;
}
}
/// Generated client implementations.
pub mod heal_control_service_client { pub mod heal_control_service_client {
#![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)] #![allow(unused_variables, dead_code, missing_docs, clippy::wildcard_imports, clippy::let_unit_value)]
use tonic::codegen::http::Uri; use tonic::codegen::http::Uri;
+2
View File
@@ -541,6 +541,8 @@ pub fn canonical_scanner_activity_v7_response_body(
Ok(body) Ok(body)
} }
pub mod scoped_dirty_usage;
pub fn canonical_scanner_dirty_usage_snapshot_request_body( pub fn canonical_scanner_dirty_usage_snapshot_request_body(
request: &proto_gen::node_service::ScannerDirtyUsageSnapshotRequest, request: &proto_gen::node_service::ScannerDirtyUsageSnapshotRequest,
) -> Result<Vec<u8>, std::num::TryFromIntError> { ) -> Result<Vec<u8>, std::num::TryFromIntError> {
+34
View File
@@ -903,6 +903,36 @@ message ScannerDirtyUsageSnapshotResponse {
bytes response_proof = 7; bytes response_proof = 7;
} }
// Receiver-only protocol. Producers must retain whole-cycle ACK until they
// have a durable per-bucket publication proof.
message ScannerScopedDirtyUsageEntry {
string bucket = 1;
bytes bucket_incarnation = 2;
uint64 generation = 3;
}
message ScannerScopedDirtyUsageAckRequest {
bytes challenge = 1;
uint32 protocol_version = 2;
string owner_id = 3;
string instance_id = 4;
// Only scope 1 (a complete bucket) is supported; zero is invalid.
uint32 scope = 5;
bool probe_only = 6;
repeated ScannerScopedDirtyUsageEntry entries = 7;
}
message ScannerScopedDirtyUsageAckResponse {
uint32 protocol_version = 1;
string owner_id = 2;
string instance_id = 3;
bool supported = 4;
uint32 max_entries = 5;
uint32 max_request_bytes = 6;
uint64 cleared = 7;
bytes response_proof = 8;
}
// A short-lived storage-owned read admission used only around a final // A short-lived storage-owned read admission used only around a final
// scanner metadata publication. It is intentionally separate from the // scanner metadata publication. It is intentionally separate from the
// ScannerActivity observation wire so v6/v7 rolling compatibility remains // ScannerActivity observation wire so v6/v7 rolling compatibility remains
@@ -1245,6 +1275,10 @@ service NodeService {
rpc GetLiveEvents(GetLiveEventsRequest) returns (GetLiveEventsResponse) {}; // auth-policy: read-only rpc GetLiveEvents(GetLiveEventsRequest) returns (GetLiveEventsResponse) {}; // auth-policy: read-only
} }
service ScannerControlService {
rpc ScannerScopedDirtyUsageAck(ScannerScopedDirtyUsageAckRequest) returns (ScannerScopedDirtyUsageAckResponse) {}; // auth-policy: body-bound
}
service HealControlService { service HealControlService {
rpc HealControl(HealControlRequest) returns (HealControlResponse) {}; rpc HealControl(HealControlRequest) returns (HealControlResponse) {};
} }
+213
View File
@@ -0,0 +1,213 @@
// Copyright 2024 RustFS Team
// Licensed under the Apache License, Version 2.0.
//! Bounded, authenticated receiver contract for per-bucket dirty acknowledgements.
use crate::CanonicalBodyBuilder;
use crate::proto_gen::node_service::{ScannerScopedDirtyUsageAckRequest, ScannerScopedDirtyUsageAckResponse};
use prost::Message;
pub const SCOPED_DIRTY_USAGE_PROTOCOL_VERSION: u32 = 1;
pub const SCOPED_DIRTY_USAGE_BUCKET_SCOPE: u32 = 1;
pub const SCOPED_DIRTY_USAGE_MAX_ENTRIES: u32 = 32;
pub const SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES: u32 = 8192;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ScopedDirtyUsageRequestError {
UnsupportedProtocol,
UnsupportedScope,
InvalidIdentity,
InvalidGeneration,
InvalidEntries,
TooLarge,
}
impl std::fmt::Display for ScopedDirtyUsageRequestError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(match self {
Self::UnsupportedProtocol => "unsupported scoped dirty usage protocol",
Self::UnsupportedScope => "unsupported scoped dirty usage scope",
Self::InvalidIdentity => "invalid scoped dirty usage identity",
Self::InvalidGeneration => "invalid scoped dirty usage generation",
Self::InvalidEntries => "scoped dirty usage entries must be nonempty and strictly ordered",
Self::TooLarge => "scoped dirty usage request exceeds its budget",
})
}
}
impl std::error::Error for ScopedDirtyUsageRequestError {}
pub fn validate_scoped_dirty_usage_request(
request: &ScannerScopedDirtyUsageAckRequest,
) -> Result<(), ScopedDirtyUsageRequestError> {
use ScopedDirtyUsageRequestError as E;
if request.entries.len() > SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize
|| request.encoded_len() > SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize
{
return Err(E::TooLarge);
}
if request.protocol_version != SCOPED_DIRTY_USAGE_PROTOCOL_VERSION {
return Err(E::UnsupportedProtocol);
}
if request.scope != SCOPED_DIRTY_USAGE_BUCKET_SCOPE {
return Err(E::UnsupportedScope);
}
if request.challenge.len() != 16 || request.owner_id.len() != 36 || request.instance_id.len() != 32 {
return Err(E::InvalidIdentity);
}
if request.entries.is_empty() || request.entries.windows(2).any(|pair| pair[0].bucket >= pair[1].bucket) {
return Err(E::InvalidEntries);
}
for entry in &request.entries {
if entry.bucket.is_empty()
|| entry.bucket.len() > 63
|| entry.bucket_incarnation.len() != 16
|| entry.bucket_incarnation.iter().all(|byte| *byte == 0)
{
return Err(E::InvalidIdentity);
}
if entry.generation == 0 || entry.generation == u64::MAX {
return Err(E::InvalidGeneration);
}
}
Ok(())
}
pub fn canonical_scoped_dirty_usage_request(
request: &ScannerScopedDirtyUsageAckRequest,
) -> Result<Vec<u8>, ScopedDirtyUsageRequestError> {
validate_scoped_dirty_usage_request(request)?;
let mut body = CanonicalBodyBuilder::new(b"rustfs-scoped-dirty-usage-ack-request-v1\0");
let encode = |_: std::num::TryFromIntError| ScopedDirtyUsageRequestError::TooLarge;
body.push_bytes(request.challenge.as_ref()).map_err(encode)?;
body.push_u32(request.protocol_version);
body.push_str(&request.owner_id).map_err(encode)?;
body.push_str(&request.instance_id).map_err(encode)?;
body.push_u32(request.scope);
body.push_bool(request.probe_only);
body.push_count(request.entries.len()).map_err(encode)?;
for entry in &request.entries {
body.push_str(&entry.bucket).map_err(encode)?;
body.push_bytes(entry.bucket_incarnation.as_ref()).map_err(encode)?;
body.push_u64(entry.generation);
}
Ok(body.finish())
}
pub fn canonical_scoped_dirty_usage_response(
request_body: &[u8],
response: &ScannerScopedDirtyUsageAckResponse,
) -> Result<Vec<u8>, std::num::TryFromIntError> {
let mut body = CanonicalBodyBuilder::new(b"rustfs-scoped-dirty-usage-ack-response-v1\0");
body.push_bytes(request_body)?;
body.push_u32(response.protocol_version);
body.push_str(&response.owner_id)?;
body.push_str(&response.instance_id)?;
body.push_bool(response.supported);
body.push_u32(response.max_entries);
body.push_u32(response.max_request_bytes);
body.push_u64(response.cleared);
Ok(body.finish())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::proto_gen::node_service::ScannerScopedDirtyUsageEntry;
fn request() -> ScannerScopedDirtyUsageAckRequest {
ScannerScopedDirtyUsageAckRequest {
challenge: vec![1; 16].into(),
protocol_version: 1,
owner_id: "11111111-1111-1111-1111-111111111111".into(),
instance_id: "a".repeat(32),
scope: 1,
probe_only: false,
entries: vec![ScannerScopedDirtyUsageEntry {
bucket: "photos".into(),
bucket_incarnation: vec![2; 16].into(),
generation: 8,
}],
}
}
#[test]
fn scoped_dirty_usage_binds_every_request_field() {
let base = request();
let baseline = canonical_scoped_dirty_usage_request(&base).expect("valid request");
for field in 0..9 {
let mut changed = base.clone();
match field {
0 => changed.challenge = vec![3; 16].into(),
1 => changed.protocol_version += 1,
2 => changed.owner_id = "22222222-2222-2222-2222-222222222222".into(),
3 => changed.instance_id = "b".repeat(32),
4 => changed.scope += 1,
5 => changed.probe_only = true,
6 => changed.entries[0].bucket = "videos".into(),
7 => changed.entries[0].bucket_incarnation = vec![3; 16].into(),
_ => changed.entries[0].generation += 1,
}
assert!(canonical_scoped_dirty_usage_request(&changed).map_or(true, |body| body != baseline));
}
}
#[test]
fn scoped_dirty_usage_binds_capability_and_ack_to_exact_request() {
let request = canonical_scoped_dirty_usage_request(&request()).expect("valid request");
let response = ScannerScopedDirtyUsageAckResponse {
protocol_version: 1,
owner_id: "owner".into(),
instance_id: "process".into(),
supported: true,
max_entries: 32,
max_request_bytes: 8192,
cleared: 1,
response_proof: vec![1; 32].into(),
};
let baseline = canonical_scoped_dirty_usage_response(&request, &response).expect("valid response");
for field in 0..7 {
let mut changed = response.clone();
match field {
0 => changed.protocol_version += 1,
1 => changed.owner_id.push('x'),
2 => changed.instance_id.push('x'),
3 => changed.supported = false,
4 => changed.max_entries += 1,
5 => changed.max_request_bytes += 1,
_ => changed.cleared += 1,
}
assert_ne!(
canonical_scoped_dirty_usage_response(&request, &changed).expect("response variant"),
baseline
);
}
assert_ne!(
canonical_scoped_dirty_usage_response(b"another request", &response).expect("request variant"),
baseline
);
}
#[test]
fn scoped_dirty_usage_rejects_overflow_unknown_and_duplicate_entries() {
let base = request();
let mut invalid = base.clone();
invalid.entries = vec![base.entries[0].clone(); SCOPED_DIRTY_USAGE_MAX_ENTRIES as usize + 1];
assert_eq!(validate_scoped_dirty_usage_request(&invalid), Err(ScopedDirtyUsageRequestError::TooLarge));
invalid = base.clone();
invalid.entries[0].bucket = "x".repeat(SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize);
assert_eq!(validate_scoped_dirty_usage_request(&invalid), Err(ScopedDirtyUsageRequestError::TooLarge));
invalid = base.clone();
invalid.entries.push(base.entries[0].clone());
assert_eq!(
validate_scoped_dirty_usage_request(&invalid),
Err(ScopedDirtyUsageRequestError::InvalidEntries)
);
invalid = base;
invalid.entries[0].bucket_incarnation = vec![0; 16].into();
assert_eq!(
validate_scoped_dirty_usage_request(&invalid),
Err(ScopedDirtyUsageRequestError::InvalidIdentity)
);
}
}
+3 -2
View File
@@ -90,8 +90,9 @@ pub use scanner::{
}; };
pub use scanner_io::{ pub use scanner_io::{
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState, ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState,
acknowledge_dirty_usage_generation, clear_dirty_usage_bucket, record_dirty_usage_bucket, record_scanner_maintenance_change, acknowledge_dirty_usage_generation, acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, record_dirty_usage_bucket,
scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state, scanner_maintenance_generation, record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
scanner_maintenance_generation,
}; };
pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER}; pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::atomic::{AtomicU64, Ordering};
+15 -8
View File
@@ -1703,14 +1703,18 @@ where
let (sender, receiver) = mpsc::channel::<DataUsageInfo>(1); let (sender, receiver) = mpsc::channel::<DataUsageInfo>(1);
let done_cycle = Metrics::time(Metric::ScanCycle); let done_cycle = Metrics::time(Metric::ScanCycle);
let scan_result = crate::scanner_io::nsscanner_with_storage_status( let scan_result = crate::scanner_io::nsscanner_with_storage_status_scoped(
storeapi.as_ref(), storeapi.as_ref(),
cycle_budget.token(), crate::scanner_io::ScannerCycleRequest {
cycle_budget.clone(), ctx: cycle_budget.token(),
sender, budget: cycle_budget.clone(),
cycle_info.current, updates: sender,
leader_epoch, want_cycle: cycle_info.current,
scan_mode, leader_epoch,
scan_mode,
scan_scope: crate::scanner_io::ScannerBucketScanScope::default(),
persisted_usage_baseline: usage_persist_baseline.data.clone(),
},
) )
.await; .await;
let publication_defer_reason = match &scan_result { let publication_defer_reason = match &scan_result {
@@ -3424,10 +3428,13 @@ use cycle_state::*;
use leadership::*; use leadership::*;
use usage_store::*; use usage_store::*;
#[cfg(test)]
pub(crate) use activity::scanner_activity_snapshot_digest;
pub use activity::scanner_topology_digest; pub use activity::scanner_topology_digest;
pub(crate) use activity::{ pub(crate) use activity::{
ScannerActivitySnapshot, ScannerDirtyUsageAcknowledgement, probe_scanner_activity, scanner_activity_allows_usage_publication, ScannerActivitySnapshot, ScannerDirtyUsageAcknowledgement, probe_scanner_activity, scanner_activity_allows_usage_publication,
scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest, scanner_dirty_usage_acknowledgements, scanner_activity_dirty_usage_state_for_host, scanner_activity_publication_lease_targets, scanner_activity_structural_digest,
scanner_dirty_usage_acknowledgements,
}; };
pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance}; pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance};
pub use backlog::{ pub use backlog::{
+41
View File
@@ -902,6 +902,7 @@ where
observation observation
} }
#[cfg(test)]
pub(crate) fn scanner_activity_snapshot_digest(snapshot: &ScannerActivitySnapshot) -> [u8; 32] { pub(crate) fn scanner_activity_snapshot_digest(snapshot: &ScannerActivitySnapshot) -> [u8; 32] {
let mut hasher = Sha256::new(); let mut hasher = Sha256::new();
hasher.update(u64::try_from(snapshot.len()).unwrap_or(u64::MAX).to_be_bytes()); hasher.update(u64::try_from(snapshot.len()).unwrap_or(u64::MAX).to_be_bytes());
@@ -925,6 +926,30 @@ pub(crate) fn scanner_activity_snapshot_digest(snapshot: &ScannerActivitySnapsho
hasher.finalize().into() hasher.finalize().into()
} }
/// Hash the activity inputs that make an existing scanner cache unsafe to
/// reuse. Regular namespace writes and dirty-usage generations are omitted:
/// their affected buckets are tracked separately and may be refreshed from a
/// complete authoritative cache baseline.
pub(crate) fn scanner_activity_structural_digest(snapshot: &ScannerActivitySnapshot) -> [u8; 32] {
let mut hasher = Sha256::new();
hasher.update(u64::try_from(snapshot.len()).unwrap_or(u64::MAX).to_be_bytes());
for (host, activity) in snapshot {
let host = host.as_bytes();
let instance_id = activity.instance_id.as_bytes();
hasher.update(u64::try_from(host.len()).unwrap_or(u64::MAX).to_be_bytes());
hasher.update(host);
hasher.update(u64::try_from(instance_id.len()).unwrap_or(u64::MAX).to_be_bytes());
hasher.update(instance_id);
hasher.update(activity.maintenance_generation.to_be_bytes());
hasher.update(activity.protocol_version.to_be_bytes());
hasher.update(activity.topology_digest);
hasher.update([u8::from(activity.data_movement_active)]);
hasher.update(activity.movement_generation.to_be_bytes());
hasher.update([u8::from(activity.publication_blocked)]);
}
hasher.finalize().into()
}
pub(crate) fn scanner_activity_allows_usage_publication(snapshot: &ScannerActivitySnapshot) -> bool { pub(crate) fn scanner_activity_allows_usage_publication(snapshot: &ScannerActivitySnapshot) -> bool {
!snapshot.is_empty() !snapshot.is_empty()
&& snapshot.values().all(|activity| { && snapshot.values().all(|activity| {
@@ -955,6 +980,22 @@ pub(crate) fn scanner_dirty_usage_acknowledgements(snapshot: &ScannerActivitySna
.collect() .collect()
} }
pub(crate) fn scanner_activity_dirty_usage_state_for_host<'a>(
snapshot: &'a ScannerActivitySnapshot,
host: &str,
) -> Option<(&'a str, u64, bool)> {
snapshot
.get(host)
.filter(|_| host != LOCAL_SCANNER_ACTIVITY_NODE)
.map(|activity| {
(
activity.instance_id.as_str(),
activity.dirty_usage_generation,
activity.dirty_usage_pending,
)
})
}
pub fn scanner_topology_digest(storeapi: &ECStore) -> [u8; 32] { pub fn scanner_topology_digest(storeapi: &ECStore) -> [u8; 32] {
let endpoint_pools = storeapi.endpoints(); let endpoint_pools = storeapi.endpoints();
let mut hasher = Sha256::new(); let mut hasher = Sha256::new();
+280 -51
View File
@@ -379,6 +379,12 @@ pub(super) fn decode_recovery_marker_for_reset(
if !matches!(marker_revision, DataUsageCacheRevision::Etag(_)) { if !matches!(marker_revision, DataUsageCacheRevision::Etag(_)) {
return Err(ScannerError::Other("cycle recovery marker has no object revision".to_string())); return Err(ScannerError::Other("cycle recovery marker has no object revision".to_string()));
} }
if let Ok(value) = serde_json::from_slice::<serde_json::Value>(data)
&& let Some(state) = value.get("state")
&& !matches!(state.as_str(), Some("blocked" | "cleanup-pending"))
{
return Err(ScannerError::Other("cycle recovery marker state is unsupported".to_string()));
}
let compat = serde_json::from_slice::<ScannerCycleRecoveryMarkerCompat>(data).ok(); let compat = serde_json::from_slice::<ScannerCycleRecoveryMarkerCompat>(data).ok();
let _schema_version = compat.as_ref().and_then(|marker| marker.schema_version); let _schema_version = compat.as_ref().and_then(|marker| marker.schema_version);
let primary_revision = compat let primary_revision = compat
@@ -406,7 +412,10 @@ pub(super) fn decode_recovery_marker_for_reset(
}; };
let state = match compat.as_ref().and_then(|marker| marker.state.as_deref()) { let state = match compat.as_ref().and_then(|marker| marker.state.as_deref()) {
Some("cleanup-pending") => "cleanup-pending", Some("cleanup-pending") => "cleanup-pending",
_ => "blocked", Some("blocked") | None => "blocked",
Some(_) => {
return Err(ScannerError::Other("cycle recovery marker state is unsupported".to_string()));
}
}; };
let now = unix_now_secs(); let now = unix_now_secs();
Ok(ScannerCycleRecoveryMarker { Ok(ScannerCycleRecoveryMarker {
@@ -721,17 +730,19 @@ async fn mark_cycle_recovery_cleanup_pending(
mut marker: ScannerCycleRecoveryMarker, mut marker: ScannerCycleRecoveryMarker,
marker_revision: &DataUsageCacheRevision, marker_revision: &DataUsageCacheRevision,
expected_epoch: u64, expected_epoch: u64,
owns_reset: &(impl Fn() -> bool + Sync),
) -> Result<(ScannerCycleRecoveryMarker, DataUsageCacheRevision), ScannerError> { ) -> Result<(ScannerCycleRecoveryMarker, DataUsageCacheRevision), ScannerError> {
marker.state = "cleanup-pending".to_string(); marker.state = "cleanup-pending".to_string();
marker.last_attempt_at_unix_secs = unix_now_secs(); marker.last_attempt_at_unix_secs = unix_now_secs();
let bytes = serde_json::to_vec(&marker) let bytes = serde_json::to_vec(&marker)
.map_err(|err| ScannerError::Other(format!("failed to encode cycle recovery marker: {err}")))?; .map_err(|err| ScannerError::Other(format!("failed to encode cycle recovery marker: {err}")))?;
let info = save_config_with_publication_admission_for_epoch( let info = save_reset_config(
storeapi.clone(), storeapi.clone(),
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
bytes, bytes,
marker_revision.preconditions(), marker_revision.preconditions(),
expected_epoch, expected_epoch,
owns_reset,
) )
.await .await
.map_err(|err| ScannerError::Other(format!("failed to mark cycle recovery cleanup pending: {err}")))?; .map_err(|err| ScannerError::Other(format!("failed to mark cycle recovery cleanup pending: {err}")))?;
@@ -933,6 +944,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
.get_write_lock_quiet(Duration::from_secs(5)) .get_write_lock_quiet(Duration::from_secs(5))
.await .await
.map_err(|err| ScannerError::Other(format!("scanner leader lock is busy: {err}")))?; .map_err(|err| ScannerError::Other(format!("scanner leader lock is busy: {err}")))?;
let owns_reset = || !guard.is_lock_lost() && !ctx.is_cancelled();
if guard.is_lock_lost() { if guard.is_lock_lost() {
return Err(ScannerError::Other("scanner leader lock was lost before recovery reset".to_string())); return Err(ScannerError::Other("scanner leader lock was lost before recovery reset".to_string()));
@@ -952,7 +964,27 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
} }
Err(err) => return Err(ScannerError::Other(format!("failed to read cycle recovery marker: {err}"))), Err(err) => return Err(ScannerError::Other(format!("failed to read cycle recovery marker: {err}"))),
}; };
let marker_data = marker_data.ok_or_else(|| ScannerError::Other("scanner cycle recovery marker is absent".to_string()))?; let Some(marker_data) = marker_data else {
// A delete may commit before its reply is lost. Confirm both durable
// fences before treating a retry without its marker as completed.
let (cycle, epoch, revision) = read_cycle_state_for_usage_reset(storeapi.clone()).await?;
let floor = persisted_usage_floor(storeapi.clone()).await?;
if !matches!(revision, DataUsageCacheRevision::Etag(_))
|| epoch < floor.leader_epoch
|| cycle.next < floor.next_cycle
|| !owns_reset()
|| scanner_publication_admission_for_epoch(storeapi.clone(), reset_epoch)
.await
.is_none()
{
return Err(ScannerError::Other(
"scanner cycle recovery marker is absent without a completed reset fence".to_string(),
));
}
set_scanner_cycle_recovery_status(recovery_status("healthy", None, false));
super::notify_scanner_cycle_recovery_wake();
return Ok(());
};
let (marker, force_full_rescan) = match serde_json::from_slice::<ScannerCycleRecoveryMarker>(&marker_data) { let (marker, force_full_rescan) = match serde_json::from_slice::<ScannerCycleRecoveryMarker>(&marker_data) {
Ok(marker) if validate_recovery_marker(&marker).is_ok() => (marker, false), Ok(marker) if validate_recovery_marker(&marker).is_ok() => (marker, false),
_ => (decode_recovery_marker_for_reset(&marker_data, &marker_revision)?, true), _ => (decode_recovery_marker_for_reset(&marker_data, &marker_revision)?, true),
@@ -1026,8 +1058,10 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
} }
}; };
if let Some((primary_cycle, primary_epoch)) = primary_state { if let Some((primary_cycle, primary_epoch)) = primary_state {
verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?;
let (cleanup_marker, cleanup_marker_revision) = let (cleanup_marker, cleanup_marker_revision) =
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker.clone(), &marker_revision, reset_epoch).await?; mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker.clone(), &marker_revision, reset_epoch, &owns_reset)
.await?;
set_scanner_cycle_recovery_status(recovery_status_from_marker(&cleanup_marker, "cleanup-pending")); set_scanner_cycle_recovery_status(recovery_status_from_marker(&cleanup_marker, "cleanup-pending"));
let usage_floor = persisted_usage_floor(storeapi.clone()).await?; let usage_floor = persisted_usage_floor(storeapi.clone()).await?;
let fence_epoch = primary_epoch let fence_epoch = primary_epoch
@@ -1047,12 +1081,14 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
"preserved scanner cycle state exceeds the bounded object size".to_string(), "preserved scanner cycle state exceeds the bounded object size".to_string(),
)); ));
} }
let preserved_info = save_config_with_publication_admission_for_epoch( verify_cycle_reset_intent(storeapi.clone(), &cleanup_marker_revision, &owns_reset).await?;
let preserved_info = save_reset_config(
storeapi.clone(), storeapi.clone(),
DATA_USAGE_BLOOM_NAME_PATH.as_str(), DATA_USAGE_BLOOM_NAME_PATH.as_str(),
preserved_data, preserved_data,
primary_revision.preconditions(), primary_revision.preconditions(),
reset_epoch, reset_epoch,
&owns_reset,
) )
.await .await
.map_err(|err| { .map_err(|err| {
@@ -1072,9 +1108,17 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
"scanner leader lock was lost after fencing newer cycle state".to_string(), "scanner leader lock was lost after fencing newer cycle state".to_string(),
)); ));
} }
fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), fence_epoch, Some(reset_epoch), false) verify_cycle_reset_intent(storeapi.clone(), &cleanup_marker_revision, &owns_reset).await?;
.await fence_scanner_usage_epoch_with_expected_epoch(
.map_err(|err| ScannerError::Other(format!("failed to fence preserved scanner usage epoch: {err}")))?; &ctx,
storeapi.clone(),
fence_epoch,
Some(reset_epoch),
false,
&owns_reset,
)
.await
.map_err(|err| ScannerError::Other(format!("failed to fence preserved scanner usage epoch: {err}")))?;
if guard.is_lock_lost() { if guard.is_lock_lost() {
return Err(ScannerError::Other( return Err(ScannerError::Other(
"scanner leader lock was lost after fencing newer cycle state".to_string(), "scanner leader lock was lost after fencing newer cycle state".to_string(),
@@ -1088,7 +1132,8 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
"scanner cycle state changed before recovery marker cleanup".to_string(), "scanner cycle state changed before recovery marker cleanup".to_string(),
)); ));
} }
delete_config_with_publication_admission_for_epoch( verify_cycle_reset_intent(storeapi.clone(), &cleanup_marker_revision, &owns_reset).await?;
delete_reset_config(
storeapi.clone(), storeapi.clone(),
RUSTFS_META_BUCKET, RUSTFS_META_BUCKET,
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
@@ -1100,6 +1145,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
..Default::default() ..Default::default()
}, },
reset_epoch, reset_epoch,
&owns_reset,
) )
.await .await
.map_err(|err| { .map_err(|err| {
@@ -1149,17 +1195,20 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
// Persist the cleanup-pending phase before rewriting the primary. If the // Persist the cleanup-pending phase before rewriting the primary. If the
// process dies after the rewrite, startup still sees a durable fence and // process dies after the rewrite, startup still sees a durable fence and
// cannot mistake the partially completed reset for a healthy state. // cannot mistake the partially completed reset for a healthy state.
verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?;
let (marker, marker_revision) = if marker.state == "cleanup-pending" { let (marker, marker_revision) = if marker.state == "cleanup-pending" {
(marker, marker_revision) (marker, marker_revision)
} else { } else {
mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker, &marker_revision, reset_epoch).await? mark_cycle_recovery_cleanup_pending(storeapi.clone(), marker, &marker_revision, reset_epoch, &owns_reset).await?
}; };
let rebuilt_info = save_config_with_publication_admission_for_epoch( verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?;
let rebuilt_info = save_reset_config(
storeapi.clone(), storeapi.clone(),
DATA_USAGE_BLOOM_NAME_PATH.as_str(), DATA_USAGE_BLOOM_NAME_PATH.as_str(),
data, data,
primary_revision.preconditions(), primary_revision.preconditions(),
reset_epoch, reset_epoch,
&owns_reset,
) )
.await .await
.map_err(|err| { .map_err(|err| {
@@ -1178,8 +1227,10 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
"scanner leader lock was lost after rebuilding cycle state".to_string(), "scanner leader lock was lost after rebuilding cycle state".to_string(),
)); ));
} }
verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?;
if let Err(err) = if let Err(err) =
fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), leader_epoch, Some(reset_epoch), false).await fence_scanner_usage_epoch_with_expected_epoch(&ctx, storeapi.clone(), leader_epoch, Some(reset_epoch), false, &owns_reset)
.await
{ {
set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus { set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus {
path: DATA_USAGE_BLOOM_NAME_PATH.clone(), path: DATA_USAGE_BLOOM_NAME_PATH.clone(),
@@ -1249,7 +1300,8 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
)); ));
} }
if let Err(err) = delete_config_with_publication_admission_for_epoch( verify_cycle_reset_intent(storeapi.clone(), &marker_revision, &owns_reset).await?;
if let Err(err) = delete_reset_config(
storeapi.clone(), storeapi.clone(),
RUSTFS_META_BUCKET, RUSTFS_META_BUCKET,
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
@@ -1261,6 +1313,7 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
..Default::default() ..Default::default()
}, },
reset_epoch, reset_epoch,
&owns_reset,
) )
.await .await
{ {
@@ -1310,6 +1363,57 @@ pub async fn reset_scanner_cycle_recovery(ctx: CancellationToken, storeapi: Arc<
Ok(()) Ok(())
} }
async fn verify_cycle_reset_intent(
storeapi: Arc<impl ScannerObjectIO>,
expected_revision: &DataUsageCacheRevision,
owns_reset: &(impl Fn() -> bool + Sync),
) -> Result<(), ScannerError> {
let revision = read_config_revision(storeapi, DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
.await
.map_err(|err| ScannerError::Other(format!("failed to verify scanner cycle reset intent: {err}")))?;
if &revision != expected_revision {
return Err(ScannerError::Other("scanner cycle reset intent changed".to_string()));
}
if !owns_reset() {
return Err(ScannerError::Other("scanner cycle reset ownership was lost".to_string()));
}
Ok(())
}
async fn save_reset_config(
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
path: &str,
data: Vec<u8>,
preconditions: crate::HTTPPreconditions,
expected_epoch: u64,
owns_reset: &(impl Fn() -> bool + Sync),
) -> Result<crate::ScannerObjectInfo, EcstoreError> {
let Some(_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch).await else {
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
};
if !owns_reset() {
return Err(EcstoreError::other("scanner reset ownership was lost before write"));
}
save_config_with_preconditions(storeapi, path, data, preconditions).await
}
async fn delete_reset_config(
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
bucket: &str,
path: &str,
options: ScannerObjectOptions,
expected_epoch: u64,
owns_reset: &(impl Fn() -> bool + Sync),
) -> Result<crate::ScannerObjectInfo, EcstoreError> {
let Some(_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), expected_epoch).await else {
return Err(EcstoreError::other(SCANNER_PUBLICATION_EPOCH_CHANGED));
};
if !owns_reset() {
return Err(EcstoreError::other("scanner reset ownership was lost before delete"));
}
storeapi.delete_config_object(bucket, path, options).await
}
fn scanner_usage_state_reset_paths() -> Vec<String> { fn scanner_usage_state_reset_paths() -> Vec<String> {
vec![ vec![
DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(), DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(),
@@ -1333,8 +1437,14 @@ pub(super) async fn read_usage_state_reset_slots(
Ok(slots) Ok(slots)
} }
fn usage_state_reset_floor(slots: &[ScannerUsageStateResetSlot]) -> Result<PersistedUsageFloor, ScannerError> { enum ScannerUsageResetFloor {
let mut floor = PersistedUsageFloor::default(); Missing,
Trusted(PersistedUsageFloor),
Corrupt,
}
fn usage_state_reset_floor(slots: &[ScannerUsageStateResetSlot]) -> Result<ScannerUsageResetFloor, ScannerError> {
let mut floor = None;
for slot in slots { for slot in slots {
let Some(data) = slot.data.as_deref() else { let Some(data) = slot.data.as_deref() else {
continue; continue;
@@ -1342,9 +1452,21 @@ fn usage_state_reset_floor(slots: &[ScannerUsageStateResetSlot]) -> Result<Persi
let Ok(usage) = serde_json::from_slice::<DataUsageInfo>(data) else { let Ok(usage) = serde_json::from_slice::<DataUsageInfo>(data) else {
continue; continue;
}; };
update_persisted_usage_floor(&mut floor, &usage, &slot.path)?; if !data_usage_info_has_persisted_baseline_identity(&usage)
&& !(slot.path == DATA_USAGE_OBJ_NAME_PATH.as_str() && data_usage_info_is_bootstrap_pending(&usage))
&& legacy_incomplete_usage_fence(data, &usage)
.and_then(|fence| fence.claimable_epoch())
.is_none()
{
continue;
}
update_persisted_usage_floor(floor.get_or_insert_with(PersistedUsageFloor::default), &usage, &slot.path)?;
} }
Ok(floor) Ok(match floor {
Some(floor) => ScannerUsageResetFloor::Trusted(floor),
None if slots.iter().any(|slot| slot.data.is_some()) => ScannerUsageResetFloor::Corrupt,
None => ScannerUsageResetFloor::Missing,
})
} }
async fn read_cycle_state_for_usage_reset( async fn read_cycle_state_for_usage_reset(
@@ -1401,11 +1523,12 @@ async fn delete_usage_state_reset_slot(
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>, storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
slot: &ScannerUsageStateResetSlot, slot: &ScannerUsageStateResetSlot,
expected_epoch: u64, expected_epoch: u64,
owns_reset: &(impl Fn() -> bool + Sync),
) -> Result<bool, ScannerError> { ) -> Result<bool, ScannerError> {
if matches!(slot.revision, DataUsageCacheRevision::Missing) { if matches!(slot.revision, DataUsageCacheRevision::Missing) {
return Ok(false); return Ok(false);
} }
let delete_result = delete_config_with_publication_admission_for_epoch( let delete_result = delete_reset_config(
storeapi.clone(), storeapi.clone(),
RUSTFS_META_BUCKET, RUSTFS_META_BUCKET,
&slot.path, &slot.path,
@@ -1415,6 +1538,7 @@ async fn delete_usage_state_reset_slot(
..Default::default() ..Default::default()
}, },
expected_epoch, expected_epoch,
owns_reset,
) )
.await; .await;
match delete_result { match delete_result {
@@ -1486,21 +1610,24 @@ pub(super) async fn publish_scanner_usage_bootstrap_primary(
expected_publication_epoch: u64, expected_publication_epoch: u64,
leader_epoch: Option<u64>, leader_epoch: Option<u64>,
context: ScannerUsageBootstrapPublishContext, context: ScannerUsageBootstrapPublishContext,
owns_publication: impl Fn() -> bool + Sync,
) -> Result<(), ScannerError> { ) -> Result<(), ScannerError> {
async fn inner( async fn inner(
storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>, storeapi: Arc<impl ScannerObjectIO + ScannerConfigObjectDelete>,
expected_revision: &DataUsageCacheRevision, expected_revision: &DataUsageCacheRevision,
expected_publication_epoch: u64, expected_publication_epoch: u64,
leader_epoch: Option<u64>, leader_epoch: Option<u64>,
owns_publication: &(impl Fn() -> bool + Sync),
) -> Result<(), ScannerUsageBootstrapPublishError> { ) -> Result<(), ScannerUsageBootstrapPublishError> {
let marker = scanner_usage_bootstrap_marker(std::time::SystemTime::now(), leader_epoch); let marker = scanner_usage_bootstrap_marker(std::time::SystemTime::now(), leader_epoch);
let data = serde_json::to_vec(&marker).map_err(ScannerUsageBootstrapPublishError::Encode)?; let data = serde_json::to_vec(&marker).map_err(ScannerUsageBootstrapPublishError::Encode)?;
let save_result = save_config_with_publication_admission_for_epoch( let save_result = save_reset_config(
storeapi.clone(), storeapi.clone(),
DATA_USAGE_OBJ_NAME_PATH.as_str(), DATA_USAGE_OBJ_NAME_PATH.as_str(),
data.clone(), data.clone(),
expected_revision.preconditions(), expected_revision.preconditions(),
expected_publication_epoch, expected_publication_epoch,
owns_publication,
) )
.await; .await;
if save_result if save_result
@@ -1524,7 +1651,7 @@ pub(super) async fn publish_scanner_usage_bootstrap_primary(
}) })
} }
inner(storeapi, expected_revision, expected_publication_epoch, leader_epoch) inner(storeapi, expected_revision, expected_publication_epoch, leader_epoch, &owns_publication)
.await .await
.map_err(|err| err.into_scanner_error(context)) .map_err(|err| err.into_scanner_error(context))
} }
@@ -1534,32 +1661,108 @@ pub(super) async fn reset_scanner_usage_state_slots_for_full_rebuild(
slots: &[ScannerUsageStateResetSlot], slots: &[ScannerUsageStateResetSlot],
expected_epoch: u64, expected_epoch: u64,
leader_epoch: u64, leader_epoch: u64,
owns_reset: impl Fn() -> bool + Sync,
) -> Result<Vec<String>, ScannerError> { ) -> Result<Vec<String>, ScannerError> {
let mut reset_paths = Vec::new(); let mut reset_paths = Vec::new();
let primary = slots let primary = slots
.iter() .iter()
.find(|slot| slot.path == DATA_USAGE_OBJ_NAME_PATH.as_str()) .find(|slot| slot.path == DATA_USAGE_OBJ_NAME_PATH.as_str())
.ok_or_else(|| ScannerError::Other("scanner usage reset primary slot was not inspected".to_string()))?; .ok_or_else(|| ScannerError::Other("scanner usage reset primary slot was not inspected".to_string()))?;
publish_scanner_usage_bootstrap_primary( if !owns_reset() {
storeapi.clone(), return Err(ScannerError::Other("scanner usage reset ownership was lost".to_string()));
&primary.revision, }
expected_epoch, let resume_epoch = usage_state_reset_resume_epoch(slots)?;
Some(leader_epoch), match resume_epoch {
ScannerUsageBootstrapPublishContext::Reset, Some(epoch) if epoch == leader_epoch => {}
) Some(_) => return Err(ScannerError::Other("scanner usage reset bootstrap epoch changed".to_string())),
.await?; None => {
publish_scanner_usage_bootstrap_primary(
storeapi.clone(),
&primary.revision,
expected_epoch,
Some(leader_epoch),
ScannerUsageBootstrapPublishContext::Reset,
&owns_reset,
)
.await?;
}
}
let (data, intent_revision) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.map_err(|err| ScannerError::Other(format!("failed to inspect scanner usage reset intent: {err}")))?;
data.as_deref()
.and_then(|data| serde_json::from_slice::<DataUsageInfo>(data).ok())
.filter(|usage| data_usage_info_is_bootstrap_pending(usage) && usage.scanner_epoch == Some(leader_epoch))
.ok_or_else(|| ScannerError::Other("scanner usage reset intent changed before cleanup".to_string()))?;
if !matches!(intent_revision, DataUsageCacheRevision::Etag(_))
|| (resume_epoch.is_some() && intent_revision != primary.revision)
{
return Err(ScannerError::Other("scanner usage reset intent revision changed".to_string()));
}
reset_paths.push(DATA_USAGE_OBJ_NAME_PATH.as_str().to_string()); reset_paths.push(DATA_USAGE_OBJ_NAME_PATH.as_str().to_string());
for slot in slots.iter().filter(|slot| slot.path != DATA_USAGE_OBJ_NAME_PATH.as_str()) { for slot in slots.iter().filter(|slot| slot.path != DATA_USAGE_OBJ_NAME_PATH.as_str()) {
if delete_usage_state_reset_slot(storeapi.clone(), slot, expected_epoch).await? { if let Some(usage) = slot
.data
.as_deref()
.and_then(|data| serde_json::from_slice::<DataUsageInfo>(data).ok())
&& usage_epoch(&usage) >= leader_epoch
{
return Err(ScannerError::Other(format!(
"scanner usage reset slot is not older than its intent: {}",
slot.path
)));
}
let revision = read_config_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.map_err(|err| ScannerError::Other(format!("failed to verify scanner usage reset intent: {err}")))?;
if revision != intent_revision {
return Err(ScannerError::Other("scanner usage reset intent changed during cleanup".to_string()));
}
if !owns_reset() {
return Err(ScannerError::Other("scanner usage reset ownership was lost".to_string()));
}
if delete_usage_state_reset_slot(storeapi.clone(), slot, expected_epoch, &owns_reset).await? {
reset_paths.push(slot.path.clone()); reset_paths.push(slot.path.clone());
} }
} }
let revision = read_config_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.map_err(|err| ScannerError::Other(format!("failed to confirm scanner usage reset intent: {err}")))?;
if revision != intent_revision || !owns_reset() {
return Err(ScannerError::Other(
"scanner usage reset intent or ownership changed before completion".to_string(),
));
}
invalidate_admin_data_usage_snapshot_cache().await; invalidate_admin_data_usage_snapshot_cache().await;
invalidate_data_usage_snapshot_cache().await; invalidate_data_usage_snapshot_cache().await;
Ok(reset_paths) Ok(reset_paths)
} }
fn usage_state_reset_resume_epoch(slots: &[ScannerUsageStateResetSlot]) -> Result<Option<u64>, ScannerError> {
let primary = slots.iter().find(|slot| slot.path == DATA_USAGE_OBJ_NAME_PATH.as_str());
let usage = primary
.and_then(|slot| slot.data.as_deref())
.and_then(|data| serde_json::from_slice::<DataUsageInfo>(data).ok());
match usage {
Some(usage) if usage.usage_snapshot_bootstrap_pending => {
if !data_usage_info_is_bootstrap_pending(&usage) {
return Err(ScannerError::Other("scanner usage reset bootstrap is invalid".to_string()));
}
if usage.scanner_epoch.is_none() {
// Initial bootstrap has no reset owner yet.
return Ok(None);
}
usage
.scanner_epoch
.filter(|epoch| *epoch > 0 && *epoch < u64::MAX)
.map(Some)
.ok_or_else(|| ScannerError::Other("scanner usage reset bootstrap has no valid epoch".to_string()))
}
_ => Ok(None),
}
}
pub async fn reset_scanner_usage_state_for_full_rebuild( pub async fn reset_scanner_usage_state_for_full_rebuild(
ctx: CancellationToken, ctx: CancellationToken,
storeapi: Arc<ECStore>, storeapi: Arc<ECStore>,
@@ -1584,12 +1787,31 @@ pub async fn reset_scanner_usage_state_for_full_rebuild(
}; };
let (cycle, cycle_epoch, cycle_revision) = read_cycle_state_for_usage_reset(storeapi.clone()).await?; let (cycle, cycle_epoch, cycle_revision) = read_cycle_state_for_usage_reset(storeapi.clone()).await?;
let slots = read_usage_state_reset_slots(storeapi.clone()).await?; let slots = read_usage_state_reset_slots(storeapi.clone()).await?;
let usage_floor = usage_state_reset_floor(&slots)?; let usage_floor = match usage_state_reset_floor(&slots)? {
let leader_epoch = cycle_epoch ScannerUsageResetFloor::Trusted(floor) => floor,
.max(usage_floor.leader_epoch) ScannerUsageResetFloor::Corrupt if matches!(cycle_revision, DataUsageCacheRevision::Missing) => {
.checked_add(1) return Err(ScannerError::Other("scanner usage reset has no trusted cycle or usage floor".to_string()));
.filter(|epoch| *epoch < u64::MAX) }
.ok_or_else(|| ScannerError::Other("scanner leader epoch is exhausted".to_string()))?; ScannerUsageResetFloor::Missing | ScannerUsageResetFloor::Corrupt => PersistedUsageFloor {
next_cycle: cycle.next,
leader_epoch: cycle_epoch,
},
};
let resume_epoch = usage_state_reset_resume_epoch(&slots)?;
let leader_epoch = if let Some(epoch) = resume_epoch {
if epoch != cycle_epoch || usage_floor.leader_epoch > epoch || usage_floor.next_cycle > cycle.next {
return Err(ScannerError::Other(
"scanner usage reset bootstrap conflicts with the persisted cycle fence".to_string(),
));
}
epoch
} else {
cycle_epoch
.max(usage_floor.leader_epoch)
.checked_add(1)
.filter(|epoch| *epoch < u64::MAX)
.ok_or_else(|| ScannerError::Other("scanner leader epoch is exhausted".to_string()))?
};
let rebuilt_cycle = CurrentCycle { let rebuilt_cycle = CurrentCycle {
next: cycle.next.max(usage_floor.next_cycle), next: cycle.next.max(usage_floor.next_cycle),
..Default::default() ..Default::default()
@@ -1602,21 +1824,24 @@ pub async fn reset_scanner_usage_state_for_full_rebuild(
"scanner leader lock was lost before fencing usage reset cycle state".to_string(), "scanner leader lock was lost before fencing usage reset cycle state".to_string(),
)); ));
} }
save_config_with_publication_admission_for_epoch( if resume_epoch.is_none() {
storeapi.clone(), save_reset_config(
DATA_USAGE_BLOOM_NAME_PATH.as_str(), storeapi.clone(),
cycle_data, DATA_USAGE_BLOOM_NAME_PATH.as_str(),
cycle_revision.preconditions(), cycle_data,
reset_epoch, cycle_revision.preconditions(),
) reset_epoch,
.await &|| !guard.is_lock_lost() && !ctx.is_cancelled(),
.map_err(|err| { )
if scanner_publication_epoch_changed(&err) { .await
ScannerError::Other("scanner usage reset deferred by a movement epoch change".to_string()) .map_err(|err| {
} else { if scanner_publication_epoch_changed(&err) {
ScannerError::Other(format!("failed to fence scanner cycle state for usage reset: {err}")) ScannerError::Other("scanner usage reset deferred by a movement epoch change".to_string())
} } else {
})?; ScannerError::Other(format!("failed to fence scanner cycle state for usage reset: {err}"))
}
})?;
}
if guard.is_lock_lost() { if guard.is_lock_lost() {
return Err(ScannerError::Other( return Err(ScannerError::Other(
@@ -1624,7 +1849,10 @@ pub async fn reset_scanner_usage_state_for_full_rebuild(
)); ));
} }
let reset_paths = let reset_paths =
reset_scanner_usage_state_slots_for_full_rebuild(storeapi.clone(), &slots, reset_epoch, leader_epoch).await?; reset_scanner_usage_state_slots_for_full_rebuild(storeapi.clone(), &slots, reset_epoch, leader_epoch, || {
!guard.is_lock_lost() && !ctx.is_cancelled()
})
.await?;
if guard.is_lock_lost() { if guard.is_lock_lost() {
return Err(ScannerError::Other( return Err(ScannerError::Other(
"scanner leader lock was lost after publishing usage reset marker".to_string(), "scanner leader lock was lost after publishing usage reset marker".to_string(),
@@ -2135,6 +2363,7 @@ async fn recover_legacy_incomplete_usage_floor(
expected_publication_epoch, expected_publication_epoch,
Some(primary.epoch), Some(primary.epoch),
ScannerUsageBootstrapPublishContext::Recovery, ScannerUsageBootstrapPublishContext::Recovery,
|| true,
) )
.await?; .await?;
warn!( warn!(
+7 -1
View File
@@ -191,6 +191,7 @@ pub(super) async fn initialize_usage_baseline_bootstrap(
expected_epoch, expected_epoch,
None, None,
ScannerUsageBootstrapPublishContext::Initial, ScannerUsageBootstrapPublishContext::Initial,
|| true,
) )
.await .await
} }
@@ -201,9 +202,10 @@ pub(super) async fn fence_scanner_usage_epoch_with_expected_epoch(
claimed_epoch: u64, claimed_epoch: u64,
expected_publication_epoch: Option<u64>, expected_publication_epoch: Option<u64>,
allow_bootstrap_pending: bool, allow_bootstrap_pending: bool,
owns_fence: impl Fn() -> bool,
) -> Result<(), ScannerError> { ) -> Result<(), ScannerError> {
for retry in 0..=SCANNER_PERSIST_CAS_RETRIES { for retry in 0..=SCANNER_PERSIST_CAS_RETRIES {
if ctx.is_cancelled() { if ctx.is_cancelled() || !owns_fence() {
return Err(ScannerError::Other("scanner leadership was cancelled before usage fencing".to_string())); return Err(ScannerError::Other("scanner leadership was cancelled before usage fencing".to_string()));
} }
@@ -264,6 +266,9 @@ pub(super) async fn fence_scanner_usage_epoch_with_expected_epoch(
"scanner usage epoch fence changed while preparing its conditional write".to_string(), "scanner usage epoch fence changed while preparing its conditional write".to_string(),
)); ));
}; };
if ctx.is_cancelled() || !owns_fence() {
return Err(ScannerError::Other("scanner leadership was lost before usage fencing".to_string()));
}
save_config_with_preconditions(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data, revision.preconditions()) save_config_with_preconditions(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), data, revision.preconditions())
.await .await
}; };
@@ -319,6 +324,7 @@ pub(super) async fn complete_scanner_leadership_claim(
claimed_epoch, claimed_epoch,
expected_publication_epoch, expected_publication_epoch,
allow_bootstrap_pending, allow_bootstrap_pending,
|| true,
) )
.await .await
{ {
+415 -3
View File
@@ -634,6 +634,8 @@ struct MemoryConfigStore {
cancel_after_successful_puts: Mutex<HashMap<String, (usize, CancellationToken)>>, cancel_after_successful_puts: Mutex<HashMap<String, (usize, CancellationToken)>>,
replace_after_successful_puts: Mutex<HashMap<String, (usize, Vec<u8>)>>, replace_after_successful_puts: Mutex<HashMap<String, (usize, Vec<u8>)>>,
error_after_commit_deletes: Mutex<HashSet<String>>, error_after_commit_deletes: Mutex<HashSet<String>>,
cancel_after_deletes: Mutex<HashMap<String, CancellationToken>>,
pause_next_publication_admission: Mutex<Option<(Arc<tokio::sync::Notify>, Arc<tokio::sync::Notify>)>>,
put_counts: Mutex<HashMap<String, usize>>, put_counts: Mutex<HashMap<String, usize>>,
publication_admission_blocked: AtomicBool, publication_admission_blocked: AtomicBool,
block_publication_after_admissions: AtomicUsize, block_publication_after_admissions: AtomicUsize,
@@ -4081,6 +4083,9 @@ impl crate::ScannerConfigObjectDelete for MemoryConfigStore {
revisions.remove(&key); revisions.remove(&key);
drop(revisions); drop(revisions);
drop(objects); drop(objects);
if let Some(token) = self.cancel_after_deletes.lock().await.remove(&key) {
token.cancel();
}
if self.error_after_commit_deletes.lock().await.remove(&key) { if self.error_after_commit_deletes.lock().await.remove(&key) {
return Err(EcstoreError::other("injected delete error after commit")); return Err(EcstoreError::other("injected delete error after commit"));
} }
@@ -4088,6 +4093,11 @@ impl crate::ScannerConfigObjectDelete for MemoryConfigStore {
} }
async fn scanner_data_usage_publication_admission(&self) -> Option<crate::ScannerDataUsagePublicationAdmission> { async fn scanner_data_usage_publication_admission(&self) -> Option<crate::ScannerDataUsagePublicationAdmission> {
let pause = self.pause_next_publication_admission.lock().await.take();
if let Some((entered, resume)) = pause {
entered.notify_one();
resume.notified().await;
}
if self.publication_admission_blocked.load(Ordering::Acquire) { if self.publication_admission_blocked.load(Ordering::Acquire) {
return None; return None;
} }
@@ -4589,7 +4599,7 @@ async fn scanner_legacy_usage_backup_survives_fencing_and_restart_after_real_met
.expect("publication must also read the intact backup"); .expect("publication must also read the intact backup");
assert_eq!(baseline.data.as_deref(), Some(data.as_slice())); assert_eq!(baseline.data.as_deref(), Some(data.as_slice()));
assert_eq!(baseline.revision, DataUsageCacheRevision::Missing); assert_eq!(baseline.revision, DataUsageCacheRevision::Missing);
fence_scanner_usage_epoch_with_expected_epoch(&CancellationToken::new(), store.clone(), 7, None, false) fence_scanner_usage_epoch_with_expected_epoch(&CancellationToken::new(), store.clone(), 7, None, false, || true)
.await .await
.expect("legacy backup must be fenced into v2"); .expect("legacy backup must be fenced into v2");
let fenced = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) let fenced = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
@@ -4818,6 +4828,28 @@ async fn scanner_usage_state_reset_publishes_fenced_bootstrap_marker() {
); );
} }
let cycle_before_retry = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
.await
.expect("cycle should remain before retry");
let marker_before_retry = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("bootstrap should remain before retry");
let retry = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store.clone())
.await
.expect("completed cleanup should be reentrant");
assert_eq!(retry.leader_epoch, result.leader_epoch);
assert_eq!(
read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
.await
.expect("cycle should remain"),
cycle_before_retry
);
assert_eq!(
read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("bootstrap should remain"),
marker_before_retry
);
let (floor, state) = persisted_usage_floor_for_startup(store, false) let (floor, state) = persisted_usage_floor_for_startup(store, false)
.await .await
.expect("reset marker should be resumable"); .expect("reset marker should be resumable");
@@ -4973,7 +5005,7 @@ async fn scanner_usage_state_reset_slots_reject_primary_aba() {
store.objects.lock().await.insert(key.clone(), b"newer-json".to_vec()); store.objects.lock().await.insert(key.clone(), b"newer-json".to_vec());
store.revisions.lock().await.insert(key, 2); store.revisions.lock().await.insert(key, 2);
let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3) let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3, || true)
.await .await
.expect_err("stale primary revision must not be overwritten"); .expect_err("stale primary revision must not be overwritten");
assert!( assert!(
@@ -4983,6 +5015,348 @@ async fn scanner_usage_state_reset_slots_reject_primary_aba() {
); );
} }
#[tokio::test]
async fn scanner_usage_state_reset_resumes_every_cleanup_boundary_without_rewriting_intent() {
for completed in 0..=4 {
let store = Arc::new(MemoryConfigStore::default());
let primary_path = DATA_USAGE_OBJ_NAME_PATH.as_str();
let cleanup_paths = [
format!("{primary_path}.bkp"),
LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str().to_string(),
format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()),
DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str().to_string(),
];
for path in std::iter::once(primary_path).chain(cleanup_paths.iter().map(String::as_str)) {
let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
usage.scanner_epoch = Some(1);
save_config(store.clone(), path, serde_json::to_vec(&usage).expect("fixture should encode"))
.await
.expect("fixture should persist");
}
// These objects belong to other owners, even when reset cleanup resumes.
for path in ["buckets/quota-reservations/ledger", "buckets/example/incarnation"] {
save_config(store.clone(), path, b"retain".to_vec())
.await
.expect("unrelated state should persist");
}
let slots = read_usage_state_reset_slots(store.clone()).await.expect("slots should load");
let cancelled = CancellationToken::new();
if completed == 0 {
store
.cancel_after_successful_puts
.lock()
.await
.insert(memory_config_key(RUSTFS_META_BUCKET, primary_path), (2, cancelled.clone()));
} else {
store
.cancel_after_deletes
.lock()
.await
.insert(memory_config_key(RUSTFS_META_BUCKET, &cleanup_paths[completed - 1]), cancelled.clone());
}
let err = reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || !cancelled.is_cancelled())
.await
.expect_err("interruption should stop cleanup");
assert!(err.to_string().contains("ownership"), "boundary {completed}: {err}");
for (index, path) in cleanup_paths.iter().enumerate() {
assert_eq!(
store
.objects
.lock()
.await
.contains_key(&memory_config_key(RUSTFS_META_BUCKET, path)),
index >= completed,
"boundary {completed}, slot {index}"
);
}
let intent = read_config_with_revision(store.clone(), primary_path)
.await
.expect("intent should persist");
let slots = read_usage_state_reset_slots(store.clone())
.await
.expect("restart should reload slots");
reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || true)
.await
.expect("restart should complete the same intent");
assert_eq!(
read_config_with_revision(store.clone(), primary_path)
.await
.expect("intent should remain"),
intent
);
assert_eq!(store.put_counts.lock().await[&memory_config_key(RUSTFS_META_BUCKET, primary_path)], 2);
for path in cleanup_paths {
assert!(
!store
.objects
.lock()
.await
.contains_key(&memory_config_key(RUSTFS_META_BUCKET, &path))
);
}
for path in ["buckets/quota-reservations/ledger", "buckets/example/incarnation"] {
assert_eq!(read_config(store.clone(), path).await.expect("unrelated state should remain"), b"retain");
}
}
}
#[tokio::test]
async fn scanner_usage_state_reset_stops_usage_fence_after_owner_loss() {
let store = Arc::new(MemoryConfigStore::default());
let mut usage = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
usage.scanner_epoch = Some(1);
let bytes = serde_json::to_vec(&usage).expect("baseline should encode");
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), bytes.clone())
.await
.expect("baseline should persist");
let checks = AtomicUsize::new(0);
let err = fence_scanner_usage_epoch_with_expected_epoch(&CancellationToken::new(), store.clone(), 3, Some(0), false, || {
checks.fetch_add(1, Ordering::SeqCst) == 0
})
.await
.expect_err("ownership lost during reads must prevent the write");
assert!(err.to_string().contains("leadership was lost"), "{err}");
assert_eq!(
read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("baseline should remain"),
bytes
);
}
#[tokio::test]
async fn scanner_usage_state_reset_cancels_during_publication_admission() {
for resuming in [false, true] {
let store = Arc::new(MemoryConfigStore::default());
let usage = if resuming {
scanner_usage_bootstrap_marker(std::time::SystemTime::UNIX_EPOCH, Some(3))
} else {
complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0)
};
save_config(
store.clone(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
serde_json::to_vec(&usage).expect("primary should encode"),
)
.await
.expect("primary should persist");
save_config(store.clone(), LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str(), b"corrupt".to_vec())
.await
.expect("cleanup target should persist");
let slots = read_usage_state_reset_slots(store.clone()).await.expect("slots should load");
let before = store.objects.lock().await.clone();
let revisions_before = store.revisions.lock().await.clone();
let entered = Arc::new(tokio::sync::Notify::new());
let resume = Arc::new(tokio::sync::Notify::new());
*store.pause_next_publication_admission.lock().await = Some((entered.clone(), resume.clone()));
let cancelled = CancellationToken::new();
let (result, ()) = tokio::join!(
reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || !cancelled.is_cancelled()),
async {
entered.notified().await;
cancelled.cancel();
resume.notify_one();
}
);
let err = result.expect_err("losing ownership during admission must prevent mutation");
assert!(err.to_string().contains("ownership was lost"), "resuming={resuming}: {err}");
assert_eq!(*store.objects.lock().await, before);
assert_eq!(*store.revisions.lock().await, revisions_before);
}
}
#[tokio::test]
#[serial]
async fn scanner_usage_state_reset_rejects_corruption_without_a_trusted_floor() {
let (_temp_dir, store) = setup_scanner_cycle_store_with_usage_baseline(false).await;
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), b"{corrupt".to_vec())
.await
.expect("corrupt primary should persist");
let before = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("evidence should load");
let err = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store.clone())
.await
.expect_err("corruption must not become a zero floor");
assert!(err.to_string().contains("no trusted cycle or usage floor"), "{err}");
assert_eq!(
read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("evidence should remain"),
before
);
assert!(matches!(
read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()).await,
Err(EcstoreError::ConfigNotFound)
));
let mut backup = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
backup.scanner_epoch = Some(7);
backup.scanner_cycle = Some(40);
save_config(
store.clone(),
&format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()),
serde_json::to_vec(&backup).expect("backup should encode"),
)
.await
.expect("valid backup should persist");
let result = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store)
.await
.expect("valid backup should supply the recovery floor");
assert_eq!(result.leader_epoch, 8);
assert_eq!(result.next_cycle, 41);
}
#[tokio::test]
async fn scanner_usage_state_reset_rejects_replaced_intent_and_newer_cleanup_slot() {
let store = Arc::new(MemoryConfigStore::default());
let marker = scanner_usage_bootstrap_marker(std::time::SystemTime::UNIX_EPOCH, Some(3));
let bytes = serde_json::to_vec(&marker).expect("marker should encode");
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), bytes.clone())
.await
.expect("intent should persist");
let slots = read_usage_state_reset_slots(store.clone()).await.expect("slots should load");
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), bytes)
.await
.expect("another intent should persist");
let err = reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || true)
.await
.expect_err("same epoch cannot replace an intent revision");
assert!(err.to_string().contains("intent revision changed"), "{err}");
let mut newer = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0);
newer.scanner_epoch = Some(3);
let path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str());
let bytes = serde_json::to_vec(&newer).expect("newer snapshot should encode");
save_config(store.clone(), &path, bytes.clone())
.await
.expect("newer snapshot should persist");
let slots = read_usage_state_reset_slots(store.clone())
.await
.expect("slots should reload");
let err = reset_scanner_usage_state_slots_for_full_rebuild(store.clone(), &slots, 0, 3, || true)
.await
.expect_err("cleanup cannot delete same-epoch progress");
assert!(err.to_string().contains("not older than its intent"), "{err}");
assert_eq!(read_config(store, &path).await.expect("newer snapshot should remain"), bytes);
}
#[tokio::test]
#[serial]
async fn scanner_usage_state_reset_rejects_decodable_untrusted_floor() {
let (_temp_dir, store) = setup_scanner_cycle_store_with_usage_baseline(false).await;
let invalid_identity = DataUsageInfo {
usage_snapshot_complete: true,
buckets_count: 1,
last_update: Some(std::time::SystemTime::UNIX_EPOCH),
..Default::default()
};
for usage in [DataUsageInfo::default(), invalid_identity] {
save_config(
store.clone(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
serde_json::to_vec(&usage).expect("fixture should encode"),
)
.await
.expect("untrusted primary should persist");
let before = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("primary should load");
let err = reset_scanner_usage_state_for_full_rebuild(CancellationToken::new(), store.clone())
.await
.expect_err("valid JSON alone cannot prove a usage floor");
assert!(err.to_string().contains("no trusted cycle or usage floor"), "{err}");
assert_eq!(
read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("evidence should remain"),
before
);
assert!(matches!(
read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()).await,
Err(EcstoreError::ConfigNotFound)
));
}
}
#[test]
fn full_rescan_reset_rejects_unknown_marker_phase_even_with_invalid_compat_fields() {
for state in [serde_json::json!("rewrite-v2"), serde_json::json!(7), serde_json::Value::Null] {
let marker = serde_json::json!({"state": state, "retry_count": "future-type", "schema_version": 99});
let err = super::cycle_state::decode_recovery_marker_for_reset(
&serde_json::to_vec(&marker).expect("future marker should encode"),
&DataUsageCacheRevision::Etag("intent-1".to_string()),
)
.expect_err("unknown persistent phases must remain fenced");
assert!(err.to_string().contains("state is unsupported"), "{err}");
}
}
#[tokio::test]
#[serial]
async fn full_rescan_reset_preserves_unknown_phase_and_retries_completed_cleanup() {
let (_temp_dir, store) = setup_scanner_cycle_store().await;
save_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str(), b"corrupt".to_vec())
.await
.expect("corrupt primary should persist");
save_config(
store.clone(),
DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(),
br#"{"state":"future-rewrite"}"#.to_vec(),
)
.await
.expect("future marker should persist");
let primary_before = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
.await
.expect("primary should load");
let marker_before = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
.await
.expect("marker should load");
let err = reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
.await
.expect_err("unknown phase must block explicit reset");
assert!(err.to_string().contains("state is unsupported"), "{err}");
assert_eq!(
read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
.await
.expect("primary should remain"),
primary_before
);
assert_eq!(
read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str())
.await
.expect("marker should remain"),
marker_before
);
save_config(store.clone(), DATA_USAGE_BLOOM_RECOVERY_PATH.as_str(), b"{malformed".to_vec())
.await
.expect("recoverable marker should persist");
reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
.await
.expect("reset should complete");
let primary = read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
.await
.expect("rebuilt primary should load");
let usage = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("fenced usage should load");
reset_scanner_cycle_recovery(CancellationToken::new(), store.clone())
.await
.expect("retry after marker deletion should complete");
assert_eq!(
read_config_with_revision(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
.await
.expect("rebuilt primary should remain"),
primary
);
assert_eq!(
read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("fenced usage should remain"),
usage
);
}
#[tokio::test] #[tokio::test]
async fn scanner_usage_state_reset_slots_defer_when_publication_epoch_moves() { async fn scanner_usage_state_reset_slots_defer_when_publication_epoch_moves() {
let store = Arc::new(MemoryConfigStore::default()); let store = Arc::new(MemoryConfigStore::default());
@@ -4993,7 +5367,7 @@ async fn scanner_usage_state_reset_slots_defer_when_publication_epoch_moves() {
.expect("usage reset slots should be inspected"); .expect("usage reset slots should be inspected");
store.publication_admission_blocked.store(true, Ordering::Release); store.publication_admission_blocked.store(true, Ordering::Release);
let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3) let err = reset_scanner_usage_state_slots_for_full_rebuild(store, &slots, 0, 3, || true)
.await .await
.expect_err("movement admission loss must defer reset"); .expect_err("movement admission loss must defer reset");
assert!( assert!(
@@ -8169,6 +8543,44 @@ fn scanner_activity_snapshot_digest_fences_dirty_usage_state() {
assert_ne!(scanner_activity_snapshot_digest(&clean), scanner_activity_snapshot_digest(&pending)); assert_ne!(scanner_activity_snapshot_digest(&clean), scanner_activity_snapshot_digest(&pending));
} }
#[test]
fn scanner_activity_structural_digest_ignores_regular_bucket_writes() {
let baseline = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]);
let mut written = baseline.clone();
let activity = written.get_mut("node-2").expect("node should exist");
activity.namespace_generation = 8;
activity.dirty_usage_generation = 6;
activity.dirty_usage_pending = true;
assert_ne!(scanner_activity_snapshot_digest(&baseline), scanner_activity_snapshot_digest(&written));
assert_eq!(
scanner_activity_structural_digest(&baseline),
scanner_activity_structural_digest(&written),
"bucket writes are refreshed through the dirty-bucket scope rather than invalidating every cache"
);
}
#[test]
fn scanner_activity_structural_digest_fences_restart_and_maintenance() {
let baseline = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]);
let mut restarted = baseline.clone();
restarted.get_mut("node-2").expect("node should exist").instance_id = "epoch-b".to_string();
let mut maintained = baseline.clone();
maintained
.get_mut("node-2")
.expect("node should exist")
.maintenance_generation = 4;
assert_ne!(
scanner_activity_structural_digest(&baseline),
scanner_activity_structural_digest(&restarted)
);
assert_ne!(
scanner_activity_structural_digest(&baseline),
scanner_activity_structural_digest(&maintained)
);
}
#[test] #[test]
fn scanner_dirty_usage_acknowledgements_exclude_local_and_clean_nodes() { fn scanner_dirty_usage_acknowledgements_exclude_local_and_clean_nodes() {
let snapshot = BTreeMap::from([ let snapshot = BTreeMap::from([
+121 -3
View File
@@ -21,6 +21,7 @@ use crate::{
DataUsageCacheSource, DataUsageEntry, DataUsageEntryInfo, DataUsageInfo, DataUsageScanPlanDigest, DataUsageSnapshotSetState, DataUsageCacheSource, DataUsageEntry, DataUsageEntryInfo, DataUsageInfo, DataUsageScanPlanDigest, DataUsageSnapshotSetState,
ScannerError, SizeSummary, TierStats, ScannerError, SizeSummary, TierStats,
}; };
use bytes::Bytes;
use futures::future::join_all; use futures::future::join_all;
use metrics::counter; use metrics::counter;
use rand::seq::SliceRandom as _; use rand::seq::SliceRandom as _;
@@ -54,6 +55,7 @@ use tokio_util::task::AbortOnDropHandle;
use tracing::{debug, error, warn}; use tracing::{debug, error, warn};
use crate::ScannerObjectInfo as ObjectInfo; use crate::ScannerObjectInfo as ObjectInfo;
use crate::storage_api::EcstoreScannerPeerDirtyUsageSnapshot;
use crate::storage_api::ScannerStorage; use crate::storage_api::ScannerStorage;
use crate::storage_api::scan::NamespaceLocking as _; use crate::storage_api::scan::NamespaceLocking as _;
use crate::storage_api::scanner_io::{BucketInfo, BucketOptions}; use crate::storage_api::scanner_io::{BucketInfo, BucketOptions};
@@ -111,6 +113,121 @@ pub(crate) struct ScannerBucketScanScope {
baseline_scan_plan_digest: Option<DataUsageScanPlanDigest>, baseline_scan_plan_digest: Option<DataUsageScanPlanDigest>,
} }
impl ScannerBucketScanScope {
fn is_default(&self) -> bool {
self.selected_buckets.is_none() && self.baseline_scan_plan_digest.is_none()
}
fn from_dirty_buckets(selected_buckets: HashSet<String>, baseline_scan_plan_digest: DataUsageScanPlanDigest) -> Self {
Self {
selected_buckets: Some(Arc::new(selected_buckets)),
baseline_scan_plan_digest: Some(baseline_scan_plan_digest),
}
}
}
#[derive(Clone, Copy)]
pub(super) struct ScannerCacheBaselineProof<'a> {
pub(super) data: Option<&'a Bytes>,
pub(super) expected_sources: &'a HashSet<DataUsageCacheSource>,
pub(super) leader_epoch: u64,
pub(super) want_cycle: u64,
pub(super) scan_plan_digest: DataUsageScanPlanDigest,
}
#[derive(Clone, Debug, PartialEq, Eq)]
struct ScannerPeerDirtyUsageExpectation {
instance_id: String,
generation: u64,
pending: bool,
}
fn verified_remote_dirty_usage_buckets(
expected_peers: &HashMap<String, ScannerPeerDirtyUsageExpectation>,
peer_snapshots: Vec<(String, EcstoreScannerPeerDirtyUsageSnapshot)>,
) -> Option<HashSet<String>> {
if expected_peers.is_empty() || peer_snapshots.len() != expected_peers.len() {
return None;
}
let mut received_peers = HashSet::with_capacity(peer_snapshots.len());
let mut dirty_buckets = HashSet::new();
for (host, snapshot) in peer_snapshots {
let expected = expected_peers.get(&host)?;
if !received_peers.insert(host)
|| snapshot.instance_id != expected.instance_id
|| snapshot.generation != expected.generation
|| snapshot.generation == u64::MAX
|| snapshot.protocol_version != crate::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION
|| !snapshot.complete
|| snapshot.pending_bucket_count != u64::try_from(snapshot.buckets.len()).unwrap_or(u64::MAX)
|| (expected.pending && snapshot.pending_bucket_count == 0)
{
return None;
}
dirty_buckets.extend(snapshot.buckets.into_keys());
}
(received_peers.len() == expected_peers.len()).then_some(dirty_buckets)
}
fn complete_scanner_cache_baseline_plan_digest(proof: ScannerCacheBaselineProof<'_>) -> Option<DataUsageScanPlanDigest> {
let data = proof.data?;
let baseline = serde_json::from_slice::<DataUsageInfo>(data).ok()?;
if !baseline.is_complete_bucket_usage_snapshot()
|| baseline.usage_snapshot_partial
|| baseline.usage_snapshot_converged != Some(true)
|| baseline.scanner_epoch != Some(proof.leader_epoch)
|| baseline.usage_snapshot_set_states.len() != proof.expected_sources.len()
{
return None;
}
let mut states = HashSet::with_capacity(baseline.usage_snapshot_set_states.len());
for state in &baseline.usage_snapshot_set_states {
let source = DataUsageCacheSource::new(usize::try_from(state.pool_index).ok()?, usize::try_from(state.set_index).ok()?);
if !proof.expected_sources.contains(&source)
|| !states.insert(source)
|| !state.complete
|| state.tombstone
|| state.scanner_epoch != Some(proof.leader_epoch)
|| state.scanner_cycle.is_none_or(|cycle| cycle > proof.want_cycle)
|| state.scan_plan_digest != Some(proof.scan_plan_digest.0)
{
return None;
}
}
(states == *proof.expected_sources).then_some(proof.scan_plan_digest)
}
fn scoped_scan_scope_from_dirty_buckets(
requested_scope: ScannerBucketScanScope,
dirty_buckets: HashSet<String>,
dirty_snapshot_complete: bool,
all_buckets: &[BucketInfo],
baseline_proof: ScannerCacheBaselineProof<'_>,
) -> ScannerBucketScanScope {
if !requested_scope.is_default() || !dirty_snapshot_complete {
return requested_scope;
}
let current_buckets = all_buckets.iter().map(|bucket| bucket.name.as_str()).collect::<HashSet<_>>();
let selected_buckets = dirty_buckets
.into_iter()
.filter(|bucket| current_buckets.contains(bucket.as_str()))
.collect::<HashSet<_>>();
if selected_buckets.is_empty() {
return requested_scope;
}
let Some(baseline_scan_plan_digest) = complete_scanner_cache_baseline_plan_digest(baseline_proof) else {
return requested_scope;
};
ScannerBucketScanScope::from_dirty_buckets(selected_buckets, baseline_scan_plan_digest)
}
pub(crate) fn is_scanner_metadata_corrupt_error(err: &StorageError) -> bool { pub(crate) fn is_scanner_metadata_corrupt_error(err: &StorageError) -> bool {
matches!(err, StorageError::Io(io) if io.to_string().starts_with(SCANNER_METADATA_CORRUPT_ERROR)) matches!(err, StorageError::Io(io) if io.to_string().starts_with(SCANNER_METADATA_CORRUPT_ERROR))
} }
@@ -749,7 +866,7 @@ mod io_cache;
mod io_cycle; mod io_cycle;
#[cfg(test)] #[cfg(test)]
use io_cache::{ScannerSetCacheGeneration, prepare_scoped_set_scan}; use io_cache::{ScannerSetCacheGeneration, prepare_scoped_set_scan};
pub(crate) use io_cycle::nsscanner_with_storage_status; pub(crate) use io_cycle::{ScannerCycleRequest, nsscanner_with_storage_status_scoped};
mod io_disk; mod io_disk;
#[cfg(test)] #[cfg(test)]
mod publish_gate_tests; mod publish_gate_tests;
@@ -766,8 +883,9 @@ pub(crate) use cache::{
}; };
pub use dirty_usage::{ pub use dirty_usage::{
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState, ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshot, ScannerDirtyUsageState,
acknowledge_dirty_usage_generation, clear_dirty_usage_bucket, record_dirty_usage_bucket, record_scanner_maintenance_change, acknowledge_dirty_usage_generation, acknowledge_scoped_dirty_usage, clear_dirty_usage_bucket, record_dirty_usage_bucket,
scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state, scanner_maintenance_generation, record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
scanner_maintenance_generation,
}; };
#[cfg(test)] #[cfg(test)]
pub(crate) use dirty_usage::{clear_dirty_usage_buckets_for_tests, dirty_usage_buckets_for_tests}; pub(crate) use dirty_usage::{clear_dirty_usage_buckets_for_tests, dirty_usage_buckets_for_tests};
+18
View File
@@ -282,9 +282,26 @@ pub(super) fn completed_data_usage_info(
.iter() .iter()
.map(|(bucket, usage)| (bucket.clone(), usage.size)) .map(|(bucket, usage)| (bucket.clone(), usage.size))
.collect(); .collect();
let mut usage_snapshot_set_states = results
.iter()
.map(|result| {
let source = result.info.source?;
Some(DataUsageSnapshotSetState {
pool_index: u64::try_from(source.pool_index).ok()?,
set_index: u64::try_from(source.set_index).ok()?,
scanner_cycle: Some(result.info.next_cycle),
scanner_epoch: Some(result.info.leader_epoch),
scan_plan_digest: Some(result.info.scan_plan_digest?.0),
complete: true,
tombstone: false,
})
})
.collect::<Option<Vec<_>>>()?;
usage_snapshot_set_states.sort_by_key(|state| (state.pool_index, state.set_index));
let data_usage_info = DataUsageInfo { let data_usage_info = DataUsageInfo {
last_update: Some(merged_last_update), last_update: Some(merged_last_update),
scanner_cycle: Some(results.first()?.info.next_cycle), scanner_cycle: Some(results.first()?.info.next_cycle),
scanner_epoch: Some(results.first()?.info.leader_epoch),
objects_total_count: u64::try_from(total.objects).ok()?, objects_total_count: u64::try_from(total.objects).ok()?,
versions_total_count: u64::try_from(total.versions).ok()?, versions_total_count: u64::try_from(total.versions).ok()?,
delete_markers_total_count: u64::try_from(total.delete_markers).ok()?, delete_markers_total_count: u64::try_from(total.delete_markers).ok()?,
@@ -295,6 +312,7 @@ pub(super) fn completed_data_usage_info(
bucket_sizes, bucket_sizes,
buckets_usage, buckets_usage,
usage_snapshot_complete: true, usage_snapshot_complete: true,
usage_snapshot_set_states,
..Default::default() ..Default::default()
}; };
Some((data_usage_info, merged_last_update)) Some((data_usage_info, merged_last_update))
@@ -52,6 +52,112 @@ pub enum ScannerDirtyUsageAckError {
ProcessChanged, ProcessChanged,
#[error("scanner dirty usage generation cannot be acknowledged")] #[error("scanner dirty usage generation cannot be acknowledged")]
InvalidGeneration, InvalidGeneration,
#[error("scanner dirty usage bucket incarnation fence is unavailable")]
IncarnationUnavailable,
}
/// A scoped ACK requires storage-owned lifecycle and incarnation fences.
/// Callers must only send ACKs backed by durable per-bucket publication.
pub fn acknowledge_scoped_dirty_usage(
instance_id: &str,
entries: &[(&crate::storage_api::EcstoreBucketMetadataMutationGuard, u64)],
probe_only: bool,
) -> std::result::Result<u64, ScannerDirtyUsageAckError> {
// Lock order: sorted bucket lifecycle/metadata fences (caller), then dirty map.
// No await or storage operation occurs while the dirty map is locked.
let (cleared, pending) = {
let mut dirty = dirty_usage_buckets();
let checked = entries
.iter()
.map(|(guard, generation)| {
guard
.checked_bucket_incarnation()
.map(|(bucket, _)| (bucket, *generation))
.map_err(|_| ScannerDirtyUsageAckError::IncarnationUnavailable)
})
.collect::<std::result::Result<Vec<_>, _>>()?;
let cleared = apply_scoped_dirty_usage_ack(
instance_id,
scanner_activity_epoch(),
DIRTY_USAGE_BUCKET_GENERATION.load(Ordering::Acquire),
&mut dirty,
&checked,
probe_only,
)?;
if cleared > 0 {
advance_generation(&DIRTY_USAGE_BUCKET_GENERATION);
}
(cleared, dirty.len())
};
if !probe_only {
global_metrics().record_scanner_dirty_usage_cycle_clear(usize_to_u64_saturated(cleared), usize_to_u64_saturated(pending));
}
Ok(usize_to_u64_saturated(cleared))
}
fn apply_scoped_dirty_usage_ack(
instance_id: &str,
current_instance: &str,
current_generation: u64,
dirty: &mut DirtyUsageBuckets,
entries: &[(&str, u64)],
probe_only: bool,
) -> std::result::Result<usize, ScannerDirtyUsageAckError> {
if instance_id != current_instance {
return Err(ScannerDirtyUsageAckError::ProcessChanged);
}
if current_generation == u64::MAX
|| entries
.iter()
.any(|(_, generation)| *generation == 0 || *generation == u64::MAX || *generation > current_generation)
{
return Err(ScannerDirtyUsageAckError::InvalidGeneration);
}
let mut cleared = 0;
if !probe_only {
for (bucket, generation) in entries {
if dirty.get(*bucket) == Some(generation) {
dirty.remove(*bucket);
cleared += 1;
}
}
}
Ok(cleared)
}
#[cfg(test)]
mod scoped_dirty_usage_tests {
use super::*;
#[test]
fn scoped_dirty_usage_preserves_uncovered_newer_and_replayed_generations() {
let mut dirty = HashMap::from([("hot".to_string(), 7), ("cold".to_string(), 8)]);
assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8)], true), Ok(0));
assert_eq!(dirty.len(), 2);
assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8)], false), Ok(1));
assert_eq!(dirty.get("hot"), Some(&7));
assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8)], false), Ok(0));
dirty.insert("cold".to_string(), 9);
assert_eq!(apply_scoped_dirty_usage_ack("p", "p", 9, &mut dirty, &[("cold", 8)], false), Ok(0));
assert_eq!(dirty.get("cold"), Some(&9));
}
#[test]
fn scoped_dirty_usage_rejects_restart_and_invalid_batch_before_clearing() {
let original = HashMap::from([("hot".to_string(), 7), ("cold".to_string(), 8)]);
let mut dirty = original.clone();
assert_eq!(
apply_scoped_dirty_usage_ack("old", "new", 8, &mut dirty, &[("cold", 8)], false),
Err(ScannerDirtyUsageAckError::ProcessChanged)
);
for generation in [0, 9, u64::MAX] {
assert_eq!(
apply_scoped_dirty_usage_ack("p", "p", 8, &mut dirty, &[("cold", 8), ("hot", generation)], false),
Err(ScannerDirtyUsageAckError::InvalidGeneration)
);
assert_eq!(dirty, original);
}
}
} }
pub(super) fn dirty_usage_buckets() -> MutexGuard<'static, DirtyUsageBuckets> { pub(super) fn dirty_usage_buckets() -> MutexGuard<'static, DirtyUsageBuckets> {
+94 -1
View File
@@ -71,6 +71,7 @@ where
leader_epoch, leader_epoch,
scan_mode, scan_mode,
scan_scope: ScannerBucketScanScope::default(), scan_scope: ScannerBucketScanScope::default(),
persisted_usage_baseline: None,
}; };
nsscanner_with_storage_status_scoped(store, request).await nsscanner_with_storage_status_scoped(store, request).await
} }
@@ -83,6 +84,79 @@ pub(crate) struct ScannerCycleRequest {
pub(crate) leader_epoch: u64, pub(crate) leader_epoch: u64,
pub(crate) scan_mode: HealScanMode, pub(crate) scan_mode: HealScanMode,
pub(crate) scan_scope: ScannerBucketScanScope, pub(crate) scan_scope: ScannerBucketScanScope,
pub(crate) persisted_usage_baseline: Option<Bytes>,
}
struct ScannerBucketScopeResolution<'a> {
requested_scope: ScannerBucketScanScope,
baseline_proof: ScannerCacheBaselineProof<'a>,
activity_before: &'a crate::scanner::ScannerActivitySnapshot,
dirty_usage_snapshot: &'a DirtyUsageSnapshot,
all_buckets: &'a [BucketInfo],
}
async fn resolve_scanner_bucket_scan_scope<S>(
store: &S,
distributed: bool,
resolution: ScannerBucketScopeResolution<'_>,
) -> ScannerBucketScanScope
where
S: ScannerStorage,
{
if !resolution.requested_scope.is_default()
|| !resolution.dirty_usage_snapshot.covers_all_pending
|| resolution.dirty_usage_snapshot.generation == u64::MAX
|| resolution.dirty_usage_snapshot.buckets.len() > crate::SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES
{
return resolution.requested_scope;
}
let mut dirty_buckets = resolution
.dirty_usage_snapshot
.buckets
.keys()
.cloned()
.collect::<HashSet<_>>();
if distributed {
let Some(notification_system) = store.scanner_notification_system() else {
return resolution.requested_scope;
};
let Ok(peer_snapshots) = notification_system.scanner_dirty_usage_snapshots().await else {
return resolution.requested_scope;
};
let mut expected_peers = HashMap::new();
for (host, lease_instance_id, _) in crate::scanner::scanner_activity_publication_lease_targets(resolution.activity_before)
{
let Some((activity_instance_id, generation, pending)) =
crate::scanner::scanner_activity_dirty_usage_state_for_host(resolution.activity_before, &host)
else {
return resolution.requested_scope;
};
if activity_instance_id != lease_instance_id || expected_peers.contains_key(&host) {
return resolution.requested_scope;
}
expected_peers.insert(
host,
ScannerPeerDirtyUsageExpectation {
instance_id: activity_instance_id.to_string(),
generation,
pending,
},
);
}
let Some(remote_dirty_buckets) = verified_remote_dirty_usage_buckets(&expected_peers, peer_snapshots) else {
return resolution.requested_scope;
};
dirty_buckets.extend(remote_dirty_buckets);
}
scoped_scan_scope_from_dirty_buckets(
resolution.requested_scope,
dirty_buckets,
true,
resolution.all_buckets,
resolution.baseline_proof,
)
} }
pub(crate) async fn nsscanner_with_storage_status_scoped<S>(store: &S, request: ScannerCycleRequest) -> Result<ScannerCycleResult> pub(crate) async fn nsscanner_with_storage_status_scoped<S>(store: &S, request: ScannerCycleRequest) -> Result<ScannerCycleResult>
@@ -97,6 +171,7 @@ where
leader_epoch, leader_epoch,
scan_mode, scan_mode,
scan_scope, scan_scope,
persisted_usage_baseline,
} = request; } = request;
let child_token = ctx.child_token(); let child_token = ctx.child_token();
let _tier_cycle_guard = begin_tier_registry_cycle(want_cycle, leader_epoch); let _tier_cycle_guard = begin_tier_registry_cycle(want_cycle, leader_epoch);
@@ -186,8 +261,26 @@ where
} }
bucket_plan_complete &= buckets_by_source.keys().copied().collect::<HashSet<_>>() == *expected_sources; bucket_plan_complete &= buckets_by_source.keys().copied().collect::<HashSet<_>>() == *expected_sources;
let scan_plan_digest = let scan_plan_digest =
scanner_bucket_plan_digest(&all_buckets, crate::scanner::scanner_activity_snapshot_digest(&activity_before)); scanner_bucket_plan_digest(&all_buckets, crate::scanner::scanner_activity_structural_digest(&activity_before));
let dirty_usage_snapshot = Arc::new(snapshot_dirty_usage_buckets(&all_buckets, dirty_generation_before_bucket_list)); let dirty_usage_snapshot = Arc::new(snapshot_dirty_usage_buckets(&all_buckets, dirty_generation_before_bucket_list));
let scan_scope = resolve_scanner_bucket_scan_scope(
store,
distributed,
ScannerBucketScopeResolution {
requested_scope: scan_scope,
baseline_proof: ScannerCacheBaselineProof {
data: persisted_usage_baseline.as_ref(),
expected_sources: &expected_sources,
leader_epoch,
want_cycle,
scan_plan_digest,
},
activity_before: &activity_before,
dirty_usage_snapshot: &dirty_usage_snapshot,
all_buckets: &all_buckets,
},
)
.await;
let cache_cycle_floor = Arc::new(AtomicU64::new(want_cycle)); let cache_cycle_floor = Arc::new(AtomicU64::new(want_cycle));
let tier_registry = runtime_tier_registry_for_cycle(want_cycle, leader_epoch).await; let tier_registry = runtime_tier_registry_for_cycle(want_cycle, leader_epoch).await;
let tier_registry_generation = tier_registry.generation; let tier_registry_generation = tier_registry.generation;
@@ -655,9 +655,31 @@ fn completed_data_usage_info_requires_every_set_before_publish() {
.expect("all completed sets should produce a publishable data usage snapshot"); .expect("all completed sets should produce a publishable data usage snapshot");
assert_eq!(last_update, SystemTime::UNIX_EPOCH + Duration::from_secs(20)); assert_eq!(last_update, SystemTime::UNIX_EPOCH + Duration::from_secs(20));
assert_eq!(data_usage_info.scanner_cycle, Some(0)); assert_eq!(data_usage_info.scanner_cycle, Some(0));
assert_eq!(data_usage_info.scanner_epoch, Some(0));
assert_eq!(data_usage_info.objects_total_count, 3); assert_eq!(data_usage_info.objects_total_count, 3);
assert_eq!(data_usage_info.buckets_usage.len(), 3); assert_eq!(data_usage_info.buckets_usage.len(), 3);
assert!(data_usage_info.usage_snapshot_complete); assert!(data_usage_info.usage_snapshot_complete);
assert_eq!(
data_usage_info
.usage_snapshot_set_states
.iter()
.map(|state| {
(
state.pool_index,
state.set_index,
state.scanner_cycle,
state.scanner_epoch,
state.scan_plan_digest,
state.complete,
state.tombstone,
)
})
.collect::<Vec<_>>(),
vec![
(0, 0, Some(0), Some(0), Some(TEST_PLAN_DIGEST.0), true, false),
(1, 0, Some(0), Some(0), Some(TEST_PLAN_DIGEST.0), true, false),
]
);
assert_eq!( assert_eq!(
data_usage_info data_usage_info
.buckets_usage .buckets_usage
+190
View File
@@ -17,6 +17,7 @@ use super::io_disk::tier_stats_template;
use super::*; use super::*;
use crate::scanner_budget::ScannerCycleBudgetConfig; use crate::scanner_budget::ScannerCycleBudgetConfig;
use crate::scanner_folder::ScannerItem; use crate::scanner_folder::ScannerItem;
use crate::storage_api::EcstoreScannerPeerDirtyUsageSnapshot;
use crate::storage_api::owner::{ use crate::storage_api::owner::{
EcstorePoolDecommissionInfo, EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats, EcstorePoolDecommissionInfo, EcstoreRebalStatus, EcstoreRebalanceInfo, EcstoreRebalanceMeta, EcstoreRebalanceStats,
}; };
@@ -796,6 +797,195 @@ fn complete_set_usage_cache(buckets: &[(&str, usize)], scan_plan_digest: DataUsa
cache cache
} }
fn complete_usage_baseline(
source: DataUsageCacheSource,
scan_plan_digest: DataUsageScanPlanDigest,
scanner_cycle: u64,
scanner_epoch: u64,
) -> bytes::Bytes {
let baseline = DataUsageInfo {
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(10)),
scanner_cycle: Some(scanner_cycle),
scanner_epoch: Some(scanner_epoch),
buckets_count: 1,
buckets_usage: HashMap::from([("photos".to_string(), Default::default())]),
usage_snapshot_complete: true,
usage_snapshot_converged: Some(true),
usage_snapshot_set_states: vec![DataUsageSnapshotSetState {
pool_index: u64::try_from(source.pool_index).expect("test pool index should fit"),
set_index: u64::try_from(source.set_index).expect("test set index should fit"),
scanner_cycle: Some(scanner_cycle),
scanner_epoch: Some(scanner_epoch),
scan_plan_digest: Some(scan_plan_digest.0),
complete: true,
tombstone: false,
}],
..Default::default()
};
bytes::Bytes::from(serde_json::to_vec(&baseline).expect("test baseline should encode"))
}
#[test]
fn scoped_scan_requires_a_converged_complete_baseline_with_exact_set_provenance() {
let source = DataUsageCacheSource::new(1, 2);
let expected_sources = HashSet::from([source]);
let scan_plan_digest = DataUsageScanPlanDigest([9; 32]);
let baseline = complete_usage_baseline(source, scan_plan_digest, 7, 11);
assert_eq!(
complete_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof {
data: Some(&baseline),
expected_sources: &expected_sources,
leader_epoch: 11,
want_cycle: 8,
scan_plan_digest,
}),
Some(scan_plan_digest)
);
let mut incomplete = serde_json::from_slice::<DataUsageInfo>(&baseline).expect("test baseline should decode");
incomplete.usage_snapshot_converged = Some(false);
let incomplete = bytes::Bytes::from(serde_json::to_vec(&incomplete).expect("test baseline should encode"));
assert_eq!(
complete_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof {
data: Some(&incomplete),
expected_sources: &expected_sources,
leader_epoch: 11,
want_cycle: 8,
scan_plan_digest,
}),
None
);
let mut wrong_provenance = serde_json::from_slice::<DataUsageInfo>(&baseline).expect("test baseline should decode");
wrong_provenance.usage_snapshot_set_states[0].scan_plan_digest = Some([8; 32]);
let wrong_provenance = bytes::Bytes::from(serde_json::to_vec(&wrong_provenance).expect("test baseline should encode"));
assert_eq!(
complete_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof {
data: Some(&wrong_provenance),
expected_sources: &expected_sources,
leader_epoch: 11,
want_cycle: 8,
scan_plan_digest,
}),
None
);
}
#[test]
fn scoped_scan_selects_only_current_dirty_buckets_after_baseline_validation() {
let source = DataUsageCacheSource::new(1, 2);
let expected_sources = HashSet::from([source]);
let baseline_scan_plan_digest = DataUsageScanPlanDigest([4; 32]);
let current_scan_plan_digest = DataUsageScanPlanDigest([5; 32]);
let baseline = complete_usage_baseline(source, current_scan_plan_digest, 7, 11);
let scope = scoped_scan_scope_from_dirty_buckets(
ScannerBucketScanScope::default(),
HashSet::from(["photos".to_string(), "deleted".to_string()]),
true,
&[bucket_info("photos")],
ScannerCacheBaselineProof {
data: Some(&baseline),
expected_sources: &expected_sources,
leader_epoch: 11,
want_cycle: 8,
scan_plan_digest: current_scan_plan_digest,
},
);
assert_eq!(scope.baseline_scan_plan_digest, Some(current_scan_plan_digest));
assert_eq!(
scope
.selected_buckets
.as_deref()
.expect("validated scope should select a bucket"),
&HashSet::from(["photos".to_string()])
);
assert_ne!(scope.baseline_scan_plan_digest, Some(baseline_scan_plan_digest));
}
fn peer_dirty_usage_snapshot(
instance_id: &str,
generation: u64,
complete: bool,
buckets: &[(&str, u64)],
) -> EcstoreScannerPeerDirtyUsageSnapshot {
EcstoreScannerPeerDirtyUsageSnapshot {
instance_id: instance_id.to_string(),
generation,
pending_bucket_count: u64::try_from(buckets.len()).expect("test bucket count should fit"),
protocol_version: crate::SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION,
complete,
buckets: buckets
.iter()
.map(|(bucket, generation)| ((*bucket).to_string(), *generation))
.collect(),
}
}
#[test]
fn verified_remote_dirty_usage_buckets_merges_only_complete_current_snapshots() {
let expected_peers = HashMap::from([
(
"node-a:9000".to_string(),
ScannerPeerDirtyUsageExpectation {
instance_id: "instance-a".to_string(),
generation: 7,
pending: true,
},
),
(
"node-b:9000".to_string(),
ScannerPeerDirtyUsageExpectation {
instance_id: "instance-b".to_string(),
generation: 3,
pending: false,
},
),
]);
assert_eq!(
verified_remote_dirty_usage_buckets(
&expected_peers,
vec![
(
"node-a:9000".to_string(),
peer_dirty_usage_snapshot("instance-a", 7, true, &[("photos", 7)]),
),
(
"node-b:9000".to_string(),
peer_dirty_usage_snapshot("instance-b", 3, true, &[("archive", 3)]),
),
],
),
Some(HashSet::from(["photos".to_string(), "archive".to_string()]))
);
}
#[test]
fn verified_remote_dirty_usage_buckets_rejects_incomplete_or_stale_peer_state() {
let expected_peers = HashMap::from([(
"node-a:9000".to_string(),
ScannerPeerDirtyUsageExpectation {
instance_id: "instance-a".to_string(),
generation: 7,
pending: true,
},
)]);
for snapshot in [
peer_dirty_usage_snapshot("instance-a", 7, false, &[("photos", 7)]),
peer_dirty_usage_snapshot("instance-a", 6, true, &[("photos", 6)]),
peer_dirty_usage_snapshot("instance-b", 7, true, &[("photos", 7)]),
peer_dirty_usage_snapshot("instance-a", 7, true, &[]),
] {
assert!(
verified_remote_dirty_usage_buckets(&expected_peers, vec![("node-a:9000".to_string(), snapshot)]).is_none(),
"incomplete, stale, mismatched, or empty pending peer state must fall back to a full scan"
);
}
}
#[test] #[test]
fn scoped_set_scan_preserves_unselected_usage_and_drops_deleted_buckets() { fn scoped_set_scan_preserves_unselected_usage_and_drops_deleted_buckets() {
let baseline_digest = DataUsageScanPlanDigest([1; 32]); let baseline_digest = DataUsageScanPlanDigest([1; 32]);
+5 -3
View File
@@ -38,8 +38,8 @@ pub(crate) use rustfs_ecstore::api::bucket::lifecycle::lifecycle::object_opts_fr
#[cfg(test)] #[cfg(test)]
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys as ecstore_init_bucket_metadata_sys; pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys as ecstore_init_bucket_metadata_sys;
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{ pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
get_lifecycle_config as ecstore_get_lifecycle_config, get_object_lock_config as ecstore_get_object_lock_config, BucketMetadataMutationGuard as EcstoreBucketMetadataMutationGuard, get_lifecycle_config as ecstore_get_lifecycle_config,
get_replication_config as ecstore_get_replication_config, get_object_lock_config as ecstore_get_object_lock_config, get_replication_config as ecstore_get_replication_config,
}; };
pub(crate) use rustfs_ecstore::api::bucket::replication::{ pub(crate) use rustfs_ecstore::api::bucket::replication::{
ReplicateObjectInfo, ReplicationConfig as EcstoreReplicationConfig, ReplicateObjectInfo, ReplicationConfig as EcstoreReplicationConfig,
@@ -103,7 +103,9 @@ pub(crate) use rustfs_ecstore::api::rebalance::{
RebalStatus as EcstoreRebalStatus, RebalanceInfo as EcstoreRebalanceInfo, RebalanceMeta as EcstoreRebalanceMeta, RebalStatus as EcstoreRebalStatus, RebalanceInfo as EcstoreRebalanceInfo, RebalanceMeta as EcstoreRebalanceMeta,
RebalanceStats as EcstoreRebalanceStats, RebalanceStats as EcstoreRebalanceStats,
}; };
pub(crate) use rustfs_ecstore::api::rpc::ScannerBucketListing as EcstoreScannerBucketListing; pub(crate) use rustfs_ecstore::api::rpc::{
ScannerBucketListing as EcstoreScannerBucketListing, ScannerPeerDirtyUsageSnapshot as EcstoreScannerPeerDirtyUsageSnapshot,
};
#[cfg(test)] #[cfg(test)]
pub(crate) use rustfs_ecstore::api::runtime::InstanceContext as EcstoreInstanceContext; pub(crate) use rustfs_ecstore::api::runtime::InstanceContext as EcstoreInstanceContext;
pub(crate) use rustfs_ecstore::api::runtime::{ pub(crate) use rustfs_ecstore::api::runtime::{
+177 -20
View File
@@ -39,7 +39,7 @@ use rustfs_utils::path::path_join;
use s3s::header::{CONTENT_LENGTH, CONTENT_TYPE}; use s3s::header::{CONTENT_LENGTH, CONTENT_TYPE};
use s3s::{Body, S3Request, S3Response, S3Result, s3_error}; use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, HashSet}; use std::collections::{BTreeMap, BTreeSet, HashSet};
use std::future::Future; use std::future::Future;
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::Arc; use std::sync::Arc;
@@ -261,6 +261,7 @@ struct BackgroundHealStatus<'a> {
heal_active_tasks: u64, heal_active_tasks: u64,
heal_operations: rustfs_heal::HealOperationsSnapshot, heal_operations: rustfs_heal::HealOperationsSnapshot,
cluster_status_complete: bool, cluster_status_complete: bool,
coverage: &'a BackgroundHealCoverage,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
progress: Option<BackgroundHealProgress>, progress: Option<BackgroundHealProgress>,
} }
@@ -300,6 +301,23 @@ fn background_heal_runtime_state(
type BackgroundHealProgress = rustfs_heal::HealProgress; type BackgroundHealProgress = rustfs_heal::HealProgress;
#[derive(Debug, Serialize)]
struct BackgroundHealCoverage {
expected: usize,
responded: usize,
unknown: usize,
reasons: BTreeSet<BackgroundHealCoverageReason>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)]
#[serde(rename_all = "snake_case")]
enum BackgroundHealCoverageReason {
NotificationSystemUnavailable,
PeerTopologyIncomplete,
PeerStatusUnsupported,
PeerStatusUnavailable,
}
#[derive(Debug)] #[derive(Debug)]
struct ClusterHealStatusSnapshot { struct ClusterHealStatusSnapshot {
info: BackgroundHealInfo, info: BackgroundHealInfo,
@@ -307,6 +325,7 @@ struct ClusterHealStatusSnapshot {
operations: rustfs_heal::HealOperationsSnapshot, operations: rustfs_heal::HealOperationsSnapshot,
progress: Option<BackgroundHealProgress>, progress: Option<BackgroundHealProgress>,
complete: bool, complete: bool,
coverage: BackgroundHealCoverage,
} }
fn add_priority_counts(total: &mut rustfs_heal::HealPriorityCounts, next: rustfs_heal::HealPriorityCounts) { fn add_priority_counts(total: &mut rustfs_heal::HealPriorityCounts, next: rustfs_heal::HealPriorityCounts) {
@@ -338,6 +357,7 @@ fn add_operations(total: &mut rustfs_heal::HealOperationsSnapshot, next: rustfs_
} }
fn aggregate_cluster_heal_status(snapshots: Vec<NodeHealStatusSnapshot>) -> ClusterHealStatusSnapshot { fn aggregate_cluster_heal_status(snapshots: Vec<NodeHealStatusSnapshot>) -> ClusterHealStatusSnapshot {
let responded = snapshots.len();
let mut info = BackgroundHealInfo::default(); let mut info = BackgroundHealInfo::default();
let mut operations = rustfs_heal::HealOperationsSnapshot::default(); let mut operations = rustfs_heal::HealOperationsSnapshot::default();
let mut progress = Vec::new(); let mut progress = Vec::new();
@@ -379,6 +399,12 @@ fn aggregate_cluster_heal_status(snapshots: Vec<NodeHealStatusSnapshot>) -> Clus
operations, operations,
progress, progress,
complete: true, complete: true,
coverage: BackgroundHealCoverage {
expected: responded,
responded,
unknown: 0,
reasons: BTreeSet::new(),
},
} }
} }
@@ -413,12 +439,14 @@ fn merge_peer_heal_statuses(
mut snapshots: Vec<NodeHealStatusSnapshot>, mut snapshots: Vec<NodeHealStatusSnapshot>,
peer_statuses: Vec<Result<Option<NodeHealStatusSnapshot>, String>>, peer_statuses: Vec<Result<Option<NodeHealStatusSnapshot>, String>>,
expected_nodes: usize, expected_nodes: usize,
topology_complete: bool, coverage_reason: Option<BackgroundHealCoverageReason>,
) -> S3Result<ClusterHealStatusSnapshot> { ) -> S3Result<ClusterHealStatusSnapshot> {
let mut reasons: BTreeSet<_> = coverage_reason.into_iter().collect();
for peer_status in peer_statuses { for peer_status in peer_statuses {
match peer_status { match peer_status {
Ok(Some(snapshot)) => snapshots.push(snapshot), Ok(Some(snapshot)) => snapshots.push(snapshot),
Ok(None) => { Ok(None) => {
reasons.insert(BackgroundHealCoverageReason::PeerStatusUnsupported);
warn!( warn!(
event = EVENT_ADMIN_REQUEST_FAILED, event = EVENT_ADMIN_REQUEST_FAILED,
component = LOG_COMPONENT_ADMIN_API, component = LOG_COMPONENT_ADMIN_API,
@@ -430,6 +458,7 @@ fn merge_peer_heal_statuses(
); );
} }
Err(err) => { Err(err) => {
reasons.insert(BackgroundHealCoverageReason::PeerStatusUnavailable);
warn!( warn!(
event = EVENT_ADMIN_REQUEST_FAILED, event = EVENT_ADMIN_REQUEST_FAILED,
component = LOG_COMPONENT_ADMIN_API, component = LOG_COMPONENT_ADMIN_API,
@@ -452,9 +481,12 @@ fn merge_peer_heal_statuses(
// so during a reconfiguration the count can equal `expected_nodes` while // so during a reconfiguration the count can equal `expected_nodes` while
// the topology is known-incomplete. Counting alone would report a // the topology is known-incomplete. Counting alone would report a
// definitive answer precisely when the membership itself is in doubt. // definitive answer precisely when the membership itself is in doubt.
let complete = topology_complete && snapshots.len() == expected_nodes; let complete = reasons.is_empty() && snapshots.len() == expected_nodes;
let mut status = aggregate_cluster_heal_status(snapshots); let mut status = aggregate_cluster_heal_status(snapshots);
status.complete = complete; status.complete = complete;
status.coverage.expected = expected_nodes;
status.coverage.unknown = expected_nodes.saturating_sub(status.coverage.responded);
status.coverage.reasons = reasons;
// A partial answer must never be mistakable for a definitive verdict: an // A partial answer must never be mistakable for a definitive verdict: an
// unreachable peer might be mid-heal, so reporting the reachable nodes' // unreachable peer might be mid-heal, so reporting the reachable nodes'
// "idle" (or disabled/uninitialized) as the cluster state would falsely // "idle" (or disabled/uninitialized) as the cluster state would falsely
@@ -497,7 +529,12 @@ async fn read_cluster_heal_status(
return Ok(aggregate_cluster_heal_status(snapshots)); return Ok(aggregate_cluster_heal_status(snapshots));
} }
let Some(notification_system) = notification_system else { let Some(notification_system) = notification_system else {
return Err(cluster_heal_status_unavailable("notification_system_unavailable")); return merge_peer_heal_statuses(
snapshots,
Vec::new(),
expected_nodes,
Some(BackgroundHealCoverageReason::NotificationSystemUnavailable),
);
}; };
// An incomplete peer topology (a down member's client slot, a rolling // An incomplete peer topology (a down member's client slot, a rolling
// upgrade) previously failed the whole endpoint here, before any peer was // upgrade) previously failed the whole endpoint here, before any peer was
@@ -540,7 +577,12 @@ async fn read_cluster_heal_status(
})) }))
.await; .await;
merge_peer_heal_statuses(snapshots, peer_statuses, expected_nodes, topology_complete) merge_peer_heal_statuses(
snapshots,
peer_statuses,
expected_nodes,
(!topology_complete).then_some(BackgroundHealCoverageReason::PeerTopologyIncomplete),
)
} }
async fn query_peer_replacement_recovery_status<E>( async fn query_peer_replacement_recovery_status<E>(
@@ -1164,6 +1206,7 @@ fn encode_background_heal_status(
heal_operations: rustfs_heal::HealOperationsSnapshot, heal_operations: rustfs_heal::HealOperationsSnapshot,
progress: Option<BackgroundHealProgress>, progress: Option<BackgroundHealProgress>,
cluster_status_complete: bool, cluster_status_complete: bool,
coverage: &BackgroundHealCoverage,
) -> S3Result<Vec<u8>> { ) -> S3Result<Vec<u8>> {
let status = BackgroundHealStatus { let status = BackgroundHealStatus {
info, info,
@@ -1172,6 +1215,7 @@ fn encode_background_heal_status(
heal_active_tasks: heal_operations.active_tasks, heal_active_tasks: heal_operations.active_tasks,
heal_operations, heal_operations,
cluster_status_complete, cluster_status_complete,
coverage,
progress, progress,
}; };
serde_json::to_vec(&status).map_err(|e| { serde_json::to_vec(&status).map_err(|e| {
@@ -1461,6 +1505,7 @@ impl Operation for BackgroundHealStatusHandler {
cluster_status.operations, cluster_status.operations,
cluster_status.progress, cluster_status.progress,
cluster_status.complete, cluster_status.complete,
&cluster_status.coverage,
)?; )?;
info!( info!(
event = EVENT_ADMIN_RESPONSE_EMITTED, event = EVENT_ADMIN_RESPONSE_EMITTED,
@@ -1515,13 +1560,14 @@ impl Operation for ReplacementRecoveryStatusHandler {
mod tests { mod tests {
use super::extract_heal_init_params; use super::extract_heal_init_params;
use super::{ use super::{
BackgroundHealProgress, HealInitParams, HealResp, HealRuntimeState, aggregate_cluster_heal_status, BackgroundHealCoverage, BackgroundHealCoverageReason, BackgroundHealProgress, HealInitParams, HealResp, HealRuntimeState,
aggregate_replacement_recovery_cluster_status, background_heal_runtime_state, build_heal_channel_request, aggregate_cluster_heal_status, aggregate_replacement_recovery_cluster_status, background_heal_runtime_state,
build_replacement_recovery_status_response, encode_background_heal_status, encode_heal_control_path, build_heal_channel_request, build_replacement_recovery_status_response, encode_background_heal_status,
encode_heal_start_success, encode_heal_task_status, execute_after_heal_control_capability, heal_channel_response_items, encode_heal_control_path, encode_heal_start_success, encode_heal_task_status, execute_after_heal_control_capability,
heal_channel_response_progress, heal_channel_response_summary, heal_control_response_id, json_response, heal_channel_response_items, heal_channel_response_progress, heal_channel_response_summary, heal_control_response_id,
map_heal_response, merge_peer_heal_statuses, peer_topology_complete, query_peer_heal_status, json_response, map_heal_response, merge_peer_heal_statuses, peer_topology_complete, query_peer_heal_status,
query_peer_replacement_recovery_status, reject_heal_admission, validate_heal_request_mode, validate_heal_target, query_peer_replacement_recovery_status, read_cluster_heal_status, reject_heal_admission, validate_heal_request_mode,
validate_heal_target,
}; };
use crate::storage::rpc::node_service::heal::{ use crate::storage::rpc::node_service::heal::{
NodeHealProgress, NodeHealStatusSnapshot, NodeReplacementRecoveryStatusSnapshot, encode_node_replacement_recovery_status, NodeHealProgress, NodeHealStatusSnapshot, NodeReplacementRecoveryStatusSnapshot, encode_node_replacement_recovery_status,
@@ -2175,7 +2221,13 @@ mod tests {
..Default::default() ..Default::default()
}; };
let encoded = encode_background_heal_status(&info, HealRuntimeState::Active, operations, None, true) let coverage = BackgroundHealCoverage {
expected: 1,
responded: 1,
unknown: 0,
reasons: Default::default(),
};
let encoded = encode_background_heal_status(&info, HealRuntimeState::Active, operations, None, true, &coverage)
.expect("background heal info should serialize"); .expect("background heal info should serialize");
let json: serde_json::Value = serde_json::from_slice(&encoded).expect("json should deserialize"); let json: serde_json::Value = serde_json::from_slice(&encoded).expect("json should deserialize");
@@ -2225,6 +2277,12 @@ mod tests {
rustfs_heal::HealOperationsSnapshot::default(), rustfs_heal::HealOperationsSnapshot::default(),
Some(progress), Some(progress),
true, true,
&BackgroundHealCoverage {
expected: 1,
responded: 1,
unknown: 0,
reasons: Default::default(),
},
) )
.expect("background heal info should serialize"); .expect("background heal info should serialize");
let json: serde_json::Value = serde_json::from_slice(&encoded).expect("json should deserialize"); let json: serde_json::Value = serde_json::from_slice(&encoded).expect("json should deserialize");
@@ -2398,6 +2456,84 @@ mod tests {
assert!(peer_topology_complete(1, 0, 0, 1, 0)); assert!(peer_topology_complete(1, 0, 0, 1, 0));
} }
#[tokio::test]
async fn test_background_heal_status_without_notification_preserves_local_snapshot() {
let initialized = rustfs_heal::heal_runtime_initialized();
let info = BackgroundHealInfo {
bitrot_start_cycle: 37,
current_scan_mode: HealScanMode::Deep,
..Default::default()
};
for expected in [1, 3] {
let status = tokio::time::timeout(Duration::from_secs(1), read_cluster_heal_status(info.clone(), None, expected))
.await
.expect("local status must not wait for remote peers")
.expect("missing notification must retain the local snapshot");
assert_eq!(status.info.bitrot_start_cycle, 37);
assert_eq!(status.info.current_scan_mode, HealScanMode::Deep);
assert_eq!(status.complete, expected == 1);
assert_eq!(status.coverage.expected, expected);
assert_eq!(status.coverage.responded, 1);
assert_eq!(status.coverage.unknown, expected - 1);
if expected == 1 {
assert!(status.coverage.reasons.is_empty());
} else {
assert!(matches!(status.state, HealRuntimeState::Degraded | HealRuntimeState::Active));
assert_eq!(
status.coverage.reasons,
[BackgroundHealCoverageReason::NotificationSystemUnavailable].into()
);
}
let encoded = encode_background_heal_status(
&status.info,
status.state,
status.operations,
status.progress,
status.complete,
&status.coverage,
)
.expect("fallback status must encode");
let decoded: rustfs_madmin::client::BackgroundHealStatus =
serde_json::from_slice(&encoded).expect("the actual madmin client must decode the server response");
assert_eq!(decoded.cluster_status_complete, expected == 1);
let coverage = decoded.coverage.expect("new server supplies coverage");
assert_eq!(coverage.expected, Some(expected));
assert_eq!(coverage.responded, Some(1));
assert_eq!(coverage.unknown, Some(expected - 1));
if expected > 1 {
assert_eq!(coverage.reasons, ["notification_system_unavailable"]);
}
}
assert_eq!(
rustfs_heal::heal_runtime_initialized(),
initialized,
"reading status must not initialize heal"
);
}
#[test]
fn test_background_heal_status_coverage_reasons_are_bounded() {
let local = NodeHealStatusSnapshot::for_test(true, true, BackgroundHealInfo::default(), Default::default(), None);
let peers = (0..100)
.map(|index| {
if index % 2 == 0 {
Ok(None)
} else {
Err("peer unavailable".to_owned())
}
})
.collect();
let status = merge_peer_heal_statuses(vec![local], peers, 101, None).expect("local status remains available");
assert_eq!(status.coverage.responded, 1);
assert_eq!(status.coverage.unknown, 100);
assert_eq!(status.coverage.reasons.len(), 2);
let encoded = serde_json::to_vec(&status.coverage).expect("coverage encodes");
assert!(encoded.len() < 256, "coverage must not grow with peer failures");
let decoded: rustfs_madmin::client::BackgroundHealCoverage =
serde_json::from_slice(&encoded).expect("client coverage decodes");
assert_eq!(decoded.reasons, ["peer_status_unsupported", "peer_status_unavailable"]);
}
#[test] #[test]
fn test_peer_status_merge_degrades_explicitly_and_never_claims_idle() { fn test_peer_status_merge_degrades_explicitly_and_never_claims_idle() {
let local = || { let local = || {
@@ -2413,15 +2549,20 @@ mod tests {
// but the safety property of the previous fail-closed behaviour is // but the safety property of the previous fail-closed behaviour is
// preserved: the partial answer is labelled Degraded, never Idle, so // preserved: the partial answer is labelled Degraded, never Idle, so
// unknown peer work cannot be mistaken for "nothing is running". // unknown peer work cannot be mistaken for "nothing is running".
let partial = merge_peer_heal_statuses(vec![local()], vec![Err("peer timeout".to_string())], 2, true) let partial = merge_peer_heal_statuses(vec![local()], vec![Err("peer timeout".to_string())], 2, None)
.expect("an unreachable peer degrades the answer instead of destroying it"); .expect("an unreachable peer degrades the answer instead of destroying it");
assert!(!partial.complete); assert!(!partial.complete);
assert_eq!(partial.state, HealRuntimeState::Degraded); assert_eq!(partial.state, HealRuntimeState::Degraded);
assert_eq!(partial.coverage.expected, 2);
assert_eq!(partial.coverage.responded, 1);
assert_eq!(partial.coverage.unknown, 1);
assert_eq!(partial.coverage.reasons, [BackgroundHealCoverageReason::PeerStatusUnavailable].into());
let older_peer = merge_peer_heal_statuses(vec![local()], vec![Ok(None)], 2, true) let older_peer = merge_peer_heal_statuses(vec![local()], vec![Ok(None)], 2, None)
.expect("an older peer degrades the answer instead of destroying it"); .expect("an older peer degrades the answer instead of destroying it");
assert!(!older_peer.complete); assert!(!older_peer.complete);
assert_eq!(older_peer.state, HealRuntimeState::Degraded); assert_eq!(older_peer.state, HealRuntimeState::Degraded);
assert_eq!(older_peer.coverage.reasons, [BackgroundHealCoverageReason::PeerStatusUnsupported].into());
let known_active = NodeHealStatusSnapshot::for_test( let known_active = NodeHealStatusSnapshot::for_test(
true, true,
@@ -2433,12 +2574,12 @@ mod tests {
}, },
None, None,
); );
let partial_active = merge_peer_heal_statuses(vec![known_active], vec![Ok(None)], 2, true) let partial_active = merge_peer_heal_statuses(vec![known_active], vec![Ok(None)], 2, None)
.expect("known active work may be reported as an explicit partial status"); .expect("known active work may be reported as an explicit partial status");
assert!(!partial_active.complete); assert!(!partial_active.complete);
assert_eq!(partial_active.state, HealRuntimeState::Active); assert_eq!(partial_active.state, HealRuntimeState::Active);
merge_peer_heal_statuses(Vec::new(), vec![Err("peer timeout".to_string())], 2, true) merge_peer_heal_statuses(Vec::new(), vec![Err("peer timeout".to_string())], 2, None)
.expect_err("no snapshot at all still fails closed"); .expect_err("no snapshot at all still fails closed");
} }
@@ -2459,12 +2600,22 @@ mod tests {
None, None,
) )
}; };
let full_count_incomplete_topology = merge_peer_heal_statuses(vec![snapshot()], vec![Ok(Some(snapshot()))], 2, false) let full_count_incomplete_topology = merge_peer_heal_statuses(
.expect("incomplete topology degrades the answer instead of destroying it"); vec![snapshot()],
vec![Ok(Some(snapshot()))],
2,
Some(BackgroundHealCoverageReason::PeerTopologyIncomplete),
)
.expect("incomplete topology degrades the answer instead of destroying it");
assert!(!full_count_incomplete_topology.complete); assert!(!full_count_incomplete_topology.complete);
assert_eq!(full_count_incomplete_topology.state, HealRuntimeState::Degraded); assert_eq!(full_count_incomplete_topology.state, HealRuntimeState::Degraded);
assert_eq!(full_count_incomplete_topology.coverage.unknown, 0);
assert_eq!(
full_count_incomplete_topology.coverage.reasons,
[BackgroundHealCoverageReason::PeerTopologyIncomplete].into()
);
let full_count_complete_topology = merge_peer_heal_statuses(vec![snapshot()], vec![Ok(Some(snapshot()))], 2, true) let full_count_complete_topology = merge_peer_heal_statuses(vec![snapshot()], vec![Ok(Some(snapshot()))], 2, None)
.expect("complete topology and full count is a definitive answer"); .expect("complete topology and full count is a definitive answer");
assert!(full_count_complete_topology.complete); assert!(full_count_complete_topology.complete);
assert_eq!(full_count_complete_topology.state, HealRuntimeState::Idle); assert_eq!(full_count_complete_topology.state, HealRuntimeState::Idle);
@@ -2478,6 +2629,12 @@ mod tests {
rustfs_heal::HealOperationsSnapshot::default(), rustfs_heal::HealOperationsSnapshot::default(),
None, None,
false, false,
&BackgroundHealCoverage {
expected: 2,
responded: 1,
unknown: 1,
reasons: [BackgroundHealCoverageReason::PeerStatusUnavailable].into(),
},
) )
.expect("degraded status must serialize"); .expect("degraded status must serialize");
let json: serde_json::Value = serde_json::from_slice(&encoded).expect("valid json"); let json: serde_json::Value = serde_json::from_slice(&encoded).expect("valid json");
+418 -244
View File
@@ -190,7 +190,8 @@ fn site_replicator_service_account_policy() -> S3Result<Policy> {
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("parse site replicator policy failed: {e}"))) .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("parse site replicator policy failed: {e}")))
} }
// Lock order: lifecycle -> bucket operation -> repair admission -> state -> per-bucket metadata. // Lock order: lifecycle -> bucket-mutation admission -> per-bucket mutation
// -> bucket operation -> repair admission -> state -> per-bucket metadata.
// "state" is the distributed state-object lock in // "state" is the distributed state-object lock in
// crate::site_replication::state_lock, entered through // crate::site_replication::state_lock, entered through
// update_site_replication_state (P1-15). There is no process-local state // update_site_replication_state (P1-15). There is no process-local state
@@ -434,6 +435,7 @@ pub fn register_site_replication_route(r: &mut S3Router<AdminOperation>) -> std:
// into this module: startup sits below this layer and must not depend upwards. The admin // into this module: startup sits below this layer and must not depend upwards. The admin
// router is built before startup reconciles, so the hook is always installed in time. // router is built before startup reconciles, so the hook is always installed in time.
crate::site_replication_reconcile::register_site_replication_reconciler(reconcile_site_replication_wiring); crate::site_replication_reconcile::register_site_replication_reconciler(reconcile_site_replication_wiring);
crate::site_replication_reconcile::register_site_replication_retry_drainer(reconcile_site_replication_retry_drain);
for (method, path, operation) in [ for (method, path, operation) in [
(Method::PUT, "/v3/site-replication/add", AdminOperation(&SiteReplicationAddHandler {})), (Method::PUT, "/v3/site-replication/add", AdminOperation(&SiteReplicationAddHandler {})),
@@ -1803,28 +1805,61 @@ async fn reconcile_site_replication_buckets() -> S3Result<()> {
/// (`SiteReplicationEditHandler`), so a tick landing between them would rewrite the targets /// (`SiteReplicationEditHandler`), so a tick landing between them would rewrite the targets
/// from the stale endpoint. The pending marker in the persisted state closes that window. /// from the stale endpoint. The pending marker in the persisted state closes that window.
/// Skipping costs nothing — the timer comes back. /// Skipping costs nothing — the timer comes back.
async fn site_replication_reconcile_prerequisites_ready() -> bool {
if current_iam_handle().is_none() || current_object_store_handle().is_none() {
return false;
}
if let Err(err) = migrate_collapsed_retry_queue_paths().await {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
result = "retry_queue_migration_failed",
error = ?err,
"admin site replication state"
);
return false;
}
true
}
fn reconcile_site_replication_retry_drain() -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> {
Box::pin(async {
let Some(lifecycle) = SiteReplicationLifecycleGuard::try_acquire() else {
return;
};
if !site_replication_reconcile_prerequisites_ready().await {
return;
}
match load_site_replication_state().await {
Ok(state) => {
if state.pending_endpoint_refresh.is_some() || state.pending_rotation.is_some() || state.pending_remove.is_some()
{
return;
}
}
Err(_) => return,
}
// Admission above observes a lifecycle-stable state. The lightweight
// drain itself handles only idempotent bucket setup, reloads state
// under the distributed repair lock, and shares that lock with bucket
// deletion. Do not hold this process-local guard across peer I/O: an
// outage recovery must not make admin add/edit/remove time out.
drop(lifecycle);
drain_site_replication_retry_queue_lightweight().await;
})
}
fn reconcile_site_replication_wiring() -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> { fn reconcile_site_replication_wiring() -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> {
Box::pin(async { Box::pin(async {
// The scheduler starts before IAM and the object store are guaranteed ready (IAM // The scheduler starts before IAM and the object store are guaranteed ready (IAM
// bootstrap may still be recovering), so an early tick returns quietly instead of // bootstrap may still be recovering), so an early tick returns quietly instead of
// logging a failure for every reconciler. // logging a failure for every reconciler.
if current_iam_handle().is_none() || current_object_store_handle().is_none() { let Some(lifecycle) = SiteReplicationLifecycleGuard::try_acquire() else {
return;
}
let Some(_lifecycle) = SiteReplicationLifecycleGuard::try_acquire() else {
return; return;
}; };
if let Err(err) = migrate_collapsed_retry_queue_paths().await { if !site_replication_reconcile_prerequisites_ready().await {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
result = "retry_queue_migration_failed",
error = ?err,
"admin site replication state"
);
return; return;
} }
@@ -1878,8 +1913,9 @@ fn reconcile_site_replication_wiring() -> std::pin::Pin<Box<dyn std::future::Fut
"admin site replication state" "admin site replication state"
); );
} }
// Failed peer deliveries recorded in the retry queue; runs behind the // The retry path re-checks membership from distributed state before
// same lifecycle guard and pending_* gates as the reconcilers above. // each request; release lifecycle before a potentially large replay.
drop(lifecycle);
drain_site_replication_retry_queue().await; drain_site_replication_retry_queue().await;
}) })
} }
@@ -3046,6 +3082,7 @@ fn set_pending_endpoint_refresh(state: &mut SiteReplicationState, pending: Pendi
last_error: "endpoint target refresh pending".to_string(), last_error: "endpoint target refresh pending".to_string(),
updated_at: Some(OffsetDateTime::now_utc()), updated_at: Some(OffsetDateTime::now_utc()),
edit_generation: None, edit_generation: None,
peer_unreachable: false,
deletions_recorded: false, deletions_recorded: false,
}); });
state.pending_endpoint_refresh = Some(pending); state.pending_endpoint_refresh = Some(pending);
@@ -3592,16 +3629,15 @@ const PEER_EDIT_FENCE_STALENESS_WINDOW_NANOS: u64 = 24 * 60 * 60 * 1_000_000_000
/// must be a site this state currently replicates with — the same membership /// must be a site this state currently replicates with — the same membership
/// rule the load-time mark pruning applies, so every mark recorded behind /// rule the load-time mark pruning applies, so every mark recorded behind
/// this check is one a reload would keep — and not this site itself, which /// this check is one a reload would keep — and not this site itself, which
/// never delivers edits to itself. The caller IGNORES an inadmissible fence /// never delivers edits to itself. The caller acknowledges an inadmissible
/// rather than failing the request: the delivery applies exactly as an /// fenced request without applying it: after a remove commits, an older
/// unstamped (pre-fence) delivery would, no high-water mark is read or /// in-flight retry from the departed origin must not recreate topology. Old
/// written, and the worst a forged fence achieves is forfeiting an ordering /// peers remain compatible because their unstamped edits still follow the
/// guarantee its sender was never owed. The generation itself is NOT /// pre-fence path. The generation itself is NOT bounded here: a genuine
/// bounded here: a genuine origin whose hybrid clock persisted a wall-clock /// origin whose hybrid clock persisted a wall-clock excursion allocates
/// excursion allocates arbitrarily far in the future, and refusing to /// arbitrarily far in the future, and refusing to record its marks would
/// record its marks would strip the ordering fence from exactly the /// strip the ordering fence from exactly the deliveries that still race —
/// deliveries that still race — the staleness window on the read side is /// the staleness window on the read side is what defuses forged marks instead.
/// what defuses forged marks instead.
fn peer_edit_fence_is_admissible(state: &SiteReplicationState, local_deployment_id: &str, fence: &(String, u64)) -> bool { fn peer_edit_fence_is_admissible(state: &SiteReplicationState, local_deployment_id: &str, fence: &(String, u64)) -> bool {
let (origin, generation) = fence; let (origin, generation) = fence;
if origin != local_deployment_id && state.peers.contains_key(origin) { if origin != local_deployment_id && state.peers.contains_key(origin) {
@@ -4791,105 +4827,135 @@ async fn backfill_existing_buckets_after_add(
let resync_id = Uuid::new_v4().to_string(); let resync_id = Uuid::new_v4().to_string();
for bucket in &buckets { for bucket in &buckets {
let name = &bucket.name; let operation_name = bucket.name.clone();
let lock_bucket = operation_name.clone();
let operation_state = state.clone();
let operation_local_peer = local_peer.clone();
let operation_resync_id = resync_id.clone();
let operation_bootstrap_token = bootstrap_token.map(str::to_owned);
let bucket_errors = with_site_replication_bucket_mutation_lock(store.clone(), &lock_bucket, move || async move {
let mut errors = SiteReplicationErrorSummary::default();
let name = &operation_name;
if let Err(err) = ensure_site_replication_bucket_versioning(name).await { if let Err(err) = ensure_site_replication_bucket_versioning(name).await {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
result = "backfill_versioning_setup_failed",
error = ?err,
"admin site replication state"
);
errors.push(format!("{name}: versioning setup failed: {err}"));
continue;
}
match ensure_site_replication_bucket_setup(name).await {
Ok(true) => {}
Ok(false) => {
// Runtime targets unavailable: the setup silently no-ops, which would make the
// downstream make-bucket broadcast and resync fail. Record it and skip so the
// operator sees this bucket was not propagated instead of an unqualified success.
warn!( warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE, event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name, bucket = %name,
result = "backfill_bucket_setup_skipped", result = "backfill_versioning_setup_failed",
"admin site replication state"
);
errors.push(format!("{name}: replication setup skipped (site replication runtime unavailable)"));
continue;
}
Err(err) => {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
result = "backfill_bucket_setup_failed",
error = ?err, error = ?err,
"admin site replication state" "admin site replication state"
); );
errors.push(format!("{name}: bucket setup failed: {err}")); errors.push(format!("{name}: versioning setup failed: {err}"));
return errors;
} }
} match ensure_site_replication_bucket_setup(name).await {
// Broadcast the bucket to peers so they create it too (idempotent on the peer side). Ok(true) => {}
// Read the real lock_enabled flag so peers recreate the bucket with the same object-lock Ok(false) => {
// setting — object lock cannot be added after bucket creation. // Runtime targets unavailable: the setup silently no-ops, which would make the
let lock_enabled = match metadata_sys::get(name).await { // downstream make-bucket broadcast and resync fail. Record it and skip so the
Ok(bm) => bm.lock_enabled, // operator sees this bucket was not propagated instead of an unqualified success.
Err(err) => { warn!(
warn!( event = EVENT_ADMIN_SITE_REPLICATION_STATE,
event = EVENT_ADMIN_SITE_REPLICATION_STATE, component = LOG_COMPONENT_ADMIN,
component = LOG_COMPONENT_ADMIN, subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, bucket = %name,
bucket = %name, result = "backfill_bucket_setup_skipped",
result = "backfill_bucket_metadata_read_failed", "admin site replication state"
fallback = "lock_enabled=false", );
error = ?err, errors.push(format!("{name}: replication setup skipped (site replication runtime unavailable)"));
"admin site replication state" return errors;
); }
false Err(err) => {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
result = "backfill_bucket_setup_failed",
error = ?err,
"admin site replication state"
);
errors.push(format!("{name}: bucket setup failed: {err}"));
}
} }
}; // Broadcast the bucket to peers so they create it too (idempotent on the peer side).
if let Err(err) = broadcast_site_replication_make_bucket(name, lock_enabled, None, bootstrap_token).await { // Read the real lock_enabled flag so peers recreate the bucket with the same object-lock
warn!( // setting — object lock cannot be added after bucket creation.
event = EVENT_ADMIN_SITE_REPLICATION_STATE, let lock_enabled = match metadata_sys::get(name).await {
component = LOG_COMPONENT_ADMIN, Ok(bm) => bm.lock_enabled,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, Err(err) => {
bucket = %name, warn!(
result = "backfill_make_bucket_broadcast_failed", event = EVENT_ADMIN_SITE_REPLICATION_STATE,
error = ?err, component = LOG_COMPONENT_ADMIN,
"admin site replication state" subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
); bucket = %name,
errors.push(format!("{name}: make-bucket broadcast failed: {err}")); result = "backfill_bucket_metadata_read_failed",
} fallback = "lock_enabled=false",
// Kick a resync toward every remote peer so existing objects travel across. error = ?err,
for peer in state.peers.values() { "admin site replication state"
if peer.deployment_id == local_peer.deployment_id || same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) { );
continue; false
} }
let manifest = site_bucket_resync_manifest_entry(name, peer, OffsetDateTime::now_utc()).await;
let result = if manifest.target_arn.is_empty() {
manifest
} else {
start_site_bucket_resync(name, &manifest.target_arn, &resync_id).await
}; };
if result.status == "failed" { if let Err(err) =
broadcast_site_replication_make_bucket(name, lock_enabled, None, operation_bootstrap_token.as_deref()).await
{
warn!( warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE, event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN, component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name, bucket = %name,
peer = %peer.endpoint, result = "backfill_make_bucket_broadcast_failed",
result = "backfill_resync_kick_failed", error = ?err,
detail = %result.err_detail,
"admin site replication state" "admin site replication state"
); );
errors.push(format!("{name} -> {}: resync kick failed: {}", peer.endpoint, result.err_detail)); errors.push(format!("{name}: make-bucket broadcast failed: {err}"));
}
// Kick a resync toward every remote peer so existing objects travel across.
for peer in operation_state.peers.values() {
if peer.deployment_id == operation_local_peer.deployment_id
|| same_identity_endpoint(&peer.endpoint, &operation_local_peer.endpoint)
{
continue;
}
let manifest = site_bucket_resync_manifest_entry(name, peer, OffsetDateTime::now_utc()).await;
let result = if manifest.target_arn.is_empty() {
manifest
} else {
start_site_bucket_resync(name, &manifest.target_arn, &operation_resync_id).await
};
if result.status == "failed" {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %name,
peer = %peer.endpoint,
result = "backfill_resync_kick_failed",
detail = %result.err_detail,
"admin site replication state"
);
errors.push(format!("{name} -> {}: resync kick failed: {}", peer.endpoint, result.err_detail));
}
}
errors
})
.await;
match bucket_errors {
Ok(bucket_errors) => errors.extend(bucket_errors),
Err(err) => {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
bucket = %lock_bucket,
result = "backfill_bucket_mutation_lock_failed",
error = ?err,
"admin site replication state"
);
errors.push(format!("{lock_bucket}: bucket mutation lock failed: {err}"));
} }
} }
} }
@@ -6072,146 +6138,204 @@ fn parse_peer_join_response(body: &[u8], fallback_peer: PeerInfo) -> Result<SRPe
serde_json::from_slice(body) serde_json::from_slice(body)
} }
fn ensure_add_bucket_set_matches_preflight(expected: &HashSet<String>, present: &HashSet<String>) -> S3Result<()> {
let mut missing = expected.difference(present).cloned().collect::<Vec<_>>();
if !missing.is_empty() {
missing.sort_unstable();
return Err(S3Error::with_message(
S3ErrorCode::InvalidRequest,
format!(
"bucket `{}` disappeared while site replication was being added; peers may already be joined — re-run replicate add",
missing[0]
),
));
}
let mut unexpected = present.difference(expected).cloned().collect::<Vec<_>>();
if !unexpected.is_empty() {
unexpected.sort_unstable();
return Err(S3Error::with_message(
S3ErrorCode::InvalidRequest,
format!(
"bucket `{}` appeared while site replication was being added; peers may already be joined — re-run replicate add",
unexpected[0]
),
));
}
Ok(())
}
#[async_trait::async_trait] #[async_trait::async_trait]
impl Operation for SiteReplicationAddHandler { impl Operation for SiteReplicationAddHandler {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> { async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let cred = validate_site_replication_admin_request(&req, AdminAction::SiteReplicationAddAction).await?; let cred = validate_site_replication_admin_request(&req, AdminAction::SiteReplicationAddAction).await?;
reject_site_replicator_on_public_admin(&cred)?; reject_site_replicator_on_public_admin(&cred)?;
let replicate_ilm_expiry = sr_add_replicate_ilm_expiry(&req.uri); let replicate_ilm_expiry = sr_add_replicate_ilm_expiry(&req.uri);
let local_endpoint = site_replication_local_endpoint(&req.uri, &req.headers);
let lifecycle_guard = SiteReplicationLifecycleGuard::acquire().await?; let lifecycle_guard = SiteReplicationLifecycleGuard::acquire().await?;
// Everything up to the commit below is preflight: peer probes, IAM
// work and the join fan-out all talk to the network, so none of it may
// run inside the state transaction. The snapshot read here is what the
// `updated_at` CAS in the commit validates.
let current_state = load_site_replication_state().await?;
if pending_endpoint_refresh(&current_state).is_some() {
return Err(s3_error!(InvalidRequest, "endpoint target refresh is pending"));
}
let local_peer = current_local_peer(&req, &current_state);
let mut sites: Vec<PeerSite> = read_site_replication_json(req, &cred.secret_key, true).await?; let mut sites: Vec<PeerSite> = read_site_replication_json(req, &cred.secret_key, true).await?;
// The web console's "Set Up Site Replication" omits the local deployment from the payload; let admin_access_key = cred.access_key.clone();
// inject it so the add preflight (which requires the local deployment) succeeds. No-op for `mc`. let admission_store = current_object_store_handle()
ensure_local_site_present(&mut sites, &local_peer); .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?;
validate_add_sites(&sites, &local_peer)?; let list_store = admission_store.clone();
let preflight_infos = add_preflight_infos(&sites, &current_state, &local_peer).await?; let (state, edit_generation, local_peer, service_account_secret_key, mut initial_sync_errors, _add_guard) =
validate_add_preflight_topology(&preflight_infos, &local_peer)?; with_site_replication_bucket_mutation_admission_lock(admission_store, move || async move {
let expected_updated_at = current_state.updated_at; // The writer starts before the local bucket snapshot and stays
require_add_peer_tls_capability(&sites, &local_peer).await?; // held through every peer join and the topology commit. A
// Early exit on a state that moved under the preflight probes, BEFORE // delete followed by a same-name create therefore cannot hide
// the IAM write and the join fan-out change anything remote. Advisory // behind an unchanged final name set. Peer bootstrap callbacks
// only — the binding check is the CAS inside the commit — but it fences // use their internal path and do not acquire this public-
// the common race off the side-effect path and refreshes the merge // mutation admission lock.
// base so the CAS window is only the join round trips. let current_state = load_site_replication_state().await?;
let latest_state = load_site_replication_state().await?; if pending_endpoint_refresh(&current_state).is_some() {
ensure_edit_precondition(&latest_state, expected_updated_at, None, "add preflight")?; return Err(s3_error!(InvalidRequest, "endpoint target refresh is pending"));
let current_state = latest_state; }
let (service_account_access_key, service_account_secret_key) = let local_peer = local_peer_at_endpoint(local_endpoint, &current_state);
ensure_site_replicator_service_account(&cred.access_key, false).await?; // The web console's "Set Up Site Replication" omits the local deployment from the payload;
let bootstrap_buckets = preflight_infos // inject it so the add preflight (which requires the local deployment) succeeds. No-op for `mc`.
.iter() ensure_local_site_present(&mut sites, &local_peer);
.filter(|info| !same_identity_endpoint(&info.endpoint, &local_peer.endpoint)) validate_add_sites(&sites, &local_peer)?;
.flat_map(|info| info.buckets.keys().cloned()) let preflight_infos = add_preflight_infos(&sites, &current_state, &local_peer).await?;
.collect(); validate_add_preflight_topology(&preflight_infos, &local_peer)?;
let add_in_progress_guard = SiteReplicationAddInProgressGuard::start(lifecycle_guard, bootstrap_buckets)?; let expected_updated_at = current_state.updated_at;
let mut state = merge_add_sites( require_add_peer_tls_capability(&sites, &local_peer).await?;
current_state, // Early exit on a state that moved under the preflight probes, BEFORE
local_peer.clone(), // the IAM write and the join fan-out change anything remote. Advisory
sites.clone(), // only — the binding check is the CAS inside the commit — but it fences
service_account_access_key.clone(), // the common race off the side-effect path and refreshes the merge
cred.access_key.clone(), // base so the CAS window is only the join round trips.
replicate_ilm_expiry, let latest_state = load_site_replication_state().await?;
); ensure_edit_precondition(&latest_state, expected_updated_at, None, "add preflight")?;
state.sync_state_initialized = true; let current_state = latest_state;
let join_req = SRPeerJoinEnvelope { let (service_account_access_key, service_account_secret_key) =
request: SRPeerJoinReq { ensure_site_replicator_service_account(&admin_access_key, false).await?;
svc_acct_access_key: service_account_access_key, let expected_buckets: HashSet<String> =
svc_acct_secret_key: service_account_secret_key.clone(), preflight_infos.iter().flat_map(|info| info.buckets.keys().cloned()).collect();
svc_acct_parent: String::new(), let bootstrap_buckets: HashSet<String> = preflight_infos
peers: state.peers.clone(), .iter()
updated_at: state.updated_at, .filter(|info| !same_identity_endpoint(&info.endpoint, &local_peer.endpoint))
}, .flat_map(|info| info.buckets.keys().cloned())
defer_sync_state_enable: true, .collect();
}; let add_in_progress_guard =
let peer_join_path = SiteReplicationAddInProgressGuard::start(lifecycle_guard, bootstrap_buckets.clone())?;
with_site_replication_bootstrap_token(SITE_REPLICATION_PEER_JOIN_PATH, &add_in_progress_guard.token.to_string()); let mut state = merge_add_sites(
current_state,
local_peer.clone(),
sites.clone(),
service_account_access_key.clone(),
admin_access_key,
replicate_ilm_expiry,
);
state.sync_state_initialized = true;
let join_req = SRPeerJoinEnvelope {
request: SRPeerJoinReq {
svc_acct_access_key: service_account_access_key,
svc_acct_secret_key: service_account_secret_key.clone(),
svc_acct_parent: String::new(),
peers: state.peers.clone(),
updated_at: state.updated_at,
},
defer_sync_state_enable: true,
};
let peer_join_path = with_site_replication_bootstrap_token(
SITE_REPLICATION_PEER_JOIN_PATH,
&add_in_progress_guard.token.to_string(),
);
let mut joined_endpoints = HashSet::new(); let mut joined_endpoints = HashSet::new();
let mut initial_sync_errors = SiteReplicationErrorSummary::default(); let mut initial_sync_errors = SiteReplicationErrorSummary::default();
for (site, preflight) in sites.iter().zip(preflight_infos.iter()) { for (site, preflight) in sites.iter().zip(preflight_infos.iter()) {
if same_identity_endpoint(&site.endpoint, &local_peer.endpoint) if same_identity_endpoint(&site.endpoint, &local_peer.endpoint)
|| !joined_endpoints.insert(site_identity_key(&site.endpoint)) || !joined_endpoints.insert(site_identity_key(&site.endpoint))
{ {
continue; continue;
} }
let mut peer_join_req = join_req.clone(); let mut peer_join_req = join_req.clone();
peer_join_req.request.svc_acct_parent = site.access_key.clone(); peer_join_req.request.svc_acct_parent = site.access_key.clone();
let connection = PeerConnection::try_from(site)?; let connection = PeerConnection::try_from(site)?;
let body = PeerAdminRequest::put(&connection, &peer_join_path, &site.access_key) let body = PeerAdminRequest::put(&connection, &peer_join_path, &site.access_key)
.send(&site.secret_key, &peer_join_req) .send(&site.secret_key, &peer_join_req)
.await?;
let mut fallback_peer = existing_peer_for_endpoint(&state, &site.endpoint)
.unwrap_or_else(|| normalize_peer_site(site.clone(), replicate_ilm_expiry));
fallback_peer.deployment_id = preflight.deployment_id.clone();
let join_response = parse_peer_join_response(&body, fallback_peer).map_err(|e| {
S3Error::with_message(
S3ErrorCode::InternalError,
format!("parse peer join response from {} failed: {e}", site.endpoint),
)
})?;
if !join_response.initial_sync_error_message.is_empty() {
initial_sync_errors.push(format!("{}: {}", site.endpoint, join_response.initial_sync_error_message));
}
// An explicit no-op join. The peer answered 200 but wrote nothing —
// its persisted state is already newer than the snapshot it was
// sent — so the add is only PARTIALLY configured and saying
// "configured successfully" would be a lie (rustfs/rustfs#5963).
// `None` (a MinIO peer, or one older than the field) is not a
// no-op signal and is deliberately not reported.
if join_response.applied == Some(false) {
initial_sync_errors.push(format!(
"{}: peer did not apply the join (its site replication state is newer than the snapshot it was sent); \
the site is not configured against this peer",
site.endpoint
));
}
state = reconcile_peer_with_actual_identity(state, join_response.peer);
let reconciled_peer = existing_peer_for_endpoint(&state, &site.endpoint).ok_or_else(|| {
S3Error::with_message(
S3ErrorCode::InternalError,
format!("peer join response from {} did not identify the requested site", site.endpoint),
)
})?;
validate_proposed_peer(&reconciled_peer).map_err(|err| {
S3Error::with_message(
S3ErrorCode::InvalidRequest,
format!("invalid peer join response from {}: {err}", site.endpoint),
)
})?;
}
mark_unknown_peer_sync_enabled(&mut state.peers);
// Commit. The state transaction's CAS still fences topology
// writers that do not use bucket admission. By this point
// remote sites may already have accepted their joins, so a
// mismatch asks the operator to re-run add and reconverge.
let next_state = state;
let present = list_store
.list_bucket(&BucketOptions::default())
.await
.map_err(ApiError::from)?
.into_iter()
.map(|bucket| bucket.name)
.collect::<HashSet<_>>();
ensure_add_bucket_set_matches_preflight(&expected_buckets, &present)?;
let (state, edit_generation) = update_site_replication_state(move |state| {
if state.updated_at != expected_updated_at || pending_endpoint_refresh(state).is_some() {
return Err(s3_error!(
InvalidRequest,
"site replication state changed during peer join; the peers may already be joined — re-run replicate add"
));
}
adopt_add_commit_state(state, next_state);
let edit_generation = next_peer_edit_generation(state);
Ok((state.clone(), edit_generation))
})
.await?; .await?;
Ok((
let mut fallback_peer = existing_peer_for_endpoint(&state, &site.endpoint) state,
.unwrap_or_else(|| normalize_peer_site(site.clone(), replicate_ilm_expiry)); edit_generation,
fallback_peer.deployment_id = preflight.deployment_id.clone(); local_peer,
let join_response = parse_peer_join_response(&body, fallback_peer).map_err(|e| { service_account_secret_key,
S3Error::with_message( initial_sync_errors,
S3ErrorCode::InternalError, add_in_progress_guard,
format!("parse peer join response from {} failed: {e}", site.endpoint), ))
) })
})?; .await?;
if !join_response.initial_sync_error_message.is_empty() {
initial_sync_errors.push(format!("{}: {}", site.endpoint, join_response.initial_sync_error_message));
}
// An explicit no-op join. The peer answered 200 but wrote nothing —
// its persisted state is already newer than the snapshot it was
// sent — so the add is only PARTIALLY configured and saying
// "configured successfully" would be a lie (rustfs/rustfs#5963).
// `None` (a MinIO peer, or one older than the field) is not a
// no-op signal and is deliberately not reported.
if join_response.applied == Some(false) {
initial_sync_errors.push(format!(
"{}: peer did not apply the join (its site replication state is newer than the snapshot it was sent); \
the site is not configured against this peer",
site.endpoint
));
}
state = reconcile_peer_with_actual_identity(state, join_response.peer);
let reconciled_peer = existing_peer_for_endpoint(&state, &site.endpoint).ok_or_else(|| {
S3Error::with_message(
S3ErrorCode::InternalError,
format!("peer join response from {} did not identify the requested site", site.endpoint),
)
})?;
validate_proposed_peer(&reconciled_peer).map_err(|err| {
S3Error::with_message(
S3ErrorCode::InvalidRequest,
format!("invalid peer join response from {}: {err}", site.endpoint),
)
})?;
}
mark_unknown_peer_sync_enabled(&mut state.peers);
// Commit. The CAS runs inside the transaction, against the state the
// transaction itself loaded — the peer round trips above took however
// long they took, and only this check can tell whether the topology
// this add was planned against is still the current one. The error
// says so: by this point the remote sites already accepted their
// joins, and re-running the add is what reconverges the local side.
let next_state = state;
let (state, edit_generation) = update_site_replication_state(move |state| {
if state.updated_at != expected_updated_at || pending_endpoint_refresh(state).is_some() {
return Err(s3_error!(
InvalidRequest,
"site replication state changed during peer join; the peers may already be joined — re-run replicate add"
));
}
adopt_add_commit_state(state, next_state);
let edit_generation = next_peer_edit_generation(state);
Ok((state.clone(), edit_generation))
})
.await?;
// The finalize fan-out delivers peer-edit payloads, so it carries the // The finalize fan-out delivers peer-edit payloads, so it carries the
// generation allocated in the commit above: the receiving site orders // generation allocated in the commit above: the receiving site orders
@@ -7185,8 +7309,14 @@ impl Operation for SRPeerEditHandler {
// The fence is self-reported — the shared service account means // The fence is self-reported — the shared service account means
// the sender cannot be identified — so it is honoured only after // the sender cannot be identified — so it is honoured only after
// the admissibility check, against the same state it will gate. // the admissibility check, against the same state it will gate.
let commit_fence = let commit_fence = match commit_fence {
commit_fence.filter(|fence| peer_edit_fence_is_admissible(state, &local_peer.deployment_id, fence)); Some(fence) if peer_edit_fence_is_admissible(state, &local_peer.deployment_id, &fence) => Some(fence),
// A fenced edit can only come from a current remote peer. If
// that origin left while the retry was in flight, applying
// its body here would resurrect the removed topology.
Some(_) => return Ok(StateCommit::Unchanged(PeerEditOutcome::Acked)),
None => None,
};
// Ordering fence: the sending site allocates the generation under // Ordering fence: the sending site allocates the generation under
// its state-object lock, so a delivery that lost the race carries // its state-object lock, so a delivery that lost the race carries
// a generation this site has already passed. Applying it would // a generation this site has already passed. Applying it would
@@ -8886,6 +9016,41 @@ mod tests {
); );
} }
#[test]
fn add_admission_starts_before_preflight_and_rejects_bucket_set_changes() {
let expected = HashSet::from(["remote-owned".to_string(), "shared".to_string()]);
let present = HashSet::from(["shared".to_string()]);
let err = ensure_add_bucket_set_matches_preflight(&expected, &present)
.expect_err("a missing bootstrap bucket must reject the topology commit");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
let present = HashSet::from([
"remote-owned".to_string(),
"shared".to_string(),
"created-during-add".to_string(),
]);
let err = ensure_add_bucket_set_matches_preflight(&expected, &present)
.expect_err("a bucket created during add must reject the topology commit");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
let src = include_str!("site_replication.rs");
let add = src
.split("impl Operation for SiteReplicationAddHandler")
.nth(1)
.and_then(|rest| rest.split("pub struct SiteReplicationRemoveHandler").next())
.expect("add handler block");
let admission = add
.find("with_site_replication_bucket_mutation_admission_lock")
.expect("distributed mutation admission");
let preflight = add.find("add_preflight_infos").expect("bucket preflight");
let validation = add
.find("ensure_add_bucket_set_matches_preflight")
.expect("bucket-set validation");
let commit = add.find("adopt_add_commit_state").expect("topology commit");
assert!(admission < preflight && preflight < validation && validation < commit);
}
#[test] #[test]
fn test_tls_capability_gates_run_before_add_or_edit_state_side_effects() { fn test_tls_capability_gates_run_before_add_or_edit_state_side_effects() {
let src = include_str!("site_replication.rs"); let src = include_str!("site_replication.rs");
@@ -9182,13 +9347,19 @@ mod tests {
); );
// Fence hardening: origin and generation are self-reported by a // Fence hardening: origin and generation are self-reported by a
// caller the shared service account cannot identify, so the handler // caller the shared service account cannot identify, so the handler
// must pass the fence through the admissibility check — against the // must admit the fence against the same state it gates. An origin
// same state the fence gates, i.e. inside the transaction — before // removed while a retry was in flight is acknowledged without
// reading or raising any high-water mark. // applying the stale body; otherwise it could recreate topology.
assert!( assert!(
handler_block.contains(".filter(|fence| peer_edit_fence_is_admissible(state, &local_peer.deployment_id, fence))"), handler_block.contains(
"Some(fence) if peer_edit_fence_is_admissible(state, &local_peer.deployment_id, &fence) => Some(fence)"
),
"SRPeerEditHandler must admit a fence only through peer_edit_fence_is_admissible inside the state transaction" "SRPeerEditHandler must admit a fence only through peer_edit_fence_is_admissible inside the state transaction"
); );
assert!(
handler_block.contains("Some(_) => return Ok(StateCommit::Unchanged(PeerEditOutcome::Acked))"),
"SRPeerEditHandler must not apply a fenced edit after its origin leaves the current topology"
);
// P1-15 PR2: both halves of the fence and the edit they fence share // P1-15 PR2: both halves of the fence and the edit they fence share
// ONE transaction. Checking the fence against a state read outside the // ONE transaction. Checking the fence against a state read outside the
// lock would let the check pass on one snapshot and the write land on // lock would let the check pass on one snapshot and the write land on
@@ -10352,8 +10523,9 @@ mod tests {
/// A fence is self-reported: every site authenticates peer traffic with /// A fence is self-reported: every site authenticates peer traffic with
/// the same site-replicator credential, so a compromised peer can stamp /// the same site-replicator credential, so a compromised peer can stamp
/// ANY origin with ANY generation. An origin the receiver does not /// ANY origin with ANY generation. An origin the receiver does not
/// replicate with — or the receiver itself — is ignored and plants no /// replicate with — or the receiver itself — is inadmissible and plants
/// mark; a mark a compromised peer plants for a CURRENT origin cannot /// no mark; the handler acknowledges such a request without applying its
/// body. A mark a compromised peer plants for a CURRENT origin cannot
/// silence that origin, because the staleness window refuses to fence on /// silence that origin, because the staleness window refuses to fence on
/// a mark implausibly far above the genuine deliveries. /// a mark implausibly far above the genuine deliveries.
#[test] #[test]
@@ -12791,6 +12963,7 @@ mod tests {
last_error: "site replication is not enabled".to_string(), last_error: "site replication is not enabled".to_string(),
updated_at: Some(OffsetDateTime::now_utc()), updated_at: Some(OffsetDateTime::now_utc()),
edit_generation: None, edit_generation: None,
peer_unreachable: false,
deletions_recorded: false, deletions_recorded: false,
}], }],
..Default::default() ..Default::default()
@@ -12989,6 +13162,7 @@ mod tests {
last_error: "peer offline".to_string(), last_error: "peer offline".to_string(),
updated_at: Some(OffsetDateTime::now_utc()), updated_at: Some(OffsetDateTime::now_utc()),
edit_generation: None, edit_generation: None,
peer_unreachable: false,
deletions_recorded: false, deletions_recorded: false,
}], }],
..Default::default() ..Default::default()
+63 -37
View File
@@ -75,7 +75,8 @@ use crate::auth::get_condition_values_with_client_info;
use crate::error::ApiError; use crate::error::ApiError;
use crate::shared_types::RemoteAddr; use crate::shared_types::RemoteAddr;
use crate::site_replication::{ use crate::site_replication::{
site_replication_bucket_meta_hook, site_replication_delete_bucket_hook, site_replication_make_bucket_hook, cancel_site_replication_delete_bucket, commit_site_replication_delete_bucket, prepare_site_replication_delete_bucket,
site_replication_bucket_meta_hook, site_replication_make_bucket_hook, with_site_replication_bucket_mutation_lock,
}; };
use crate::storage::storage_api::lock_bucket_targets_metadata; use crate::storage::storage_api::lock_bucket_targets_metadata;
use http::StatusCode; use http::StatusCode;
@@ -1331,23 +1332,34 @@ impl DefaultBucketUsecase {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
}; };
let make_result = store // Keep the local namespace mutation and its peer hook ordered across
.make_bucket( // every node in this site. Otherwise a delete waiting for repair
&bucket, // coordination can arrive after this create on remote sites.
&MakeBucketOptions { let operation_bucket = bucket.clone();
force_create: false, let operation_store = store.clone();
lock_enabled, let make_result = with_site_replication_bucket_mutation_lock(store, &bucket, move || async move {
..Default::default() let make_result = operation_store
}, .make_bucket(
) &operation_bucket,
.await; &MakeBucketOptions {
force_create: false,
lock_enabled,
..Default::default()
},
)
.await;
if make_result.is_ok() {
crate::storage::invalidate_bucket_validation_cache(&operation_bucket);
if let Err(err) = site_replication_make_bucket_hook(&operation_bucket, lock_enabled).await {
warn!(bucket = %operation_bucket, error = ?err, "site replication make bucket hook failed");
}
}
make_result
})
.await?;
match make_result { match make_result {
Ok(()) => { Ok(()) => {}
// Invalidate the bucket validation cache so subsequent GETs
// see the newly created bucket immediately.
crate::storage::invalidate_bucket_validation_cache(&bucket);
}
Err(StorageError::BucketExists(_)) => { Err(StorageError::BucketExists(_)) => {
// Per S3 spec: bucket namespace is global. Owner recreating returns 200 OK; // Per S3 spec: bucket namespace is global. Owner recreating returns 200 OK;
// non-owner gets 409 BucketAlreadyExists. // non-owner gets 409 BucketAlreadyExists.
@@ -1358,10 +1370,6 @@ impl DefaultBucketUsecase {
Err(e) => return Err(ApiError::from(e).into()), Err(e) => return Err(ApiError::from(e).into()),
} }
if let Err(err) = site_replication_make_bucket_hook(&bucket, lock_enabled).await {
warn!(bucket = %bucket, error = ?err, "site replication make bucket hook failed");
}
let output = CreateBucketOutput::default(); let output = CreateBucketOutput::default();
counter!("rustfs_create_bucket_total").increment(1); counter!("rustfs_create_bucket_total").increment(1);
let result = Ok(S3Response::new(output)); let result = Ok(S3Response::new(output));
@@ -1397,16 +1405,41 @@ impl DefaultBucketUsecase {
authorize_request(&mut req, Action::S3Action(S3Action::ForceDeleteBucketAction)).await?; authorize_request(&mut req, Action::S3Action(S3Action::ForceDeleteBucketAction)).await?;
} }
store // Keep the local namespace mutation and its peer hook ordered across
.delete_bucket( // every node in this site so an older delete cannot overtake a new
&input.bucket, // same-name make while it waits for repair coordination.
&DeleteBucketOptions { let operation_bucket = input.bucket.clone();
force, let operation_store = store.clone();
..Default::default() with_site_replication_bucket_mutation_lock(store, &input.bucket, move || async move {
}, let intent = prepare_site_replication_delete_bucket(&operation_bucket, force).await?;
) let delete_result = operation_store
.await .delete_bucket(
.map_err(ApiError::from)?; &operation_bucket,
&DeleteBucketOptions {
force,
..Default::default()
},
)
.await;
match delete_result {
Ok(()) => {
crate::storage::invalidate_bucket_validation_cache(&operation_bucket);
if let Some(intent) = intent
&& let Err(err) = commit_site_replication_delete_bucket(&intent).await
{
warn!(bucket = %operation_bucket, error = ?err, "site replication delete bucket hook failed");
}
Ok::<(), S3Error>(())
}
Err(err) => {
if let Some(intent) = intent {
cancel_site_replication_delete_bucket(intent).await;
}
Err(S3Error::from(ApiError::from(err)))
}
}
})
.await??;
// Drop every cached object body for the now-deleted bucket so dead // Drop every cached object body for the now-deleted bucket so dead
// bytes do not sit resident until TTL. Covers both the normal and the // bytes do not sit resident until TTL. Covers both the normal and the
@@ -1415,16 +1448,9 @@ impl DefaultBucketUsecase {
let cache_adapter = current_object_data_cache_for_context(self.context.as_deref()); let cache_adapter = current_object_data_cache_for_context(self.context.as_deref());
let _ = invalidate_object_data_cache_bucket_after_delete(&cache_adapter, &input.bucket).await; let _ = invalidate_object_data_cache_bucket_after_delete(&cache_adapter, &input.bucket).await;
// Invalidate bucket validation cache
crate::storage::invalidate_bucket_validation_cache(&input.bucket);
// Re-evaluate lifecycle and replication after bucket removal. // Re-evaluate lifecycle and replication after bucket removal.
rustfs_scanner::record_scanner_maintenance_change(&input.bucket); rustfs_scanner::record_scanner_maintenance_change(&input.bucket);
if let Err(err) = site_replication_delete_bucket_hook(&input.bucket, force).await {
warn!(bucket = %input.bucket, error = ?err, "site replication delete bucket hook failed");
}
// Notify peers to drop their cached metadata for the now-deleted bucket. // Notify peers to drop their cached metadata for the now-deleted bucket.
let request_context = req.extensions.get::<request_context::RequestContext>().cloned(); let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
notify_bucket_metadata_delete(input.bucket.clone(), request_context); notify_bucket_metadata_delete(input.bucket.clone(), request_context);
+47
View File
@@ -208,6 +208,7 @@ const EVENT_PEER_ADDR_UNAVAILABLE: &str = "peer_addr_unavailable";
const EVENT_RPC_SIGNATURE_VERIFICATION_FAILED: &str = "rpc_signature_verification_failed"; const EVENT_RPC_SIGNATURE_VERIFICATION_FAILED: &str = "rpc_signature_verification_failed";
const EVENT_GRPC_TRACE_CONTEXT_PROPAGATION_FAILED: &str = "grpc_trace_context_propagation_failed"; const EVENT_GRPC_TRACE_CONTEXT_PROPAGATION_FAILED: &str = "grpc_trace_context_propagation_failed";
const HEAL_CONTROL_TONIC_RPC_PATH: &str = "/node_service.HealControlService/HealControl"; const HEAL_CONTROL_TONIC_RPC_PATH: &str = "/node_service.HealControlService/HealControl";
const SCANNER_SCOPED_DIRTY_USAGE_ACK_TONIC_RPC_PATH: &str = "/node_service.ScannerControlService/ScannerScopedDirtyUsageAck";
const TIER_MUTATION_PREPARE_TONIC_RPC_PATH: &str = "/node_service.TierMutationControlService/PrepareTierMutation"; const TIER_MUTATION_PREPARE_TONIC_RPC_PATH: &str = "/node_service.TierMutationControlService/PrepareTierMutation";
const TIER_MUTATION_COMMIT_TONIC_RPC_PATH: &str = "/node_service.TierMutationControlService/CommitTierMutation"; const TIER_MUTATION_COMMIT_TONIC_RPC_PATH: &str = "/node_service.TierMutationControlService/CommitTierMutation";
const TIER_MUTATION_ABORT_TONIC_RPC_PATH: &str = "/node_service.TierMutationControlService/AbortTierMutation"; const TIER_MUTATION_ABORT_TONIC_RPC_PATH: &str = "/node_service.TierMutationControlService/AbortTierMutation";
@@ -1856,6 +1857,7 @@ fn process_connection(
); );
let rpc_service = RpcRequestPathService::new( let rpc_service = RpcRequestPathService::new(
Routes::new(node_service) Routes::new(node_service)
.add_service(InterceptedService::new(storage::tonic_service::make_scanner_control_server(), check_auth))
.add_service(heal_control_service) .add_service(heal_control_service)
.add_service(tier_mutation_control_service) .add_service(tier_mutation_control_service)
.prepare(), .prepare(),
@@ -2259,6 +2261,7 @@ fn check_auth(req: Request<()>) -> std::result::Result<Request<()>, Status> {
.strip_prefix(TONIC_RPC_PREFIX) .strip_prefix(TONIC_RPC_PREFIX)
.and_then(|suffix| suffix.strip_prefix('/')) .and_then(|suffix| suffix.strip_prefix('/'))
.or_else(|| (target.uri.path() == HEAL_CONTROL_TONIC_RPC_PATH).then_some("HealControl")) .or_else(|| (target.uri.path() == HEAL_CONTROL_TONIC_RPC_PATH).then_some("HealControl"))
.or_else(|| (target.uri.path() == SCANNER_SCOPED_DIRTY_USAGE_ACK_TONIC_RPC_PATH).then_some("ScannerScopedDirtyUsageAck"))
.or_else(|| (target.uri.path() == TIER_MUTATION_PREPARE_TONIC_RPC_PATH).then_some("PrepareTierMutation")) .or_else(|| (target.uri.path() == TIER_MUTATION_PREPARE_TONIC_RPC_PATH).then_some("PrepareTierMutation"))
.or_else(|| (target.uri.path() == TIER_MUTATION_COMMIT_TONIC_RPC_PATH).then_some("CommitTierMutation")) .or_else(|| (target.uri.path() == TIER_MUTATION_COMMIT_TONIC_RPC_PATH).then_some("CommitTierMutation"))
.or_else(|| (target.uri.path() == TIER_MUTATION_ABORT_TONIC_RPC_PATH).then_some("AbortTierMutation")) .or_else(|| (target.uri.path() == TIER_MUTATION_ABORT_TONIC_RPC_PATH).then_some("AbortTierMutation"))
@@ -3427,6 +3430,50 @@ mod tests {
rustfs_common::set_global_local_node_name(&previous_node_name).await; rustfs_common::set_global_local_node_name(&previous_node_name).await;
} }
#[tokio::test]
#[serial_test::serial]
async fn scoped_dirty_usage_peer_probe_reaches_handler_through_production_auth() {
let _ = rustfs_credentials::set_global_rpc_secret("rpc-http-test-secret".to_string());
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind scoped ACK auth test");
let addr = listener.local_addr().expect("listener address");
let previous_node_name = rustfs_common::get_global_local_node_name().await;
rustfs_common::set_global_local_node_name(&addr.to_string()).await;
let node = InterceptedService::new(NodeServiceServer::new(make_server()), check_auth);
let scanner = InterceptedService::new(storage::tonic_service::make_scanner_control_server(), check_auth);
let service = RpcRequestPathService::new(Routes::new(node).add_service(scanner).prepare());
let server = tokio::spawn(async move {
let (socket, _) = listener.accept().await.expect("accept test connection");
ConnBuilder::new(TokioExecutor::new())
.serve_connection(TokioIo::new(socket), TowerToHyperService::new(service))
.await
.expect("serve scoped ACK auth test");
});
let host = rustfs_utils::XHost::try_from(addr.to_string()).expect("peer address");
let client = storage::PeerRestClient::new(host, format!("http://{addr}"));
let result = client
.scanner_scoped_dirty_usage_capability(
"11111111-1111-1111-1111-111111111111".to_string(),
"a".repeat(32),
vec![rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry {
bucket: "photos".into(),
bucket_incarnation: vec![1; 16].into(),
generation: 8,
}],
)
.await;
client.evict_connection().await;
server.abort();
let _ = server.await;
rustfs_common::set_global_local_node_name(&previous_node_name).await;
let error = result
.expect_err("probe must fail closed without the requested storage owner")
.to_string();
assert!(
error.contains("storage layer is not initialized") || error.contains("scoped dirty usage peer or process changed"),
"signed probe must pass production path authentication and reach owner validation: {error}"
);
}
#[tokio::test] #[tokio::test]
#[serial_test::serial] #[serial_test::serial]
async fn peer_rest_heal_control_uses_production_auth_and_keeps_validation_errors_online() { async fn peer_rest_heal_control_uses_production_auth_and_keeps_validation_errors_online() {
+427 -35
View File
@@ -22,6 +22,57 @@ pub(crate) const SITE_REPLICATION_BUCKET_OP_CONFIGURE_REPLICATION: &str = "confi
pub(crate) static SITE_REPLICATION_BUCKET_OP_LOCK: LazyLock<RwLock<()>> = LazyLock::new(|| RwLock::new(())); pub(crate) static SITE_REPLICATION_BUCKET_OP_LOCK: LazyLock<RwLock<()>> = LazyLock::new(|| RwLock::new(()));
const SITE_REPLICATION_BUCKET_MUTATION_LOCK_PREFIX: &str = "config/site-replication/bucket-mutation";
pub(crate) const SITE_REPLICATION_BUCKET_MUTATION_ADMISSION_LOCK_PATH: &str =
"config/site-replication/bucket-mutation-admission.lock";
pub(crate) fn site_replication_bucket_mutation_lock_path(bucket: &str) -> String {
format!("{SITE_REPLICATION_BUCKET_MUTATION_LOCK_PREFIX}/{bucket}.lock")
}
pub(crate) async fn with_site_replication_bucket_mutation_lock<F, Fut, T>(
store: Arc<ECStore>,
bucket: &str,
operation: F,
) -> S3Result<T>
where
F: FnOnce() -> Fut + Send + 'static,
Fut: std::future::Future<Output = T> + Send + 'static,
T: Send + 'static,
{
let mutation_store = store.clone();
let mutation_path = site_replication_bucket_mutation_lock_path(bucket);
with_config_object_read_lock(
store,
SITE_REPLICATION_BUCKET_MUTATION_ADMISSION_LOCK_PATH.to_string(),
move || async move {
with_config_object_write_lock(mutation_store, mutation_path, operation)
.await
.map_err(|err| S3Error::from(ApiError::from(err)))
},
)
.await
.map_err(|err| S3Error::from(ApiError::from(err)))?
}
/// Exclude every local bucket namespace mutation from an add's local preflight
/// snapshot until its topology commit. Peer bootstrap callbacks do not enter
/// this public-mutation admission path, so they can finish while the writer is
/// held; post-commit fan-out and backfill must run after it is released.
pub(crate) async fn with_site_replication_bucket_mutation_admission_lock<F, Fut, T>(
store: Arc<ECStore>,
operation: F,
) -> S3Result<T>
where
F: FnOnce() -> Fut + Send + 'static,
Fut: std::future::Future<Output = S3Result<T>> + Send + 'static,
T: Send + 'static,
{
with_config_object_write_lock(store, SITE_REPLICATION_BUCKET_MUTATION_ADMISSION_LOCK_PATH.to_string(), operation)
.await
.map_err(|err| S3Error::from(ApiError::from(err)))?
}
#[derive(Debug, Default)] #[derive(Debug, Default)]
pub(crate) struct SiteReplicationBootstrapPlan { pub(crate) struct SiteReplicationBootstrapPlan {
pub(crate) iam_items: Vec<SRIAMItem>, pub(crate) iam_items: Vec<SRIAMItem>,
@@ -329,6 +380,91 @@ pub(crate) fn site_replication_bootstrap_plan(info: &SRInfo) -> S3Result<SiteRep
Ok(plan) Ok(plan)
} }
/// Build only the two bucket operations needed by the lightweight retry
/// drain. The full bootstrap plan scans every bucket and IAM record; doing
/// that on a 30-second recovery cadence would make lifecycle admission scale
/// with the whole site instead of the one queued bucket.
pub(crate) fn site_replication_bucket_retry_plan_for(
bucket: &SRBucketInfo,
replicate_ilm_expiry: bool,
) -> S3Result<SiteReplicationBootstrapPlan> {
let mut plan = SiteReplicationBootstrapPlan {
bucket_make_ops: vec![bootstrap_bucket_make_op_path(bucket)],
bucket_configure_ops: vec![bootstrap_bucket_op_path(
&bucket.bucket,
SITE_REPLICATION_BUCKET_OP_CONFIGURE_REPLICATION,
)],
..Default::default()
};
append_bootstrap_bucket_items(&mut plan, bucket, replicate_ilm_expiry)?;
Ok(plan)
}
pub(crate) fn site_replication_bucket_retry_plan_from_info(
bucket: &SRBucketInfo,
replicate_ilm_expiry: bool,
) -> S3Result<SiteReplicationBootstrapPlan> {
let mut plan = site_replication_bucket_retry_plan_for(bucket, replicate_ilm_expiry)?;
// Omit only metadata the make/configure operations can reproduce exactly.
// Non-default versioning fields and operator-authored replication rules
// remain in the plan; their extra request cost intentionally defers the
// event to the complete drain when the lightweight budget is too small.
plan.bucket_items.retain(|item| !retry_bucket_metadata_is_redundant(item));
Ok(plan)
}
fn retry_bucket_metadata_is_redundant(item: &SRBucketMeta) -> bool {
match item.r#type.as_str() {
"version-config" => item.versioning.as_deref().is_some_and(|raw| {
deserialize::<VersioningConfiguration>(&decode_bucket_meta_wire_value(raw)).is_ok_and(|config| {
config
== VersioningConfiguration {
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
..Default::default()
}
})
}),
"replication-config" => item.replication_config.as_deref().is_some_and(|raw| {
deserialize::<ReplicationConfiguration>(&decode_bucket_meta_wire_value(raw))
.is_ok_and(|config| config.role.trim().is_empty() && config.rules.iter().all(is_derived_site_replication_rule))
}),
// `Some("")` is the in-memory sentinel used when the bucket is lock
// enabled but has no object-lock configuration body. The make query
// carries lockEnabled=true; sending an empty metadata body is neither
// useful nor parseable.
"object-lock-config" => item.object_lock_config.as_deref() == Some(""),
_ => false,
}
}
pub(crate) async fn site_replication_bucket_retry_plan(
bucket: &str,
replicate_ilm_expiry: bool,
) -> S3Result<SiteReplicationBootstrapPlan> {
let Some(store) = current_object_store_handle() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
let bucket_info = match store.get_bucket_info(bucket, &BucketOptions::default()).await {
Ok(bucket_info) => bucket_info,
Err(err) if is_err_bucket_not_found(&err) => return Ok(SiteReplicationBootstrapPlan::default()),
Err(err) => return Err(ApiError::from(err).into()),
};
let lock_enabled = bucket_info.object_locking;
let metadata = metadata_sys::get(bucket).await.map_err(ApiError::from)?;
let mut bucket_info = SRBucketInfo {
bucket: bucket.to_string(),
created_at: bucket_info.created,
location: current_region().map(|region| region.to_string()).unwrap_or_default(),
api_version: Some(SITE_REPL_API_VERSION.to_string()),
..Default::default()
};
populate_sr_bucket_info_from_metadata(&mut bucket_info, &metadata).await;
if lock_enabled && bucket_info.object_lock_config.is_none() {
bucket_info.object_lock_config = Some(String::new());
}
site_replication_bucket_retry_plan_from_info(&bucket_info, replicate_ilm_expiry)
}
pub async fn site_replication_make_bucket_hook(bucket: &str, lock_enabled: bool) -> S3Result<()> { pub async fn site_replication_make_bucket_hook(bucket: &str, lock_enabled: bool) -> S3Result<()> {
let _bucket_op_guard = SITE_REPLICATION_BUCKET_OP_LOCK.read().await; let _bucket_op_guard = SITE_REPLICATION_BUCKET_OP_LOCK.read().await;
let runtime = { let runtime = {
@@ -393,20 +529,273 @@ pub(crate) async fn broadcast_site_replication_make_bucket(
broadcast_site_replication_json_using_runtime(runtime, &configure_path, &serde_json::json!({})).await broadcast_site_replication_json_using_runtime(runtime, &configure_path, &serde_json::json!({})).await
} }
pub async fn site_replication_delete_bucket_hook(bucket: &str, force_delete: bool) -> S3Result<()> { const SITE_REPLICATION_DELETE_INTENT_PENDING: &str =
"bucket deletion reserved; local completion and peer delivery are not yet known";
#[derive(Clone)]
struct SiteReplicationDeleteBucketReservation {
peer: PeerInfo,
previous: Option<SiteReplicationRetryEvent>,
observed: SiteReplicationRetryEvent,
}
pub(crate) struct SiteReplicationDeleteBucketIntent {
path: String,
reservations: Vec<SiteReplicationDeleteBucketReservation>,
displaced: Vec<SiteReplicationRetryEvent>,
}
fn site_replication_delete_bucket_path(bucket: &str, force_delete: bool) -> String {
let operation = if force_delete { let operation = if force_delete {
"force-delete-bucket" "force-delete-bucket"
} else { } else {
"delete-bucket" "delete-bucket"
}; };
let path = format!( format!(
"/rustfs/admin/v3/site-replication/peer/bucket-ops?{}", "/rustfs/admin/v3/site-replication/peer/bucket-ops?{}",
form_urlencoded::Serializer::new(String::new()) form_urlencoded::Serializer::new(String::new())
.append_pair("bucket", bucket) .append_pair("bucket", bucket)
.append_pair("operation", operation) .append_pair("operation", operation)
.finish() .finish()
); )
broadcast_site_replication_json(&path, &serde_json::json!({})).await }
/// Reserve every destructive peer delivery before the local namespace is
/// changed. The state transaction either persists the complete set or writes
/// nothing, so a full/unreadable queue fails the S3 delete closed.
pub(crate) async fn prepare_site_replication_delete_bucket(
bucket: &str,
force_delete: bool,
) -> S3Result<Option<SiteReplicationDeleteBucketIntent>> {
let path = site_replication_delete_bucket_path(bucket, force_delete);
let reservation_path = path.clone();
update_site_replication_state_when_changed(move |state| {
if !state.enabled() {
return Ok(StateCommit::Unchanged(None));
}
let local_peer = current_local_runtime_peer(state);
let peers = state
.peers
.values()
.filter(|peer| {
peer.deployment_id != local_peer.deployment_id && !same_identity_endpoint(&peer.endpoint, &local_peer.endpoint)
})
.cloned()
.collect::<Vec<_>>();
if peers.is_empty() {
return Ok(StateCommit::Unchanged(None));
}
let mut reservations = Vec::with_capacity(peers.len());
let mut displaced = Vec::new();
for peer in peers {
let previous = state
.retry_queue
.iter()
.find(|event| retry_event_matches(event, &peer, &reservation_path))
.cloned();
displaced.extend(upsert_site_replication_retry_event(
&mut state.retry_queue,
&peer,
&reservation_path,
SITE_REPLICATION_DELETE_INTENT_PENDING,
None,
)?);
let observed = state
.retry_queue
.iter()
.find(|event| retry_event_matches(event, &peer, &reservation_path))
.cloned()
.ok_or_else(|| {
S3Error::with_message(
S3ErrorCode::InternalError,
"site replication delete reservation disappeared before commit".to_string(),
)
})?;
reservations.push(SiteReplicationDeleteBucketReservation {
peer,
previous,
observed,
});
}
Ok(StateCommit::Changed(Some(SiteReplicationDeleteBucketIntent {
path: reservation_path,
reservations,
displaced,
})))
})
.await
}
/// Roll back a reservation when the local storage delete definitively failed.
/// A concurrently revised reservation is preserved; it belongs to a newer
/// observation and this operation has no authority to settle it.
pub(crate) async fn cancel_site_replication_delete_bucket(intent: SiteReplicationDeleteBucketIntent) {
let path = intent.path.clone();
let result = update_site_replication_state_when_changed(move |state| {
let mut changed = false;
for reservation in intent.reservations {
let Some(index) = state.retry_queue.iter().position(|event| {
retry_event_matches(event, &reservation.peer, &reservation.observed.path)
&& event.id == reservation.observed.id
&& event.updated_at == reservation.observed.updated_at
}) else {
continue;
};
if let Some(previous) = reservation.previous {
state.retry_queue[index] = previous;
} else {
state.retry_queue.remove(index);
}
changed = true;
}
let mut restored_all = true;
for displaced in intent.displaced {
let duplicate = state.retry_queue.iter().any(|event| {
event.id == displaced.id
|| (event.peer_deployment_id == displaced.peer_deployment_id && event.path == displaced.path)
});
if duplicate {
continue;
}
if state.retry_queue.len() >= SITE_REPLICATION_RETRY_QUEUE_LIMIT {
restored_all = false;
continue;
}
state.retry_queue.push(displaced);
changed = true;
}
Ok(if changed {
StateCommit::Changed(restored_all)
} else {
StateCommit::Unchanged(restored_all)
})
})
.await;
match result {
Ok(true) => {}
Ok(false) => warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
path,
result = "delete_intent_cancel_incomplete",
"admin site replication state"
),
Err(err) => warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
path,
result = "delete_intent_cancel_failed",
error = ?err,
"admin site replication state"
),
}
}
async fn broadcast_site_replication_delete_bucket(intent: &SiteReplicationDeleteBucketIntent) -> S3Result<()> {
let sends = intent.reservations.iter().cloned().map(|reservation| {
let request_path = intent.path.clone();
async move {
let fallback_peer = reservation.peer.clone();
let observed = reservation.observed.clone();
let delivery_path = request_path.clone();
let delivery = with_site_replication_state_read_lock(move |state| async move {
let Some(current_peer) = state.peers.get(&fallback_peer.deployment_id).cloned() else {
return Ok(None);
};
let service_account_secret_key =
match site_replicator_service_account_secret(&state.service_account_access_key).await {
Ok(secret) => secret,
Err(err) => {
let Some(secret) = legacy_site_replicator_state_secret(&state) else {
return Err(err);
};
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_SITE_REPLICATION,
result = "legacy_state_service_account_secret_fallback",
error = ?err,
"admin site replication state"
);
secret
}
};
let result = async {
let transport = PeerTransport::for_runtime_peer(&current_peer).await?;
PeerAdminRequest::put(&transport.connection, &delivery_path, &state.service_account_access_key)
.with_client(&transport.client)
.send(&service_account_secret_key, &serde_json::json!({}))
.await
}
.await;
Ok(Some((current_peer, result)))
})
.await;
match delivery {
Ok(Some((current_peer, Ok(_)))) => {
dequeue_observed_site_replication_retry_event(&current_peer, &observed).await;
None
}
Ok(Some((current_peer, Err(err)))) => {
// Keep the failed deletion operator-visible, but never
// replay it automatically: without a bucket-incarnation
// fence, a delayed delete could erase a recreated bucket.
enqueue_site_replication_retry_event(&current_peer, &request_path, &err).await;
Some(err)
}
Ok(None) => {
dequeue_observed_site_replication_retry_event(&reservation.peer, &observed).await;
None
}
Err(err) => {
enqueue_site_replication_retry_event(&reservation.peer, &request_path, &err).await;
Some(err)
}
}
}
});
futures::future::join_all(sends)
.await
.into_iter()
.flatten()
.next()
.map_or(Ok(()), Err)
}
pub(crate) async fn commit_site_replication_delete_bucket(intent: &SiteReplicationDeleteBucketIntent) -> S3Result<()> {
let _bucket_op_guard = SITE_REPLICATION_BUCKET_OP_LOCK.read().await;
let store =
current_object_store_handle().ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()))?;
let retry_peers = intent
.reservations
.iter()
.map(|reservation| reservation.peer.clone())
.collect::<Vec<_>>();
let retry_path = intent.path.clone();
let delivery_intent = SiteReplicationDeleteBucketIntent {
path: intent.path.clone(),
reservations: intent.reservations.clone(),
displaced: Vec::new(),
};
match with_config_object_write_lock(store, SITE_REPLICATION_REPAIR_EXECUTION_LOCK_PATH.to_string(), move || async move {
broadcast_site_replication_delete_bucket(&delivery_intent).await
})
.await
{
Ok(result) => result,
Err(err) => {
let err: S3Error = ApiError::from(err).into();
for peer in &retry_peers {
enqueue_site_replication_retry_event(peer, &retry_path, &err).await;
}
Err(err)
}
}
} }
pub async fn site_replication_bucket_meta_hook(mut item: SRBucketMeta) -> S3Result<()> { pub async fn site_replication_bucket_meta_hook(mut item: SRBucketMeta) -> S3Result<()> {
@@ -515,6 +904,39 @@ pub(crate) fn maybe_time(value: OffsetDateTime) -> Option<OffsetDateTime> {
(value != OffsetDateTime::UNIX_EPOCH).then_some(value) (value != OffsetDateTime::UNIX_EPOCH).then_some(value)
} }
async fn populate_sr_bucket_info_from_metadata(entry: &mut SRBucketInfo, metadata: &BucketMetadata) {
entry.policy = raw_config_to_string(&metadata.policy_config_json).and_then(|raw| serde_json::from_str(&raw).ok());
entry.versioning = raw_config_to_base64(&metadata.versioning_config_xml);
entry.tags = raw_config_to_base64(&metadata.tagging_config_xml);
entry.object_lock_config = raw_config_to_base64(&metadata.object_lock_config_xml);
entry.sse_config = raw_config_to_base64(&metadata.encryption_config_xml);
entry.replication_config = raw_config_to_base64(&metadata.replication_config_xml);
entry.quota_config = raw_config_to_base64(&metadata.quota_config_json);
// Expiry subset only: this entry feeds both the bootstrap/repair plan
// (peers must not receive transition rules) and cross-site consistency
// views (transition rules are site-local and would read as false
// mismatches). A deleted expiry state is a `None` value with the
// deletion's axis so repair can converge peers that missed the live
// delete.
let expiry_statement = lifecycle_expiry_statement(metadata);
entry.expiry_lc_config = expiry_statement.as_ref().and_then(|(subset, _)| subset.clone());
entry.cors_config = raw_config_to_base64(&metadata.cors_config_xml);
entry.policy_updated_at = maybe_time(metadata.policy_config_updated_at);
entry.tag_config_updated_at = maybe_time(metadata.tagging_config_updated_at);
entry.object_lock_config_updated_at = maybe_time(metadata.object_lock_config_updated_at);
entry.sse_config_updated_at = maybe_time(metadata.encryption_config_updated_at);
entry.versioning_config_updated_at = maybe_time(metadata.versioning_config_updated_at);
entry.replication_config_updated_at = maybe_time(metadata.replication_config_updated_at);
entry.quota_config_updated_at = maybe_time(metadata.quota_config_updated_at);
// The expiry axis, not the whole-config write time: local transition-only
// edits inflate the latter, and a repair item stamped with it could
// out-rank a newer real expiry edit on a third site.
entry.expiry_lc_config_updated_at = expiry_statement.map(|(_, axis)| axis);
entry.cors_config_updated_at = maybe_time(metadata.cors_config_updated_at);
entry.replication_targets_online =
Some(site_replication_targets_online(&entry.bucket, &metadata.replication_config_xml).await);
}
pub(crate) async fn build_sr_info(state: &SiteReplicationState, local_peer: &PeerInfo) -> S3Result<SRInfo> { pub(crate) async fn build_sr_info(state: &SiteReplicationState, local_peer: &PeerInfo) -> S3Result<SRInfo> {
let Some(store) = current_object_store_handle() else { let Some(store) = current_object_store_handle() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
@@ -546,37 +968,7 @@ pub(crate) async fn build_sr_info(state: &SiteReplicationState, local_peer: &Pee
}; };
if let Some(metadata) = metadata { if let Some(metadata) = metadata {
entry.policy = raw_config_to_string(&metadata.policy_config_json).and_then(|raw| serde_json::from_str(&raw).ok()); populate_sr_bucket_info_from_metadata(&mut entry, &metadata).await;
entry.versioning = raw_config_to_base64(&metadata.versioning_config_xml);
entry.tags = raw_config_to_base64(&metadata.tagging_config_xml);
entry.object_lock_config = raw_config_to_base64(&metadata.object_lock_config_xml);
entry.sse_config = raw_config_to_base64(&metadata.encryption_config_xml);
entry.replication_config = raw_config_to_base64(&metadata.replication_config_xml);
entry.quota_config = raw_config_to_base64(&metadata.quota_config_json);
// Expiry subset only: this entry feeds both the bootstrap/repair
// plan (peers must not receive transition rules) and cross-site
// consistency views (transition rules are site-local and would
// read as false mismatches). A deleted expiry state is a `None`
// value with the deletion's axis so repair can converge peers
// that missed the live delete.
let expiry_statement = lifecycle_expiry_statement(&metadata);
entry.expiry_lc_config = expiry_statement.as_ref().and_then(|(subset, _)| subset.clone());
entry.cors_config = raw_config_to_base64(&metadata.cors_config_xml);
entry.policy_updated_at = maybe_time(metadata.policy_config_updated_at);
entry.tag_config_updated_at = maybe_time(metadata.tagging_config_updated_at);
entry.object_lock_config_updated_at = maybe_time(metadata.object_lock_config_updated_at);
entry.sse_config_updated_at = maybe_time(metadata.encryption_config_updated_at);
entry.versioning_config_updated_at = maybe_time(metadata.versioning_config_updated_at);
entry.replication_config_updated_at = maybe_time(metadata.replication_config_updated_at);
entry.quota_config_updated_at = maybe_time(metadata.quota_config_updated_at);
// The expiry axis, not the whole-config write time: local
// transition-only edits inflate the latter, and a repair item
// stamped with it could out-rank a newer real expiry edit on a
// third site.
entry.expiry_lc_config_updated_at = expiry_statement.map(|(_, axis)| axis);
entry.cors_config_updated_at = maybe_time(metadata.cors_config_updated_at);
entry.replication_targets_online =
Some(site_replication_targets_online(&bucket.name, &metadata.replication_config_xml).await);
} }
info.buckets.insert(bucket.name, entry); info.buckets.insert(bucket.name, entry);
+7 -6
View File
@@ -47,6 +47,7 @@ use self::identity::{
canonical_endpoint, deployment_id_for_endpoint, mark_unknown_peer_sync_enabled, normalize_peer_map_by_identity_with, canonical_endpoint, deployment_id_for_endpoint, mark_unknown_peer_sync_enabled, normalize_peer_map_by_identity_with,
same_identity_endpoint, same_identity_endpoint,
}; };
pub(crate) use self::state_lock::with_site_replication_state_read_lock;
use self::state_lock::{SITE_REPLICATION_STATE_PATH, with_site_replication_state_lock}; use self::state_lock::{SITE_REPLICATION_STATE_PATH, with_site_replication_state_lock};
use crate::auth::constant_time_eq; use crate::auth::constant_time_eq;
use crate::config::get_config_snapshot; use crate::config::get_config_snapshot;
@@ -64,12 +65,12 @@ use crate::storage_api::site_replication::s3::{
#[cfg(test)] #[cfg(test)]
use crate::storage_api::site_replication::save_config as save_admin_config; use crate::storage_api::site_replication::save_config as save_admin_config;
use crate::storage_api::site_replication::{ use crate::storage_api::site_replication::{
ARN, BUCKET_REPLICATION_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, BucketOperations, BucketOptions, BucketTarget, ARN, BUCKET_REPLICATION_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, BucketMetadata, BucketOperations,
BucketTargetSys, BucketTargetType, BucketTargets, Credentials, ECStore, OperatorRuleContract, StorageError, BucketOptions, BucketTarget, BucketTargetSys, BucketTargetType, BucketTargets, Credentials, ECStore, OperatorRuleContract,
VersioningApi as _, assign_site_replication_rule_priorities, delete_config_no_lock, deserialize, is_site_replication_role, StorageError, VersioningApi as _, assign_site_replication_rule_priorities, delete_config_no_lock, deserialize,
lock_bucket_targets_metadata, metadata_sys, read_config as read_admin_config, read_config_no_lock, is_err_bucket_not_found, is_site_replication_role, lock_bucket_targets_metadata, metadata_sys,
replication_target_arn_deployment_id, save_config_no_lock, serialize, site_replication_rule_deployment_id, read_config as read_admin_config, read_config_no_lock, replication_target_arn_deployment_id, save_config_no_lock, serialize,
with_config_object_read_lock, with_config_object_write_lock, site_replication_rule_deployment_id, with_config_object_read_lock, with_config_object_write_lock,
}; };
use base64_simd::STANDARD as BASE64_STANDARD; use base64_simd::STANDARD as BASE64_STANDARD;
use base64_simd::URL_SAFE_NO_PAD; use base64_simd::URL_SAFE_NO_PAD;
+3 -1
View File
@@ -649,7 +649,9 @@ pub(crate) async fn persist_site_replication_repair_task(
let path = path.to_string(); let path = path.to_string();
update_site_replication_state(move |state| { update_site_replication_state(move |state| {
match failure.as_deref() { match failure.as_deref() {
Some(error) => upsert_site_replication_retry_event(&mut state.retry_queue, &peer, &path, error, None), Some(error) => {
upsert_site_replication_retry_event(&mut state.retry_queue, &peer, &path, error, None)?;
}
None => { None => {
dequeue_site_replication_retry_events_including_escalated(&mut state.retry_queue, &peer, &path); dequeue_site_replication_retry_events_including_escalated(&mut state.retry_queue, &peer, &path);
// A repair is the operator's accountability transfer for the // A repair is the operator's accountability transfer for the
File diff suppressed because it is too large Load Diff
+31 -6
View File
@@ -28,14 +28,18 @@
//! process-local lock must never be reintroduced in front of it as if it //! process-local lock must never be reintroduced in front of it as if it
//! added protection. All IO inside the closure must use the `*_no_lock` //! added protection. All IO inside the closure must use the `*_no_lock`
//! config helpers — the locked variants would self-deadlock on the same //! config helpers — the locked variants would self-deadlock on the same
//! object lock. Do not perform peer network calls or take other config locks //! object lock. Write-lock closures must not perform peer network calls or
//! inside the closure. //! take other config locks. A read-lock closure may carry bounded peer
//! delivery only when the receiver cannot write this state. Peer-edit
//! delivery must run after the read lock is released.
//! //!
//! Lock order: lifecycle -> bucket operation -> repair admission //! Lock order: lifecycle -> bucket-mutation admission -> per-bucket mutation
//! -> state object lock -> per-bucket metadata. //! -> bucket operation -> repair admission -> state object lock ->
//! per-bucket metadata. A path may skip levels, but must not acquire an
//! earlier level while holding a later one.
use super::{S3Error, S3ErrorCode, S3Result}; use super::{S3Error, S3ErrorCode, S3Result, SiteReplicationState, load_site_replication_state_no_lock};
use crate::storage_api::site_replication::{ECStore, with_config_object_write_lock}; use crate::storage_api::site_replication::{ECStore, with_config_object_read_lock, with_config_object_write_lock};
use std::sync::Arc; use std::sync::Arc;
use crate::runtime_sources::current_object_store_handle; use crate::runtime_sources::current_object_store_handle;
@@ -57,6 +61,27 @@ where
with_site_replication_state_lock_on(store, operation).await with_site_replication_state_lock_on(store, operation).await
} }
/// Hold the distributed state-object read lock while `operation` validates a
/// topology snapshot. The closure may carry a bounded peer delivery only when
/// its receiver cannot write site replication state; peer-edit delivery must
/// run after this lock is released. Topology writers use the matching write
/// lock through [`with_site_replication_state_lock`].
pub(crate) async fn with_site_replication_state_read_lock<T, F, Fut>(operation: F) -> S3Result<T>
where
T: Send + 'static,
F: FnOnce(SiteReplicationState) -> Fut + Send + 'static,
Fut: std::future::Future<Output = S3Result<T>> + Send + 'static,
{
let store = current_object_store_handle().ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init"))?;
let read_store = store.clone();
with_config_object_read_lock(store, SITE_REPLICATION_STATE_PATH.to_string(), move || async move {
let state = load_site_replication_state_no_lock(read_store).await?;
operation(state).await
})
.await
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("lock site replication state failed: {e}")))?
}
/// Context-store variant for callers that resolve their store from an /// Context-store variant for callers that resolve their store from an
/// explicit [`AppContext`] (the service-side reload driven over node RPC). /// explicit [`AppContext`] (the service-side reload driven over node RPC).
/// ///
+568 -41
View File
@@ -33,6 +33,18 @@ use temp_env::with_var;
use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener; use tokio::net::TcpListener;
#[test]
fn test_bucket_mutation_lock_path_is_bucket_scoped() {
assert_eq!(
site_replication_bucket_mutation_lock_path("photos"),
"config/site-replication/bucket-mutation/photos.lock"
);
assert_ne!(
site_replication_bucket_mutation_lock_path("photos"),
site_replication_bucket_mutation_lock_path("videos")
);
}
fn valid_test_ca_pem(name: &str) -> String { fn valid_test_ca_pem(name: &str) -> String {
rcgen::generate_simple_self_signed(vec![name.to_string()]) rcgen::generate_simple_self_signed(vec![name.to_string()])
.expect("generate test CA") .expect("generate test CA")
@@ -381,6 +393,30 @@ async fn peer_clients_do_not_follow_redirects() {
assert!(tls_server.await.expect("custom redirect TLS server task")); assert!(tls_server.await.expect("custom redirect TLS server task"));
} }
#[tokio::test]
async fn peer_http_error_body_cannot_spoof_an_unreachable_peer() {
let (endpoint, ca_pem, server) = spawn_test_tls_server_with_response(
b"HTTP/1.1 500 Internal Server Error\r\ncontent-length: 27\r\nconnection: close\r\n\r\ndownstream failed (connect)",
)
.await;
let connection = validate_peer_connection_inner(&endpoint, false, &ca_pem, true).expect("custom CA peer connection");
let client =
build_custom_site_replication_peer_client(&empty_outbound_tls_state(), &connection).expect("custom CA peer client");
let err = PeerAdminRequest::post(&connection, SITE_REPLICATION_PEER_DEVNULL_PATH, "access-key")
.with_client(&client)
.send("secret-key", &serde_json::json!({}))
.await
.expect_err("HTTP 500 must fail");
let detail = err.to_string();
assert!(detail.contains("downstream failed (connect)"));
assert!(
!retry_error_indicates_peer_unreachable(&detail),
"an untrusted response body must not enable the fast reachability probe"
);
assert!(server.await.expect("HTTP error TLS server task"));
}
fn peer(name: &str, endpoint: &str) -> PeerInfo { fn peer(name: &str, endpoint: &str) -> PeerInfo {
PeerInfo { PeerInfo {
name: name.to_string(), name: name.to_string(),
@@ -419,6 +455,7 @@ fn drain_event(peer: &str, path: &str, retry_count: u32, updated_at: Option<Offs
last_error: "remote-operation-failed".to_string(), last_error: "remote-operation-failed".to_string(),
updated_at, updated_at,
edit_generation: None, edit_generation: None,
peer_unreachable: false,
deletions_recorded: false, deletions_recorded: false,
} }
} }
@@ -532,25 +569,25 @@ fn test_record_failed_iam_delivery_records_deletions_and_flags_entry() {
// Non-deletion failure: entry flagged, no record. // Non-deletion failure: entry flagged, no record.
let mut user_update = user_delete_item("alice"); let mut user_update = user_delete_item("alice");
user_update.iam_user.as_mut().expect("iam user").is_delete_req = false; user_update.iam_user.as_mut().expect("iam user").is_delete_req = false;
record_failed_iam_delivery(&mut state, &target, &user_update, "peer offline"); record_failed_iam_delivery(&mut state, &target, &user_update, "peer offline").expect("record failure");
assert_eq!(state.retry_queue.len(), 1); assert_eq!(state.retry_queue.len(), 1);
assert_eq!(state.retry_queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH); assert_eq!(state.retry_queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH);
assert!(state.retry_queue[0].deletions_recorded); assert!(state.retry_queue[0].deletions_recorded);
assert!(state.iam_deletion_replays.is_empty()); assert!(state.iam_deletion_replays.is_empty());
// Deletion failure: recorded for replay, entry stays flagged. // Deletion failure: recorded for replay, entry stays flagged.
record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline"); record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline").expect("record failure");
assert_eq!(state.iam_deletion_replays.len(), 1); assert_eq!(state.iam_deletion_replays.len(), 1);
assert_eq!(state.iam_deletion_replays[0].entity, "iam-user:alice"); assert_eq!(state.iam_deletion_replays[0].entity, "iam-user:alice");
assert!(state.retry_queue[0].deletions_recorded); assert!(state.retry_queue[0].deletions_recorded);
assert_eq!(state.retry_queue.len(), 1, "IAM failures stay collapsed per peer"); assert_eq!(state.retry_queue.len(), 1, "IAM failures stay collapsed per peer");
// Same entity again: newest body replaces the record. // Same entity again: newest body replaces the record.
record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline"); record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline").expect("record failure");
assert_eq!(state.iam_deletion_replays.len(), 1); assert_eq!(state.iam_deletion_replays.len(), 1);
// Different entity: second record. // Different entity: second record.
record_failed_iam_delivery(&mut state, &target, &policy_delete_item("readonly"), "peer offline"); record_failed_iam_delivery(&mut state, &target, &policy_delete_item("readonly"), "peer offline").expect("record failure");
assert_eq!(state.iam_deletion_replays.len(), 2); assert_eq!(state.iam_deletion_replays.len(), 2);
// A legacy entry (created without recording) is never stamped. // A legacy entry (created without recording) is never stamped.
@@ -565,8 +602,9 @@ fn test_record_failed_iam_delivery_records_deletions_and_flags_entry() {
SITE_REPLICATION_PEER_IAM_ITEM_WIRE_PATH, SITE_REPLICATION_PEER_IAM_ITEM_WIRE_PATH,
"peer offline", "peer offline",
None, None,
); )
record_failed_iam_delivery(&mut state, &legacy, &user_delete_item("bob"), "peer offline"); .expect("upsert retry event");
record_failed_iam_delivery(&mut state, &legacy, &user_delete_item("bob"), "peer offline").expect("record failure");
let legacy_event = state let legacy_event = state
.retry_queue .retry_queue
.iter() .iter()
@@ -589,12 +627,13 @@ fn test_record_failed_iam_delivery_overflow_degrades_to_escalation() {
}; };
let mut state = deletion_replay_state(&target); let mut state = deletion_replay_state(&target);
for index in 0..SITE_REPLICATION_IAM_DELETION_REPLAY_LIMIT_PER_PEER { for index in 0..SITE_REPLICATION_IAM_DELETION_REPLAY_LIMIT_PER_PEER {
record_failed_iam_delivery(&mut state, &target, &policy_delete_item(&format!("p{index}")), "peer offline"); record_failed_iam_delivery(&mut state, &target, &policy_delete_item(&format!("p{index}")), "peer offline")
.expect("record failure");
} }
assert!(state.retry_queue[0].deletions_recorded); assert!(state.retry_queue[0].deletions_recorded);
assert_eq!(state.iam_deletion_replays.len(), SITE_REPLICATION_IAM_DELETION_REPLAY_LIMIT_PER_PEER); assert_eq!(state.iam_deletion_replays.len(), SITE_REPLICATION_IAM_DELETION_REPLAY_LIMIT_PER_PEER);
record_failed_iam_delivery(&mut state, &target, &policy_delete_item("one-too-many"), "peer offline"); record_failed_iam_delivery(&mut state, &target, &policy_delete_item("one-too-many"), "peer offline").expect("record failure");
assert_eq!( assert_eq!(
state.iam_deletion_replays.len(), state.iam_deletion_replays.len(),
SITE_REPLICATION_IAM_DELETION_REPLAY_LIMIT_PER_PEER, SITE_REPLICATION_IAM_DELETION_REPLAY_LIMIT_PER_PEER,
@@ -620,33 +659,23 @@ fn test_settle_replayed_iam_retry_events_settles_or_escalates() {
// Fully recorded: settles. // Fully recorded: settles.
let mut state = deletion_replay_state(&target); let mut state = deletion_replay_state(&target);
record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline"); record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline").expect("record failure");
state.retry_queue[0].updated_at = Some(snapshot_at); state.retry_queue[0].updated_at = Some(snapshot_at);
let observed = state.retry_queue[0].clone();
let replayed: Vec<String> = state.iam_deletion_replays.iter().map(|record| record.id.clone()).collect(); let replayed: Vec<String> = state.iam_deletion_replays.iter().map(|record| record.id.clone()).collect();
assert!(settle_replayed_iam_retry_events( assert!(settle_replayed_iam_retry_events(&mut state, &target, &observed, &replayed));
&mut state,
&target,
SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH,
Some(snapshot_at),
&replayed,
));
assert!(state.retry_queue.is_empty()); assert!(state.retry_queue.is_empty());
assert!(state.iam_deletion_replays.is_empty()); assert!(state.iam_deletion_replays.is_empty());
// Not fully recorded: replayed records are still removed, but the entry // Not fully recorded: replayed records are still removed, but the entry
// escalates instead of settling. // escalates instead of settling.
let mut state = deletion_replay_state(&target); let mut state = deletion_replay_state(&target);
record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline"); record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline").expect("record failure");
state.retry_queue[0].updated_at = Some(snapshot_at); state.retry_queue[0].updated_at = Some(snapshot_at);
state.retry_queue[0].deletions_recorded = false; state.retry_queue[0].deletions_recorded = false;
let observed = state.retry_queue[0].clone();
let replayed: Vec<String> = state.iam_deletion_replays.iter().map(|record| record.id.clone()).collect(); let replayed: Vec<String> = state.iam_deletion_replays.iter().map(|record| record.id.clone()).collect();
assert!(!settle_replayed_iam_retry_events( assert!(!settle_replayed_iam_retry_events(&mut state, &target, &observed, &replayed));
&mut state,
&target,
SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH,
Some(snapshot_at),
&replayed,
));
assert!(state.iam_deletion_replays.is_empty()); assert!(state.iam_deletion_replays.is_empty());
assert_eq!(state.retry_queue.len(), 1); assert_eq!(state.retry_queue.len(), 1);
assert_eq!(state.retry_queue[0].last_error, SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER); assert_eq!(state.retry_queue[0].last_error, SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER);
@@ -654,17 +683,13 @@ fn test_settle_replayed_iam_retry_events_settles_or_escalates() {
// Newer failure since the snapshot: entry untouched and drain-eligible, // Newer failure since the snapshot: entry untouched and drain-eligible,
// residual (unreplayed) record kept for the next pass. // residual (unreplayed) record kept for the next pass.
let mut state = deletion_replay_state(&target); let mut state = deletion_replay_state(&target);
record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline"); record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "peer offline").expect("record failure");
state.retry_queue[0].updated_at = Some(snapshot_at);
let observed = state.retry_queue[0].clone();
let replayed: Vec<String> = state.iam_deletion_replays.iter().map(|record| record.id.clone()).collect(); let replayed: Vec<String> = state.iam_deletion_replays.iter().map(|record| record.id.clone()).collect();
record_failed_iam_delivery(&mut state, &target, &user_delete_item("bob"), "peer offline"); record_failed_iam_delivery(&mut state, &target, &user_delete_item("bob"), "peer offline").expect("record failure");
state.retry_queue[0].updated_at = Some(snapshot_at + time::Duration::seconds(5)); state.retry_queue[0].updated_at = Some(snapshot_at + time::Duration::seconds(5));
assert!(!settle_replayed_iam_retry_events( assert!(!settle_replayed_iam_retry_events(&mut state, &target, &observed, &replayed));
&mut state,
&target,
SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH,
Some(snapshot_at),
&replayed,
));
assert_eq!(state.retry_queue.len(), 1); assert_eq!(state.retry_queue.len(), 1);
assert_ne!(state.retry_queue[0].last_error, SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER); assert_ne!(state.retry_queue[0].last_error, SITE_REPLICATION_RETRY_SNAPSHOT_REPLAYED_MARKER);
assert!( assert!(
@@ -673,6 +698,24 @@ fn test_settle_replayed_iam_retry_events_settles_or_escalates() {
); );
assert_eq!(state.iam_deletion_replays.len(), 1); assert_eq!(state.iam_deletion_replays.len(), 1);
assert_eq!(state.iam_deletion_replays[0].entity, "iam-user:bob"); assert_eq!(state.iam_deletion_replays[0].entity, "iam-user:bob");
// A newer deletion of the same entity gets a fresh replay-record id. An
// older settlement therefore removes neither its body nor its queue
// revision, even if the persisted timestamps happen to be equal.
let mut state = deletion_replay_state(&target);
record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "first failure").expect("record failure");
state.retry_queue[0].updated_at = Some(snapshot_at);
let observed = state.retry_queue[0].clone();
let replayed = vec![state.iam_deletion_replays[0].id.clone()];
record_failed_iam_delivery(&mut state, &target, &user_delete_item("alice"), "newer failure").expect("record newer failure");
state.retry_queue[0].updated_at = Some(snapshot_at);
assert_ne!(state.iam_deletion_replays[0].id, replayed[0]);
assert!(!settle_replayed_iam_retry_events(&mut state, &target, &observed, &replayed));
assert_eq!(state.retry_queue.len(), 1);
assert_ne!(state.retry_queue[0].id, observed.id);
assert_eq!(state.iam_deletion_replays.len(), 1);
assert_eq!(state.iam_deletion_replays[0].entity, "iam-user:alice");
} }
/// Merging legacy wire-path rows into the collapsed entry must not launder an /// Merging legacy wire-path rows into the collapsed entry must not launder an
@@ -748,6 +791,281 @@ fn test_classify_site_replication_retry_event_actions() {
assert_eq!(classify("/rustfs/admin/v3/site-replication/peer/unknown"), None); assert_eq!(classify("/rustfs/admin/v3/site-replication/peer/unknown"), None);
} }
#[test]
fn test_bucket_make_retry_replays_matching_configure_before_settlement() {
let make_photos =
"/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning".to_string();
let configure_photos =
"/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=configure-replication".to_string();
let plan = SiteReplicationBootstrapPlan {
bucket_make_ops: vec![
make_photos.clone(),
"/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=videos&operation=make-with-versioning".to_string(),
],
bucket_configure_ops: vec![
"/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=videos&operation=configure-replication".to_string(),
configure_photos.clone(),
],
bucket_items: vec![
SRBucketMeta {
bucket: "videos".to_string(),
r#type: "tags".to_string(),
..Default::default()
},
SRBucketMeta {
bucket: "photos".to_string(),
r#type: "policy".to_string(),
..Default::default()
},
],
..Default::default()
};
let tasks = bucket_op_retry_replay_tasks(&plan, SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING, "photos")
.expect("make retry plan should include its configure follow-up");
assert_eq!(
tasks.iter().map(SiteReplicationRepairTask::path).collect::<Vec<_>>(),
vec![
make_photos.as_str(),
"/rustfs/admin/v3/site-replication/peer/bucket-meta",
configure_photos.as_str()
]
);
assert!(matches!(tasks[0], SiteReplicationRepairTask::BucketMake(_)));
assert!(matches!(&tasks[1], SiteReplicationRepairTask::BucketMetadata(item) if item.bucket == "photos"));
assert!(matches!(tasks[2], SiteReplicationRepairTask::Replication(_)));
}
#[test]
fn test_bucket_make_retry_without_matching_configure_fails_closed() {
let plan = SiteReplicationBootstrapPlan {
bucket_make_ops: vec![
"/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning".to_string(),
],
..Default::default()
};
let err = match bucket_op_retry_replay_tasks(&plan, SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING, "photos") {
Ok(_) => panic!("make retry must not settle without a matching configure operation"),
Err(err) => err,
};
assert_eq!(err.code(), &S3ErrorCode::InternalError);
}
#[test]
fn test_retry_drain_bounds_each_peer_round_to_one_small_request_chain() {
let plan = SiteReplicationBootstrapPlan {
iam_items: vec![SRIAMItem::default(); 3],
bucket_make_ops: vec![
"/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning".to_string(),
],
bucket_configure_ops: vec![
"/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=configure-replication".to_string(),
],
bucket_items: vec![SRBucketMeta {
bucket: "photos".to_string(),
r#type: "tags".to_string(),
..Default::default()
}],
..Default::default()
};
let make = RetryDrainAction::BucketOpReplay {
operation: SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING.to_string(),
bucket: "photos".to_string(),
};
assert!(is_lightweight_retry_drain_action(&make));
assert!(!is_lightweight_retry_drain_action(&RetryDrainAction::IamSnapshot));
assert!(!is_lightweight_retry_drain_action(&RetryDrainAction::PeerEdit));
assert_eq!(retry_drain_request_count(&make, Some(&plan)), 3);
assert!(retry_drain_request_count(&make, Some(&plan)) <= SITE_REPLICATION_RETRY_DRAIN_MAX_REQUESTS_PER_PEER);
assert!(
retry_drain_request_count(&RetryDrainAction::IamSnapshot, Some(&plan))
> SITE_REPLICATION_RETRY_DRAIN_MAX_REQUESTS_PER_PEER
);
assert!(
retry_drain_request_count(&RetryDrainAction::PeerEdit, Some(&plan)) > SITE_REPLICATION_RETRY_DRAIN_MAX_REQUESTS_PER_PEER
);
}
#[test]
fn test_lightweight_retry_peer_rotation_covers_all_queued_peers() {
let limit = SITE_REPLICATION_RETRY_DRAIN_PEER_CONCURRENCY;
for peer_count in 1..=(limit * 3 + 1) {
let rounds = peer_count.div_ceil(limit);
let mut seen = HashSet::new();
for round in 7..(7 + rounds as i64) {
let start = lightweight_retry_peer_rotation(peer_count, round);
for offset in 0..limit.min(peer_count) {
seen.insert((start + offset) % peer_count);
}
}
assert_eq!(
seen.len(),
peer_count,
"every peer must enter the bounded lightweight window within {rounds} rounds"
);
}
}
#[test]
fn test_lightweight_bucket_retry_plan_is_targeted_and_preserves_make_options() {
let created_at = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp");
let bucket = SRBucketInfo {
bucket: "photos".to_string(),
created_at: Some(created_at),
object_lock_config: Some(String::new()),
tags: Some("dGFncy14bWw=".to_string()),
tag_config_updated_at: Some(created_at),
..Default::default()
};
let plan = site_replication_bucket_retry_plan_for(&bucket, false).expect("targeted retry plan");
assert!(plan.iam_items.is_empty());
assert_eq!(plan.bucket_make_ops.len(), 1);
assert!(plan.bucket_make_ops[0].contains("bucket=photos"));
assert!(plan.bucket_make_ops[0].contains("lockEnabled=true"));
assert!(plan.bucket_make_ops[0].contains("createdAt="));
assert_eq!(plan.bucket_items.len(), 2);
assert_eq!(plan.bucket_items[0].r#type, "tags");
assert_eq!(plan.bucket_items[1].r#type, "object-lock-config");
assert_eq!(plan.bucket_configure_ops.len(), 1);
assert!(plan.bucket_configure_ops[0].contains("operation=configure-replication"));
let tasks =
bucket_op_retry_replay_tasks(&plan, SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING, "photos").expect("retry task chain");
assert!(matches!(tasks[0], SiteReplicationRepairTask::BucketMake(_)));
assert!(matches!(tasks[1], SiteReplicationRepairTask::BucketMetadata(_)));
assert!(matches!(tasks[2], SiteReplicationRepairTask::BucketMetadata(_)));
assert!(matches!(tasks[3], SiteReplicationRepairTask::Replication(_)));
}
#[test]
fn test_lightweight_bucket_retry_plan_orders_real_metadata_and_counts_it() {
let versioning = bucket_versioning_xml().expect("canonical versioning config");
let replication = serialize(&site_repl_config("remote-dep")).expect("derived replication config");
let bucket = SRBucketInfo {
bucket: "photos".to_string(),
policy: Some(serde_json::json!({"Version":"2012-10-17","Statement":[]})),
tags: Some(BASE64_STANDARD.encode_to_string("<Tagging/>")),
versioning: Some(BASE64_STANDARD.encode_to_string(&versioning)),
replication_config: Some(BASE64_STANDARD.encode_to_string(&replication)),
..Default::default()
};
let plan = site_replication_bucket_retry_plan_from_info(&bucket, false).expect("targeted retry plan");
let tasks = bucket_op_retry_replay_tasks(&plan, SITE_REPLICATION_BUCKET_OP_MAKE_WITH_VERSIONING, "photos")
.expect("bucket replay tasks");
assert!(matches!(tasks.first(), Some(SiteReplicationRepairTask::BucketMake(_))));
assert!(matches!(tasks.last(), Some(SiteReplicationRepairTask::Replication(_))));
assert!(
tasks[1..tasks.len() - 1]
.iter()
.all(|task| matches!(task, SiteReplicationRepairTask::BucketMetadata(_)))
);
assert_eq!(tasks.len(), 4, "make + policy + tags + configure must all count against the budget");
assert!(
tasks.len() <= SITE_REPLICATION_RETRY_DRAIN_MAX_REQUESTS_PER_PEER,
"the complete metadata chain must fit the bounded lightweight replay"
);
let mut operator_replication = site_repl_config("remote-dep");
operator_replication.rules.push(operator_rule("operator-backup"));
let mut bucket_with_operator_rule = bucket;
bucket_with_operator_rule.replication_config =
Some(BASE64_STANDARD.encode_to_string(&serialize(&operator_replication).expect("operator replication config")));
let plan = site_replication_bucket_retry_plan_from_info(&bucket_with_operator_rule, false).expect("targeted retry plan");
assert!(
plan.bucket_items.iter().any(|item| item.r#type == "replication-config"),
"operator-authored replication rules cannot be replaced by configure-replication"
);
}
#[test]
fn test_delete_bucket_broadcast_fences_target_membership_through_delivery() {
let hooks = include_str!("hooks.rs");
let delete_broadcast = hooks
.split("async fn broadcast_site_replication_delete_bucket")
.nth(1)
.and_then(|rest| rest.split("pub(crate) async fn commit_site_replication_delete_bucket").next())
.expect("delete-bucket broadcast should exist");
assert!(
delete_broadcast.contains("with_site_replication_state_read_lock(move |state| async move {")
&& delete_broadcast.contains("state.peers.get(&fallback_peer.deployment_id)")
&& delete_broadcast.contains("site_replicator_service_account_secret(&state.service_account_access_key)")
&& delete_broadcast
.contains("PeerAdminRequest::put(&transport.connection, &delivery_path, &state.service_account_access_key)"),
"a destructive bucket delivery must resolve current topology and credentials under the distributed state read lock"
);
assert!(
delete_broadcast.contains("enqueue_site_replication_retry_event(&current_peer, &request_path, &err).await"),
"a failed destructive delivery must remain visible for operator repair"
);
let usecase = include_str!("../app/bucket_usecase.rs");
let delete = usecase
.split("async fn execute_delete_bucket_inner")
.nth(1)
.and_then(|rest| rest.split("pub async fn execute_head_bucket").next())
.expect("delete bucket usecase");
assert!(
delete
.find("prepare_site_replication_delete_bucket")
.expect("durable reservation")
< delete.find(".delete_bucket(").expect("local delete"),
"destructive peer liabilities must be persisted before the local bucket is deleted"
);
}
#[test]
fn test_bucket_retry_settlement_preserves_a_newer_same_path_failure() {
let peer = peer("remote", "https://remote.example.com");
let path = "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=configure-replication";
let observed_at = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp");
let observed = drain_event("remote", path, 1, Some(observed_at));
let mut queue = vec![observed.clone()];
queue[0].id = "evt-remote-new-revision".to_string();
queue[0].retry_count += 1;
assert_eq!(settle_observed_site_replication_retry_event(&mut queue, &peer, &observed), 0);
assert_eq!(queue.len(), 1, "a newer same-timestamp failure must survive stale settlement");
let current = queue[0].clone();
assert_eq!(settle_observed_site_replication_retry_event(&mut queue, &peer, &current), 1);
assert!(queue.is_empty());
}
#[test]
fn test_reachable_probe_promotion_is_fenced_by_the_observed_event() {
let now = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp");
let path = "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning";
let mut event = drain_event("remote", path, 3, Some(now));
event.peer_unreachable = true;
let recovered = event.clone();
let mut state = SiteReplicationState {
retry_queue: vec![event],
..Default::default()
};
state
.peers
.insert("remote".to_string(), peer("remote", "https://remote.example.com"));
assert_eq!(mark_reachable_deferred_retry_events(&mut state, &[recovered.clone()]), 1);
assert_eq!(state.retry_queue[0].updated_at, None);
assert!(!state.retry_queue[0].peer_unreachable);
assert_eq!(
actionable_site_replication_retry_events(&state, now).len(),
1,
"a successful probe must make the event replayable in the same drain tick"
);
state.retry_queue[0].updated_at = Some(now + time::Duration::seconds(1));
state.retry_queue[0].peer_unreachable = true;
assert_eq!(mark_reachable_deferred_retry_events(&mut state, &[recovered]), 0);
assert_eq!(state.retry_queue[0].updated_at, Some(now + time::Duration::seconds(1)));
assert!(state.retry_queue[0].peer_unreachable);
}
#[test] #[test]
fn test_retry_snapshot_fingerprint_detects_concurrent_iam_change() { fn test_retry_snapshot_fingerprint_detects_concurrent_iam_change() {
let old = SRIAMItem { let old = SRIAMItem {
@@ -828,6 +1146,100 @@ fn test_site_replication_retry_backoff_schedule() {
assert!(elapsed(30, 86_401)); assert!(elapsed(30, 86_401));
} }
#[test]
fn test_retry_error_marks_peer_unreachable_only_for_connection_failures() {
let mut queue = Vec::new();
let peer = peer("remote", "https://remote.example.com");
let bucket_make = "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning";
upsert_site_replication_retry_event(
&mut queue,
&peer,
bucket_make,
"peer request to https://remote.example.com failed (connect): connection refused",
None,
)
.expect("upsert retry event");
assert!(queue[0].peer_unreachable);
upsert_site_replication_retry_event(
&mut queue,
&peer,
bucket_make,
"peer request to https://remote.example.com failed (timeout): request exceeded 10 seconds",
None,
)
.expect("upsert retry event");
assert!(
!queue[0].peer_unreachable,
"a whole-request timeout does not prove the peer is unreachable"
);
upsert_site_replication_retry_event(
&mut queue,
&peer,
bucket_make,
"peer request to https://remote.example.com failed with 500 Internal Server Error: downstream failed (connect)",
None,
)
.expect("upsert retry event");
assert!(
!queue[0].peer_unreachable,
"application failures and their untrusted bodies must keep the normal replay backoff"
);
upsert_site_replication_retry_event(
&mut queue,
&peer,
bucket_make,
"peer request to https://remote.example.com failed with 500 Internal Server Error: backend failed (connect): spoofed",
None,
)
.expect("upsert retry event");
assert!(!queue[0].peer_unreachable, "peer response bodies must not spoof transport failures");
}
#[test]
fn test_connect_timeout_is_classified_as_a_connection_failure() {
assert_eq!(classify_peer_transport_error(true, true, "tcp connect timed out"), "connect");
assert_eq!(classify_peer_transport_error(false, true, "request timed out"), "timeout");
assert_eq!(
classify_peer_transport_error(false, true, "request timed out for https://tls-gateway.example"),
"timeout"
);
assert_eq!(classify_peer_transport_error(true, false, "tls handshake failed"), "tls handshake");
}
#[test]
fn test_retry_event_peer_unreachable_is_legacy_serde_default() {
let json = r#"{
"id":"evt-legacy",
"peer_deployment_id":"remote",
"peer_endpoint":"https://remote.example.com",
"path":"/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning",
"retry_count":1,
"failed":false,
"last_error":"peer request to https://remote.example.com failed (connect): connection refused"
}"#;
let mut event: SiteReplicationRetryEvent = serde_json::from_str(json).expect("legacy retry event decodes");
assert!(!event.peer_unreachable);
let now = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp");
event.updated_at = Some(now - time::Duration::seconds(30));
let mut state = SiteReplicationState::default();
state
.peers
.insert("remote".to_string(), peer("remote", "https://remote.example.com"));
state.retry_queue.push(event);
assert_eq!(
deferred_site_replication_retry_events(&state, now).len(),
1,
"rolling-upgrade records must retain fast recovery from their trusted outer error shape"
);
}
/// The actionable subset respects classification, peer membership and /// The actionable subset respects classification, peer membership and
/// backoff; everything else stays untouched in the queue. /// backoff; everything else stays untouched in the queue.
#[test] #[test]
@@ -915,6 +1327,51 @@ fn test_deferred_site_replication_retry_events_partition() {
assert_eq!(actionable[0].path, "/rustfs/admin/v3/site-replication/peer/bucket-meta"); assert_eq!(actionable[0].path, "/rustfs/admin/v3/site-replication/peer/bucket-meta");
} }
#[test]
fn test_deferred_retry_events_probe_fresh_peer_transport_failures() {
let now = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp");
let mut state = SiteReplicationState::default();
state
.peers
.insert("remote".to_string(), peer("remote", "https://remote.example.com"));
let bucket_make = "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning";
let mut fresh_transport_failure = drain_event("remote", bucket_make, 1, Some(now - time::Duration::seconds(30)));
fresh_transport_failure.peer_unreachable = true;
state.retry_queue.push(fresh_transport_failure);
let deferred = deferred_site_replication_retry_events(&state, now);
assert_eq!(
deferred.len(),
1,
"fresh transport failures must be eligible for a cheap reachability probe"
);
assert_eq!(deferred[0].path, bucket_make);
let actionable = actionable_site_replication_retry_events(&state, now);
assert!(actionable.is_empty(), "the event is still protected from direct replay by normal backoff");
}
#[test]
fn test_deferred_retry_events_do_not_probe_fresh_application_failures() {
let now = OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("timestamp");
let mut state = SiteReplicationState::default();
state
.peers
.insert("remote".to_string(), peer("remote", "https://remote.example.com"));
let bucket_make = "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=photos&operation=make-with-versioning";
state
.retry_queue
.push(drain_event("remote", bucket_make, 1, Some(now - time::Duration::seconds(30))));
assert!(
deferred_site_replication_retry_events(&state, now).is_empty(),
"reachable peers that reject an operation must keep the base replay backoff"
);
assert!(actionable_site_replication_retry_events(&state, now).is_empty());
}
/// The drain settles a peer-edit success under a freshly allocated /// The drain settles a peer-edit success under a freshly allocated
/// generation; legacy queue entries carry `edit_generation: None` and /// generation; legacy queue entries carry `edit_generation: None` and
/// must be cleared by that generation-scoped settlement (`(Some, None)` /// must be cleared by that generation-scoped settlement (`(Some, None)`
@@ -996,7 +1453,7 @@ fn test_escalate_up_to_marks_snapshot_replayed_and_keeps_newer_failures() {
// successful Bob update on the shared wire path cannot erase it even // successful Bob update on the shared wire path cannot erase it even
// before the drain runs. // before the drain runs.
let mut queue = Vec::new(); let mut queue = Vec::new();
upsert_site_replication_retry_event(&mut queue, &target, path, "alice delete failed", None); upsert_site_replication_retry_event(&mut queue, &target, path, "alice delete failed", None).expect("upsert retry event");
assert_eq!(queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH); assert_eq!(queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH);
assert_eq!(dequeue_site_replication_retry_events(&mut queue, &target, path), 0); assert_eq!(dequeue_site_replication_retry_events(&mut queue, &target, path), 0);
assert_eq!(queue.len(), 1); assert_eq!(queue.len(), 1);
@@ -1005,7 +1462,7 @@ fn test_escalate_up_to_marks_snapshot_replayed_and_keeps_newer_failures() {
// A later hook failure overwrites the marker and re-arms the drain. // A later hook failure overwrites the marker and re-arms the drain.
let mut queue = vec![drain_event("remote", path, 2, Some(snapshot_at))]; let mut queue = vec![drain_event("remote", path, 2, Some(snapshot_at))];
escalate_site_replication_retry_events_up_to(&mut queue, &target, path, Some(snapshot_at)); escalate_site_replication_retry_events_up_to(&mut queue, &target, path, Some(snapshot_at));
upsert_site_replication_retry_event(&mut queue, &target, path, "peer offline", None); upsert_site_replication_retry_event(&mut queue, &target, path, "peer offline", None).expect("upsert retry event");
assert!(classify_site_replication_retry_event(&queue[0]).is_some()); assert!(classify_site_replication_retry_event(&queue[0]).is_some());
// Legacy entry without a timestamp: escalated. // Legacy entry without a timestamp: escalated.
@@ -1781,17 +2238,85 @@ fn test_retry_event_upsert_marks_repeated_failures() {
}; };
let mut queue = Vec::new(); let mut queue = Vec::new();
upsert_site_replication_retry_event(&mut queue, &peer, "/rustfs/admin/v3/site-replication/peer/iam-item", "first", None); upsert_site_replication_retry_event(&mut queue, &peer, "/rustfs/admin/v3/site-replication/peer/iam-item", "first", None)
upsert_site_replication_retry_event(&mut queue, &peer, "/rustfs/admin/v3/site-replication/peer/iam-item", "second", None); .expect("upsert retry event");
upsert_site_replication_retry_event(&mut queue, &peer, "/rustfs/admin/v3/site-replication/peer/iam-item", "third", None); let first_revision = queue[0].id.clone();
upsert_site_replication_retry_event(&mut queue, &peer, "/rustfs/admin/v3/site-replication/peer/iam-item", "second", None)
.expect("upsert retry event");
let second_revision = queue[0].id.clone();
upsert_site_replication_retry_event(&mut queue, &peer, "/rustfs/admin/v3/site-replication/peer/iam-item", "third", None)
.expect("upsert retry event");
assert_eq!(queue.len(), 1); assert_eq!(queue.len(), 1);
assert_ne!(first_revision, second_revision);
assert_ne!(second_revision, queue[0].id, "each failure must advance the settlement revision");
assert_eq!(queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH); assert_eq!(queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH);
assert_eq!(queue[0].retry_count, SITE_REPLICATION_RETRY_FAILED_AFTER); assert_eq!(queue[0].retry_count, SITE_REPLICATION_RETRY_FAILED_AFTER);
assert!(queue[0].failed); assert!(queue[0].failed);
assert_eq!(queue[0].last_error, "third"); assert_eq!(queue[0].last_error, "third");
} }
#[test]
fn retry_queue_capacity_never_evicts_destructive_bucket_liabilities() {
let target = PeerInfo {
deployment_id: "remote-dep".to_string(),
..peer("remote", "https://remote.example.com")
};
let destructive = |index: usize| SiteReplicationRetryEvent {
id: format!("delete-{index}"),
peer_deployment_id: target.deployment_id.clone(),
peer_endpoint: target.endpoint.clone(),
path: format!("{SITE_REPLICATION_PEER_BUCKET_OPS_PATH}?bucket=bucket-{index}&operation=delete-bucket"),
..Default::default()
};
let mut queue = (0..SITE_REPLICATION_RETRY_QUEUE_LIMIT).map(destructive).collect::<Vec<_>>();
let original_ids = queue.iter().map(|event| event.id.clone()).collect::<HashSet<_>>();
let new_path = format!("{SITE_REPLICATION_PEER_BUCKET_OPS_PATH}?bucket=overflow&operation=force-delete-bucket");
let err = upsert_site_replication_retry_event(&mut queue, &target, &new_path, "reserve delete", None)
.expect_err("an all-destructive full queue must fail closed");
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable);
assert_eq!(queue.len(), SITE_REPLICATION_RETRY_QUEUE_LIMIT);
assert_eq!(queue.iter().map(|event| event.id.clone()).collect::<HashSet<_>>(), original_ids);
queue[0] = SiteReplicationRetryEvent {
id: "iam-snapshot".to_string(),
peer_deployment_id: target.deployment_id.clone(),
peer_endpoint: target.endpoint.clone(),
path: SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH.to_string(),
deletions_recorded: true,
..Default::default()
};
upsert_site_replication_retry_event(&mut queue, &target, &new_path, "reserve delete", None)
.expect_err("a collapsed IAM liability may contain a deletion and must not be evicted");
assert!(queue.iter().any(|event| event.id == "iam-snapshot"));
queue[0] = SiteReplicationRetryEvent {
id: "rebuildable".to_string(),
peer_deployment_id: target.deployment_id.clone(),
peer_endpoint: target.endpoint.clone(),
path: SITE_REPLICATION_PEER_EDIT_PATH.to_string(),
..Default::default()
};
let preserved_delete_ids = queue
.iter()
.filter(|event| is_destructive_bucket_retry_path(&event.path))
.map(|event| event.id.clone())
.collect::<HashSet<_>>();
let evicted = upsert_site_replication_retry_event(&mut queue, &target, &new_path, "reserve delete", None)
.expect("a rebuildable row may make room for a destructive liability");
assert_eq!(evicted.len(), 1);
assert_eq!(evicted[0].id, "rebuildable");
assert_eq!(queue.len(), SITE_REPLICATION_RETRY_QUEUE_LIMIT);
assert!(
preserved_delete_ids
.iter()
.all(|id| queue.iter().any(|event| &event.id == id))
);
assert!(queue.iter().any(|event| event.path == new_path));
}
/// P1-15 review follow-up: a successful peer-edit delivery only proves the /// P1-15 review follow-up: a successful peer-edit delivery only proves the
/// peer reached the state THAT delivery carried. Settling it must not /// peer reached the state THAT delivery carried. Settling it must not
/// erase a retry event a newer edit left behind, or the local site sits on /// erase a retry event a newer edit left behind, or the local site sits on
@@ -1807,7 +2332,8 @@ fn retry_settlement_must_not_erase_a_newer_generation_failure() {
// Edit A (generation 5) delivered successfully and is stalled before // Edit A (generation 5) delivered successfully and is stalled before
// settling. Edit B (generation 6) commits meanwhile, fails delivery to // settling. Edit B (generation 6) commits meanwhile, fails delivery to
// the same peer, and enqueues. // the same peer, and enqueues.
upsert_site_replication_retry_event(&mut queue, &peer, SITE_REPLICATION_PEER_EDIT_PATH, "peer offline", Some(6)); upsert_site_replication_retry_event(&mut queue, &peer, SITE_REPLICATION_PEER_EDIT_PATH, "peer offline", Some(6))
.expect("upsert retry event");
// A resumes: its own settlement must leave B's retry alone. // A resumes: its own settlement must leave B's retry alone.
assert_eq!( assert_eq!(
@@ -1818,7 +2344,8 @@ fn retry_settlement_must_not_erase_a_newer_generation_failure() {
assert_eq!(queue[0].edit_generation, Some(6)); assert_eq!(queue[0].edit_generation, Some(6));
// An even older delivery failing afterwards must not lower the fence. // An even older delivery failing afterwards must not lower the fence.
upsert_site_replication_retry_event(&mut queue, &peer, SITE_REPLICATION_PEER_EDIT_PATH, "still offline", Some(4)); upsert_site_replication_retry_event(&mut queue, &peer, SITE_REPLICATION_PEER_EDIT_PATH, "still offline", Some(4))
.expect("upsert retry event");
assert_eq!(queue[0].edit_generation, Some(6)); assert_eq!(queue[0].edit_generation, Some(6));
// B's own delivery succeeding is what clears it. // B's own delivery succeeding is what clears it.
@@ -1831,7 +2358,7 @@ fn retry_settlement_must_not_erase_a_newer_generation_failure() {
// Collapsed broadcast failures live under an internal snapshot path; // Collapsed broadcast failures live under an internal snapshot path;
// an unrelated success on their shared wire path cannot settle them. // an unrelated success on their shared wire path cannot settle them.
let iam_path = "/rustfs/admin/v3/site-replication/peer/iam-item"; let iam_path = "/rustfs/admin/v3/site-replication/peer/iam-item";
upsert_site_replication_retry_event(&mut queue, &peer, iam_path, "peer offline", None); upsert_site_replication_retry_event(&mut queue, &peer, iam_path, "peer offline", None).expect("upsert retry event");
assert_eq!(dequeue_site_replication_retry_events(&mut queue, &peer, iam_path), 0); assert_eq!(dequeue_site_replication_retry_events(&mut queue, &peer, iam_path), 0);
assert_eq!(queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH); assert_eq!(queue[0].path, SITE_REPLICATION_RETRY_IAM_SNAPSHOT_PATH);
} }
+17 -13
View File
@@ -331,6 +331,7 @@ pub(crate) fn runtime_peer_connection(peer: &PeerInfo) -> S3Result<PeerConnectio
}) })
} }
#[derive(Clone)]
pub(crate) struct PeerTransport { pub(crate) struct PeerTransport {
pub(crate) connection: PeerConnection, pub(crate) connection: PeerConnection,
pub(crate) client: reqwest::Client, pub(crate) client: reqwest::Client,
@@ -697,19 +698,7 @@ impl<'a> PeerAdminRequest<'a> {
} }
let response = req.send().await.map_err(|e| { let response = req.send().await.map_err(|e| {
let classify = if e.is_timeout() { let classify = classify_peer_transport_error(e.is_connect(), e.is_timeout(), &e.to_string());
"timeout"
} else if e.is_connect() && e.to_string().to_ascii_lowercase().contains("dns") {
"dns resolution"
} else if e.to_string().to_ascii_lowercase().contains("certificate")
|| e.to_string().to_ascii_lowercase().contains("tls")
{
"tls handshake"
} else if e.is_connect() {
"connect"
} else {
"request"
};
S3Error::with_message(S3ErrorCode::InternalError, format!("peer request to {url} failed ({classify}): {e}")) S3Error::with_message(S3ErrorCode::InternalError, format!("peer request to {url} failed ({classify}): {e}"))
})?; })?;
@@ -825,6 +814,21 @@ impl<'a> PeerAdminRequest<'a> {
} }
} }
pub(crate) fn classify_peer_transport_error(is_connect: bool, is_timeout: bool, detail: &str) -> &'static str {
let detail = detail.to_ascii_lowercase();
if is_connect && detail.contains("dns") {
"dns resolution"
} else if is_connect && (detail.contains("certificate") || detail.contains("tls")) {
"tls handshake"
} else if is_connect {
"connect"
} else if is_timeout {
"timeout"
} else {
"request"
}
}
pub(crate) fn peer_error_may_be_secret_mismatch(detail: &str) -> bool { pub(crate) fn peer_error_may_be_secret_mismatch(detail: &str) -> bool {
let detail = detail.to_ascii_lowercase(); let detail = detail.to_ascii_lowercase();
detail.contains("signaturedoesnotmatch") detail.contains("signaturedoesnotmatch")
+45 -3
View File
@@ -28,16 +28,19 @@ use std::pin::Pin;
use std::sync::OnceLock; use std::sync::OnceLock;
use std::time::Duration; use std::time::Duration;
use tokio::time::Instant;
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
use tracing::warn; use tracing::warn;
const RECONCILE_INTERVAL: Duration = Duration::from_secs(600); const RECONCILE_INTERVAL: Duration = Duration::from_secs(600);
pub(crate) const RETRY_DRAIN_INTERVAL: Duration = Duration::from_secs(30);
/// A reconciler reports its own failures; the outcome carries no value because neither /// A reconciler reports its own failures; the outcome carries no value because neither
/// caller can act on one — a site that cannot repair its replication wiring still serves S3. /// caller can act on one — a site that cannot repair its replication wiring still serves S3.
type ReconcileHook = fn() -> Pin<Box<dyn Future<Output = ()> + Send>>; type ReconcileHook = fn() -> Pin<Box<dyn Future<Output = ()> + Send>>;
static RECONCILER: OnceLock<ReconcileHook> = OnceLock::new(); static RECONCILER: OnceLock<ReconcileHook> = OnceLock::new();
static RETRY_DRAINER: OnceLock<ReconcileHook> = OnceLock::new();
/// Install the admin layer's reconciler. Idempotent: a second call is ignored, which keeps /// Install the admin layer's reconciler. Idempotent: a second call is ignored, which keeps
/// repeated router construction (tests, the embedded server) from panicking. /// repeated router construction (tests, the embedded server) from panicking.
@@ -45,6 +48,12 @@ pub(crate) fn register_site_replication_reconciler(reconcile: ReconcileHook) {
let _ = RECONCILER.set(reconcile); let _ = RECONCILER.set(reconcile);
} }
/// Install the admin layer's lightweight retry drain. Idempotent for the same
/// reason as [`register_site_replication_reconciler`].
pub(crate) fn register_site_replication_retry_drainer(drain: ReconcileHook) {
let _ = RETRY_DRAINER.set(drain);
}
/// Repair drifted site-replication wiring, immediately and then on a timer. /// Repair drifted site-replication wiring, immediately and then on a timer.
/// ///
/// The first pass runs inside the spawned task rather than on the caller's path: it walks /// The first pass runs inside the spawned task rather than on the caller's path: it walks
@@ -62,16 +71,38 @@ pub(crate) fn spawn_site_replication_reconcile_task(ctx: CancellationToken) {
return; return;
} }
spawn_reconcile_loop(ctx.clone(), RECONCILE_INTERVAL, &RECONCILER, true);
if RETRY_DRAINER.get().is_none() {
warn!("site replication retry drainer is not registered; periodic retry drain disabled");
return;
}
spawn_reconcile_loop(ctx, RETRY_DRAIN_INTERVAL, &RETRY_DRAINER, false);
}
fn spawn_reconcile_loop(
ctx: CancellationToken,
interval: Duration,
hook: &'static OnceLock<ReconcileHook>,
run_immediately: bool,
) {
tokio::spawn(async move { tokio::spawn(async move {
let mut ticker = tokio::time::interval(RECONCILE_INTERVAL); let first_tick = if run_immediately {
Instant::now()
} else {
Instant::now() + interval
};
let mut ticker = tokio::time::interval_at(first_tick, interval);
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop { loop {
tokio::select! { tokio::select! {
_ = ctx.cancelled() => break, _ = ctx.cancelled() => break,
// The first tick fires immediately, which is the startup repair pass. // The heavy reconciler owns the startup repair pass. The lightweight
// retry drain starts on its normal cadence so it cannot steal that
// first lifecycle lock and defer bucket/IAM repair for a full interval.
_ = ticker.tick() => { _ = ticker.tick() => {
if let Some(reconcile) = RECONCILER.get() { if let Some(reconcile) = hook.get() {
reconcile().await; reconcile().await;
} }
} }
@@ -79,3 +110,14 @@ pub(crate) fn spawn_site_replication_reconcile_task(ctx: CancellationToken) {
} }
}); });
} }
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn retry_drain_runs_faster_than_heavy_reconcile() {
assert!(RETRY_DRAIN_INTERVAL < RECONCILE_INTERVAL);
assert!(RETRY_DRAIN_INTERVAL <= Duration::from_secs(60));
}
}
+200
View File
@@ -493,6 +493,13 @@ impl std::fmt::Debug for NodeService {
} }
} }
pub(crate) fn make_scanner_control_server() -> scanner_control_service_server::ScannerControlServiceServer<NodeService> {
let limit = rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize;
scanner_control_service_server::ScannerControlServiceServer::new(make_server())
.max_decoding_message_size(limit)
.max_encoding_message_size(limit)
}
pub fn make_server() -> NodeService { pub fn make_server() -> NodeService {
let context = runtime_sources::current_app_context(); let context = runtime_sources::current_app_context();
make_server_for_context(context) make_server_for_context(context)
@@ -1087,6 +1094,74 @@ impl NodeService {
} }
} }
#[tonic::async_trait]
impl scanner_control_service_server::ScannerControlService for NodeService {
async fn scanner_scoped_dirty_usage_ack(
&self,
request: Request<ScannerScopedDirtyUsageAckRequest>,
) -> Result<Response<ScannerScopedDirtyUsageAckResponse>, Status> {
use rustfs_protos::scoped_dirty_usage::*;
static ADMISSION: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(4);
let canonical =
canonical_scoped_dirty_usage_request(request.get_ref()).map_err(|err| Status::invalid_argument(err.to_string()))?;
verify_tonic_canonical_body_digest(&request, &canonical)
.map_err(|_| Status::permission_denied("scoped dirty usage authentication failed"))?;
let _admission = ADMISSION
.try_acquire()
.map_err(|_| Status::resource_exhausted("scoped dirty usage receiver is busy"))?;
let request = request.into_inner();
let store = self
.resolve_object_store()
.ok_or_else(|| Status::unavailable("storage layer is not initialized"))?;
if store.id.is_nil()
|| request.owner_id != store.id.to_string()
|| request.instance_id != rustfs_scanner::scanner_activity_epoch()
{
return Err(Status::failed_precondition("scoped dirty usage peer or process changed"));
}
let cleared = timeout(Duration::from_secs(30), async {
// Strict bucket order is validated before admission. Acquire every
// lifecycle/metadata fence before clearing any dirty record.
let mut guards = Vec::with_capacity(request.entries.len());
for entry in &request.entries {
let incarnation = Uuid::from_slice(entry.bucket_incarnation.as_ref())
.map_err(|_| Status::invalid_argument("invalid bucket incarnation"))?;
let guard =
crate::storage::storage_api::acquire_scanner_bucket_incarnation_fence(&entry.bucket, incarnation, store.id)
.await
.map_err(|_| Status::failed_precondition("trusted bucket incarnation is unavailable"))?;
guards.push(guard);
}
let entries = guards
.iter()
.zip(&request.entries)
.map(|(guard, entry)| (guard, entry.generation))
.collect::<Vec<_>>();
rustfs_scanner::acknowledge_scoped_dirty_usage(&request.instance_id, &entries, request.probe_only)
.map_err(|err| Status::failed_precondition(err.to_string()))
})
.await
.map_err(|_| Status::deadline_exceeded("scoped dirty usage incarnation validation timed out"))??;
let mut response = ScannerScopedDirtyUsageAckResponse {
protocol_version: SCOPED_DIRTY_USAGE_PROTOCOL_VERSION,
owner_id: request.owner_id,
instance_id: request.instance_id,
supported: true,
max_entries: SCOPED_DIRTY_USAGE_MAX_ENTRIES,
max_request_bytes: SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES,
cleared,
response_proof: Bytes::new(),
};
let body = canonical_scoped_dirty_usage_response(&canonical, &response)
.map_err(|_| Status::internal("scoped dirty usage response is too large"))?;
response.response_proof = sign_tonic_rpc_response_proof(&body)
.map_err(|_| Status::unavailable("scoped dirty usage response authentication is unavailable"))?
.into();
Ok(Response::new(response))
}
}
#[tonic::async_trait] #[tonic::async_trait]
impl Node for NodeService { impl Node for NodeService {
async fn ping(&self, request: Request<PingRequest>) -> Result<Response<PingResponse>, Status> { async fn ping(&self, request: Request<PingRequest>) -> Result<Response<PingResponse>, Status> {
@@ -2623,6 +2698,7 @@ mod tests {
use rustfs_kms::KmsServiceManager; use rustfs_kms::KmsServiceManager;
use rustfs_protos::CanonicalMutationBody as _; use rustfs_protos::CanonicalMutationBody as _;
use rustfs_protos::models::PingBodyBuilder; use rustfs_protos::models::PingBodyBuilder;
use rustfs_protos::proto_gen::node_service::scanner_control_service_server::ScannerControlService as _;
use rustfs_protos::proto_gen::node_service::{ use rustfs_protos::proto_gen::node_service::{
BackgroundHealStatusRequest, BatchGenerallyLockRequest, CancelDecommissionRequest, CheckPartsRequest, BackgroundHealStatusRequest, BatchGenerallyLockRequest, CancelDecommissionRequest, CheckPartsRequest,
ClearDecommissionRequest, ControlPlaneErrorCode, DeleteBucketMetadataRequest, DeleteBucketRequest, DeletePathsRequest, ClearDecommissionRequest, ControlPlaneErrorCode, DeleteBucketMetadataRequest, DeleteBucketRequest, DeletePathsRequest,
@@ -5990,6 +6066,74 @@ mod tests {
); );
} }
fn scoped_dirty_usage_request() -> rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageAckRequest {
rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageAckRequest {
challenge: vec![7; 16].into(),
protocol_version: 1,
owner_id: "11111111-1111-1111-1111-111111111111".into(),
instance_id: "a".repeat(32),
scope: 1,
probe_only: false,
entries: vec![rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry {
bucket: "photos".into(),
bucket_incarnation: vec![1; 16].into(),
generation: 8,
}],
}
}
#[tokio::test]
async fn scoped_dirty_usage_authenticates_before_storage_and_rejects_tampering() {
use rustfs_protos::scoped_dirty_usage::canonical_scoped_dirty_usage_request;
let service = create_test_node_service();
let unsigned = service
.scanner_scoped_dirty_usage_ack(Request::new(scoped_dirty_usage_request()))
.await
.expect_err("unsigned ACK must not access storage");
assert_eq!(unsigned.code(), tonic::Code::PermissionDenied);
for field in 0..9 {
let mut signed = Request::new(scoped_dirty_usage_request());
let canonical = canonical_scoped_dirty_usage_request(signed.get_ref()).expect("canonical request");
set_tonic_canonical_body_digest(&mut signed, &canonical).expect("digest");
mark_v2_authenticated(&mut signed);
match field {
0 => signed.get_mut().challenge = vec![3; 16].into(),
1 => signed.get_mut().owner_id = "22222222-2222-2222-2222-222222222222".into(),
2 => signed.get_mut().instance_id = "b".repeat(32),
3 => signed.get_mut().probe_only = true,
4 => signed.get_mut().entries[0].bucket = "videos".into(),
5 => signed.get_mut().entries[0].bucket_incarnation = vec![2; 16].into(),
6 => signed.get_mut().entries[0].generation += 1,
7 => signed.get_mut().scope += 1,
_ => signed.get_mut().protocol_version += 1,
}
let error = service
.scanner_scoped_dirty_usage_ack(signed)
.await
.expect_err("tampered ACK must fail");
assert_eq!(
error.code(),
if field < 7 {
tonic::Code::PermissionDenied
} else {
tonic::Code::InvalidArgument
}
);
}
let mut signed = Request::new(scoped_dirty_usage_request());
let canonical = canonical_scoped_dirty_usage_request(signed.get_ref()).expect("canonical request");
set_tonic_canonical_body_digest(&mut signed, &canonical).expect("digest");
mark_v2_authenticated(&mut signed);
assert_eq!(
service
.scanner_scoped_dirty_usage_ack(signed)
.await
.expect_err("missing owner cannot advertise capability")
.code(),
tonic::Code::Unavailable
);
}
#[tokio::test] #[tokio::test]
async fn test_scanner_activity_requires_body_bound_auth_before_storage_lookup() { async fn test_scanner_activity_requires_body_bound_auth_before_storage_lookup() {
let service = create_test_node_service(); let service = create_test_node_service();
@@ -6485,6 +6629,62 @@ mod tests {
) )
} }
#[tokio::test]
async fn scoped_dirty_usage_transport_rejects_oversized_unknown_and_duplicate_fields() {
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("bind scoped ACK transport test");
let addr = listener.local_addr().expect("test listener address");
let (shutdown, stopped) = tokio::sync::oneshot::channel();
let server = tokio::spawn(async move {
tonic::transport::Server::builder()
.add_service(super::make_scanner_control_server())
.serve_with_incoming_shutdown(TcpListenerStream::new(listener), async {
let _ = stopped.await;
})
.await
.expect("scoped ACK transport server");
});
let client = reqwest::Client::builder()
.no_proxy()
.http2_prior_knowledge()
.build()
.expect("HTTP/2 client");
let limit = rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize;
for tag in [0x78, 0x0a] {
// Unknown varint field 15, or repeated empty singular challenge:
// both decode to a tiny default struct despite the large wire body.
for oversized in [false, true] {
let mut payload = [tag, 0].repeat(if oversized { (limit - 4) / 2 } else { limit / 2 });
if oversized {
// Unknown fixed32 field 15 makes a valid cap+1 protobuf.
payload.extend_from_slice(&[0x7d, 0, 0, 0, 0]);
}
assert_eq!(payload.len(), limit + usize::from(oversized));
let mut frame = vec![0];
frame.extend_from_slice(&u32::try_from(payload.len()).expect("bounded test payload").to_be_bytes());
frame.extend_from_slice(&payload);
let response = client
.post(format!("http://{addr}/node_service.ScannerControlService/ScannerScopedDirtyUsageAck"))
.header("content-type", "application/grpc")
.header("te", "trailers")
.body(frame)
.send()
.await
.expect("send raw protobuf frame");
let status = response.headers().get("grpc-status").expect("gRPC failure status");
assert_eq!(
status.to_str().expect("status text"),
if oversized { "11" } else { "3" },
"cap+1 must fail in the codec, while cap bytes reach request validation"
);
}
}
drop(client);
shutdown.send(()).expect("stop test server");
server.await.expect("join test server");
}
#[tokio::test] #[tokio::test]
async fn heal_control_transport_enforces_codec_limit_and_fails_closed() { async fn heal_control_transport_enforces_codec_limit_and_fails_closed() {
let Some(mut client) = connect_test_heal_control_client().await else { let Some(mut client) = connect_test_heal_control_client().await else {
+9 -1
View File
@@ -379,7 +379,7 @@ pub(crate) mod tonic_service_consumer {
#[cfg(test)] #[cfg(test)]
pub(crate) use super::super::tonic_service::{heal_topology_fingerprint, make_heal_control_server_for_source}; pub(crate) use super::super::tonic_service::{heal_topology_fingerprint, make_heal_control_server_for_source};
pub(crate) use super::super::tonic_service::{ pub(crate) use super::super::tonic_service::{
make_heal_control_server_with_cache, make_server, make_tier_mutation_control_server, make_heal_control_server_with_cache, make_scanner_control_server, make_server, make_tier_mutation_control_server,
}; };
} }
@@ -1704,6 +1704,14 @@ pub(crate) async fn acquire_bucket_metadata_transaction_lock(
ecstore_bucket::metadata_sys::acquire_bucket_metadata_transaction_lock(bucket).await ecstore_bucket::metadata_sys::acquire_bucket_metadata_transaction_lock(bucket).await
} }
pub(crate) async fn acquire_scanner_bucket_incarnation_fence(
bucket: &str,
incarnation: uuid::Uuid,
owner_id: uuid::Uuid,
) -> Result<ecstore_bucket::metadata_sys::BucketMetadataMutationGuard> {
ecstore_bucket::metadata_sys::acquire_scanner_bucket_incarnation_fence(bucket, incarnation, owner_id).await
}
pub(crate) async fn update_bucket_targets_under_transaction_lock( pub(crate) async fn update_bucket_targets_under_transaction_lock(
guard: &ecstore_bucket::metadata_sys::BucketMetadataMutationGuard, guard: &ecstore_bucket::metadata_sys::BucketMetadataMutationGuard,
bucket: &str, bucket: &str,
+1
View File
@@ -13,6 +13,7 @@
// limitations under the License. // limitations under the License.
pub(crate) use crate::storage::rpc::node_service::make_heal_control_server_with_cache; pub(crate) use crate::storage::rpc::node_service::make_heal_control_server_with_cache;
pub(crate) use crate::storage::rpc::node_service::make_scanner_control_server;
#[cfg(test)] #[cfg(test)]
pub(crate) use crate::storage::rpc::node_service::{heal::heal_topology_fingerprint, make_heal_control_server_for_source}; pub(crate) use crate::storage::rpc::node_service::{heal::heal_topology_fingerprint, make_heal_control_server_for_source};
pub use crate::storage::rpc::{make_heal_control_server, make_server, make_tier_mutation_control_server}; pub use crate::storage::rpc::{make_heal_control_server, make_server, make_tier_mutation_control_server};
+3 -3
View File
@@ -176,7 +176,7 @@ pub(crate) mod server {
heal_topology_fingerprint, make_heal_control_server_for_source, heal_topology_fingerprint, make_heal_control_server_for_source,
}; };
pub(crate) use crate::storage::storage_api::tonic_service_consumer::{ pub(crate) use crate::storage::storage_api::tonic_service_consumer::{
make_heal_control_server_with_cache, make_server, make_tier_mutation_control_server, make_heal_control_server_with_cache, make_scanner_control_server, make_server, make_tier_mutation_control_server,
}; };
} }
} }
@@ -244,8 +244,8 @@ pub(crate) mod site_replication {
pub(crate) use crate::storage::storage_api::{Endpoint, Endpoints, PoolEndpoints}; pub(crate) use crate::storage::storage_api::{Endpoint, Endpoints, PoolEndpoints};
pub(crate) use crate::storage::storage_api::{ pub(crate) use crate::storage::storage_api::{
ECStore, EndpointServerPools, StorageError, delete_config_no_lock, lock_bucket_targets_metadata, read_config, ECStore, EndpointServerPools, StorageError, delete_config_no_lock, is_err_bucket_not_found, lock_bucket_targets_metadata,
read_config_no_lock, save_config_no_lock, with_config_object_read_lock, with_config_object_write_lock, read_config, read_config_no_lock, save_config_no_lock, with_config_object_read_lock, with_config_object_write_lock,
}; };
pub(crate) mod metadata_sys { pub(crate) mod metadata_sys {
+292 -106
View File
@@ -1,10 +1,11 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
"""Fail when a critical scheduled validation has not started recently.""" """Require recent scheduled attempts and completed successes on the default branch."""
from __future__ import annotations from __future__ import annotations
import argparse import argparse
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
import io
import json import json
import os import os
from pathlib import Path from pathlib import Path
@@ -13,7 +14,7 @@ import sys
import tempfile import tempfile
import unittest import unittest
from unittest import mock from unittest import mock
from urllib.parse import quote, urlencode from urllib.parse import parse_qs, quote, urlencode, urlsplit
from urllib.request import Request, urlopen from urllib.request import Request, urlopen
@@ -75,15 +76,8 @@ def stale_reason(
run: dict[str, object] | None, run: dict[str, object] | None,
now: datetime, now: datetime,
max_age_hours: int, max_age_hours: int,
never_ran_grace_until: datetime | None = None,
) -> str | None: ) -> str | None:
if run is None: if run is None:
# The grace deadline only covers a workflow whose first scheduled slot
# has not arrived yet (for example a monthly cron enabled mid-month).
# A recorded-but-old run proves the schedule used to fire and stopped,
# so the grace never masks that case.
if never_ran_grace_until is not None and now <= never_ran_grace_until:
return None
return "no scheduled run has been recorded" return "no scheduled run has been recorded"
created_at = parse_timestamp(run.get("created_at")) created_at = parse_timestamp(run.get("created_at"))
age = now - created_at age = now - created_at
@@ -93,14 +87,23 @@ def stale_reason(
def fetch_latest_scheduled_run( def fetch_latest_scheduled_run(
repository: str, workflow: str, token: str, api_url: str repository: str,
workflow: str,
token: str,
api_url: str,
default_branch: str,
successful: bool = False,
) -> dict[str, object] | None: ) -> dict[str, object] | None:
owner, repo = repository.split("/", 1) owner, repo = repository.split("/", 1)
workflow_name = Path(workflow).name workflow_name = Path(workflow).name
query = {"event": "schedule", "branch": default_branch, "per_page": 1}
if successful:
# Filter on the server: the last success may be beyond a page of failures.
query["status"] = "success"
endpoint = ( endpoint = (
f"{api_url.rstrip('/')}/repos/{quote(owner, safe='')}/{quote(repo, safe='')}" f"{api_url.rstrip('/')}/repos/{quote(owner, safe='')}/{quote(repo, safe='')}"
f"/actions/workflows/{quote(workflow_name, safe='')}/runs?" f"/actions/workflows/{quote(workflow_name, safe='')}/runs?"
+ urlencode({"event": "schedule", "per_page": 1}) + urlencode(query)
) )
request = Request( request = Request(
endpoint, endpoint,
@@ -110,57 +113,104 @@ def fetch_latest_scheduled_run(
"X-GitHub-Api-Version": "2022-11-28", "X-GitHub-Api-Version": "2022-11-28",
}, },
) )
with urlopen(request, timeout=30) as response: # Two requests per manifest entry must fit the watchdog's ten-minute job.
with urlopen(request, timeout=15) as response:
payload = json.load(response) payload = json.load(response)
runs = payload.get("workflow_runs") runs = payload.get("workflow_runs") if isinstance(payload, dict) else None
if not isinstance(runs, list): if not isinstance(runs, list):
raise ValueError(f"GitHub returned no workflow_runs list for {workflow}") raise ValueError(f"GitHub returned no workflow_runs list for {workflow}")
total_count = payload.get("total_count")
if not isinstance(total_count, int) or isinstance(total_count, bool) or total_count < len(runs):
raise ValueError(f"GitHub returned an invalid run count for {workflow}")
if not runs: if not runs:
if total_count:
raise ValueError(f"GitHub returned an empty first page with recorded runs for {workflow}")
return None return None
if not isinstance(runs[0], dict): run = runs[0]
if not isinstance(run, dict):
raise ValueError(f"GitHub returned an invalid workflow run for {workflow}") raise ValueError(f"GitHub returned an invalid workflow run for {workflow}")
return runs[0] if run.get("event") != "schedule" or run.get("head_branch") != default_branch:
raise ValueError(f"GitHub returned a run outside the scheduled default-branch query for {workflow}")
if not isinstance(run.get("status"), str) or not run["status"]:
raise ValueError(f"GitHub returned no run status for {workflow}")
conclusion = run.get("conclusion")
if (conclusion is not None and not isinstance(conclusion, str)) or (
run["status"] == "completed" and not conclusion
):
raise ValueError(f"GitHub returned an invalid run conclusion for {workflow}")
if successful and (run["status"] != "completed" or conclusion != "success"):
raise ValueError(f"GitHub returned a run without a completed success for {workflow}")
parse_timestamp(run.get("created_at"))
if not isinstance(run.get("html_url"), str) or not run["html_url"]:
raise ValueError(f"GitHub returned no run URL for {workflow}")
return run
def write_report(path: Path, failures: list[tuple[str, int, str, str]]) -> None: def describe_run(run: dict[str, object] | None) -> str:
lines = ["## Scheduled validation freshness"] if run is None:
if not failures: return "No recorded run"
lines.append("") outcome = run["status"]
lines.append("All critical scheduled validations have a recent scheduled run.") if run.get("conclusion"):
else: outcome = f"{outcome}/{run['conclusion']}"
lines.extend( return f"[{outcome}]({run['html_url']}) — created {run['created_at']}"
[
"",
"The following critical validations are stale or could not be inspected:", def write_report(path: Path, rows: list[tuple[str, int, str, str, str]], default_branch: str) -> None:
"", lines = [
"| Workflow | Limit | Result | Last run |", "## Scheduled validation freshness",
"| --- | ---: | --- | --- |", "",
] f"Default branch: `{default_branch}`. Ages use scheduled-run creation time; rerunning an old commit does not refresh its evidence.",
) "Attempt outcomes are shown independently of successful-run freshness.",
for workflow, max_age_hours, reason, run_url in failures: "Success is the GitHub workflow run conclusion; suite completeness remains the responsibility of each workflow.",
link = f"[open]({run_url})" if run_url else "" "",
lines.append(f"| `{workflow}` | {max_age_hours}h | {reason} | {link} |") "| Workflow | Limit | Freshness | Last attempt | Last completed success |",
"| --- | ---: | --- | --- | --- |",
]
for workflow, max_age_hours, result, attempt, success in rows:
cells = [f"`{workflow}`", f"{max_age_hours}h", result, attempt, success]
lines.append("| " + " | ".join(cell.replace("|", "\\|").replace("\n", " ") for cell in cells) + " |")
path.write_text("\n".join(lines) + "\n") path.write_text("\n".join(lines) + "\n")
def check_freshness( def check_freshness(
config: Path, report: Path, repository: str, token: str, api_url: str config: Path, report: Path, repository: str, token: str, api_url: str, default_branch: str
) -> int: ) -> int:
now = datetime.now(timezone.utc) now = datetime.now(timezone.utc)
failures: list[tuple[str, int, str, str]] = [] rows: list[tuple[str, int, str, str, str]] = []
failed = False
for workflow, max_age_hours, never_ran_grace_until in load_validations(config): for workflow, max_age_hours, never_ran_grace_until in load_validations(config):
try: runs: dict[str, dict[str, object] | None] = {}
run = fetch_latest_scheduled_run(repository, workflow, token, api_url) reasons: list[str] = []
reason = stale_reason(run, now, max_age_hours, never_ran_grace_until) for label, successful in (("Last attempt", False), ("Last completed success", True)):
if reason is not None: try:
run_url = str(run.get("html_url", "")) if run else "" runs[label] = fetch_latest_scheduled_run(
failures.append((workflow, max_age_hours, reason, run_url)) repository, workflow, token, api_url, default_branch, successful
except Exception as error: )
failures.append( except Exception as error:
(workflow, max_age_hours, f"inspection failed: {error}", "") reasons.append(f"{label}: inspection failed: {error}")
) # A failed inspection or any recorded attempt ends first-run grace.
write_report(report, failures) initial_grace = (
return 1 if failures else 0 len(runs) == 2
and all(run is None for run in runs.values())
and never_ran_grace_until is not None
and now <= never_ran_grace_until
)
if not initial_grace:
for label, run in runs.items():
reason = stale_reason(run, now, max_age_hours)
if reason is not None:
reasons.append(f"{label}: {reason}")
failed |= bool(reasons)
result = "; ".join(reasons) if reasons else "Fresh"
if initial_grace:
result = f"Initial grace until {never_ran_grace_until.isoformat()}"
evidence = [
describe_run(runs[label]) if label in runs else "Inspection failed"
for label in ("Last attempt", "Last completed success")
]
rows.append((workflow, max_age_hours, result, *evidence))
write_report(report, rows, default_branch)
return 1 if failed else 0
class SelfTests(unittest.TestCase): class SelfTests(unittest.TestCase):
@@ -173,15 +223,6 @@ class SelfTests(unittest.TestCase):
self.assertIsNotNone(stale_reason(past_limit, self.NOW, 36)) self.assertIsNotNone(stale_reason(past_limit, self.NOW, 36))
self.assertIsNotNone(stale_reason(None, self.NOW, 36)) self.assertIsNotNone(stale_reason(None, self.NOW, 36))
def test_never_ran_grace_only_covers_missing_runs(self) -> None:
future_grace = self.NOW + timedelta(hours=1)
past_grace = self.NOW - timedelta(seconds=1)
self.assertIsNone(stale_reason(None, self.NOW, 36, future_grace))
self.assertIsNone(stale_reason(None, self.NOW, 36, self.NOW))
self.assertIsNotNone(stale_reason(None, self.NOW, 36, past_grace))
stale_run = {"created_at": "2026-08-20T23:59:59Z"}
self.assertIsNotNone(stale_reason(stale_run, self.NOW, 36, future_grace))
def test_config_rejects_duplicate_and_invalid_entries(self) -> None: def test_config_rejects_duplicate_and_invalid_entries(self) -> None:
with tempfile.TemporaryDirectory() as tmp: with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "validations.json" path = Path(tmp) / "validations.json"
@@ -238,63 +279,205 @@ class SelfTests(unittest.TestCase):
], ],
) )
def test_check_reports_missing_runs(self) -> None: @staticmethod
def run_fixture(**overrides: object) -> dict[str, object]:
return {
"status": "completed",
"conclusion": "success",
"event": "schedule",
"head_branch": "release/current",
"created_at": "2026-08-22T00:00:00Z",
"html_url": "https://github.test/rustfs/rustfs/actions/runs/1",
**overrides,
}
def check_payloads(
self, payloads: list[object], *, grace: str | None = None, workflows: int = 1
) -> tuple[int, str, list]:
with tempfile.TemporaryDirectory() as tmp: with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp) root = Path(tmp)
config = root / "validations.json" config = root / "validations.json"
report = root / "report.md" report = root / "report.md"
config.write_text( entries = [
json.dumps( {"workflow": f".github/workflows/check-{index}.yml", "max_age_hours": 36}
[ for index in range(workflows)
{"workflow": ".github/workflows/ci.yml", "max_age_hours": 36}, ]
{"workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36}, if grace is not None:
{"workflow": ".github/workflows/mint.yml", "max_age_hours": 36}, entries[0]["never_ran_grace_until"] = grace
] config.write_text(json.dumps(entries))
) responses = []
) for payload in payloads:
with mock.patch( if isinstance(payload, dict) and isinstance(payload.get("workflow_runs"), list):
__name__ + ".fetch_latest_scheduled_run", payload = {"total_count": len(payload["workflow_runs"]), **payload}
side_effect=[ responses.append(payload if isinstance(payload, Exception) else io.StringIO(json.dumps(payload)))
{"created_at": "2999-01-01T00:00:00Z"}, with (
None, mock.patch(__name__ + ".urlopen", side_effect=responses) as request,
RuntimeError("API unavailable"), mock.patch(__name__ + ".datetime", wraps=datetime) as clock,
],
): ):
self.assertEqual( clock.now.return_value = self.NOW
check_freshness( status = check_freshness(
config, config, report, "rustfs/rustfs", "test-token",
report, "https://api.github.test", "release/current",
"rustfs/rustfs",
"token",
"https://api.github.test",
),
1,
) )
contents = report.read_text() return status, report.read_text(), request.call_args_list
self.assertIn(".github/workflows/fuzz.yml", contents)
self.assertIn("inspection failed: API unavailable", contents)
self.assertNotIn(".github/workflows/ci.yml`", contents)
config.write_text( def test_requests_filter_schedule_default_branch_and_success_on_server(self) -> None:
json.dumps( attempt = self.run_fixture(status="in_progress", conclusion=None)
[{"workflow": ".github/workflows/ci.yml", "max_age_hours": 36}] success = self.run_fixture(html_url="https://github.test/rustfs/rustfs/actions/runs/2")
status, report, calls = self.check_payloads([
{"workflow_runs": [attempt], "total_count": 1001},
{"workflow_runs": [success], "total_count": 1},
])
self.assertEqual(status, 0)
self.assertEqual(len(calls), 2)
for call, successful in zip(calls, (False, True)):
request = call.args[0]
url = urlsplit(request.full_url)
self.assertEqual(url.path, "/repos/rustfs/rustfs/actions/workflows/check-0.yml/runs")
expected = {"event": ["schedule"], "branch": ["release/current"], "per_page": ["1"]}
if successful:
expected["status"] = ["success"]
self.assertEqual(parse_qs(url.query), expected)
self.assertEqual(request.get_header("Authorization"), "Bearer test-token")
self.assertEqual(call.kwargs, {"timeout": 15})
self.assertIn("[in_progress]", report)
self.assertIn(str(attempt["html_url"]), report)
self.assertIn(str(success["html_url"]), report)
def test_cancelled_attempt_cannot_refresh_expired_success(self) -> None:
attempt = self.run_fixture(conclusion="cancelled")
success = self.run_fixture(
created_at="2026-08-20T23:59:59Z", updated_at="2026-08-22T11:59:59Z",
html_url="https://github.test/rustfs/rustfs/actions/runs/2",
)
status, report, _ = self.check_payloads([
{"workflow_runs": [attempt]}, {"workflow_runs": [success]},
])
self.assertEqual(status, 1)
self.assertIn("Last completed success: last scheduled run is", report)
self.assertIn("[completed/cancelled]", report)
for run in (attempt, success):
self.assertIn(str(run["html_url"]), report)
self.assertIn(str(run["created_at"]), report)
def test_attempt_outcome_does_not_replace_recent_success(self) -> None:
success = self.run_fixture(created_at="2026-08-21T00:00:00Z")
for state, conclusion in (
("completed", "failure"), ("completed", "cancelled"),
("completed", "timed_out"), ("completed", "success"),
("queued", None), ("in_progress", None),
):
with self.subTest(state=state, conclusion=conclusion):
status, report, _ = self.check_payloads([
{"workflow_runs": [self.run_fixture(status=state, conclusion=conclusion)]},
{"workflow_runs": [success]},
])
self.assertEqual(status, 0)
self.assertIn(f"[{state}" + (f"/{conclusion}" if conclusion else "") + "]", report)
self.assertIn("Fresh", report)
self.assertNotIn("All critical scheduled validations", report)
def test_grace_requires_two_successful_queries_with_no_history(self) -> None:
for attempt, success, grace, expected in (
(None, None, "2026-08-22T12:00:00Z", 0),
(None, None, "2026-08-22T11:59:59Z", 1),
(self.run_fixture(conclusion="failure"), None, "2026-08-23T00:00:00Z", 1),
(self.run_fixture(status="queued", conclusion=None), None, "2026-08-23T00:00:00Z", 1),
(None, self.run_fixture(), "2026-08-23T00:00:00Z", 1),
):
with self.subTest(attempt=attempt, success=success, grace=grace):
status, report, _ = self.check_payloads([
{"workflow_runs": [] if attempt is None else [attempt]},
{"workflow_runs": [] if success is None else [success]},
], grace=grace)
self.assertEqual(status, expected)
self.assertEqual("Initial grace until" in report, expected == 0)
def test_api_failures_preserve_other_evidence_and_never_enter_grace(self) -> None:
good = {"workflow_runs": [self.run_fixture()]}
for first, second in (
(RuntimeError("API unavailable"), good),
(good, RuntimeError("API unavailable")),
(RuntimeError("API unavailable"), {"workflow_runs": []}),
):
with self.subTest(first=first, second=second):
status, report, calls = self.check_payloads(
[first, second], grace="2026-08-23T00:00:00Z"
) )
) self.assertEqual(status, 1)
with mock.patch( self.assertEqual(len(calls), 2)
__name__ + ".fetch_latest_scheduled_run", self.assertIn("inspection failed: API unavailable", report)
return_value={"created_at": "2999-01-01T00:00:00Z"}, self.assertNotIn("Initial grace until", report)
): if first is good or second is good:
self.assertEqual( self.assertIn(str(self.run_fixture()["html_url"]), report)
check_freshness(
config, def test_invalid_api_evidence_fails_closed(self) -> None:
report, malformed = [
"rustfs/rustfs", [], {}, {"workflow_runs": {}}, {"workflow_runs": [None]},
"token", {"workflow_runs": [], "total_count": 1},
"https://api.github.test", {"workflow_runs": [], "total_count": -1},
), {"workflow_runs": [], "total_count": None},
0, {"workflow_runs": [], "total_count": True},
) *({"workflow_runs": [self.run_fixture(**override)]} for override in (
self.assertIn("All critical scheduled validations", report.read_text()) {"event": "workflow_dispatch"}, {"head_branch": "other"},
{"created_at": "invalid"}, {"created_at": "2026-08-22T00:00:00"},
{"status": None}, {"conclusion": None}, {"conclusion": 1},
{"html_url": ""},
)),
]
for payload in malformed:
for index, label in enumerate(("Last attempt", "Last completed success")):
with self.subTest(payload=payload, label=label):
payloads = [{"workflow_runs": [self.run_fixture()]} for _ in range(2)]
payloads[index] = payload
status, report, _ = self.check_payloads(payloads, grace="2026-08-23T00:00:00Z")
self.assertEqual(status, 1)
self.assertIn(f"{label}: inspection failed", report)
self.assertNotIn("Initial grace until", report)
self.assertIn(str(self.run_fixture()["html_url"]), report)
for state, conclusion in (("in_progress", "success"), ("completed", "failure"), ("completed", "skipped")):
with self.subTest(state=state, conclusion=conclusion):
status, report, _ = self.check_payloads([
{"workflow_runs": [self.run_fixture()]},
{"workflow_runs": [self.run_fixture(status=state, conclusion=conclusion)]},
])
self.assertEqual(status, 1)
self.assertIn("without a completed success", report)
def test_report_retains_every_workflow(self) -> None:
status, report, calls = self.check_payloads([
{"workflow_runs": [self.run_fixture()]}, {"workflow_runs": [self.run_fixture()]},
{"workflow_runs": []}, {"workflow_runs": []},
RuntimeError("API unavailable"), {"workflow_runs": [self.run_fixture()]},
], workflows=3)
self.assertEqual(status, 1)
self.assertEqual(len(calls), 6)
for index in range(3):
self.assertEqual(report.count(f"`.github/workflows/check-{index}.yml`"), 1)
self.assertIn("No recorded run", report)
self.assertIn("Inspection failed", report)
def test_cli_requires_the_repository_default_branch(self) -> None:
from check_test_wiring import yaml_block
workflow = (ROOT / ".github/workflows/scheduled-validation-freshness.yml").read_text().splitlines()
job = yaml_block(workflow, "check-freshness", 2)
self.assertIsNotNone(job)
start = job.index(" - name: Check latest scheduled runs")
end = next((index for index in range(start + 1, len(job)) if job[index].startswith(" - ")), len(job))
environment = yaml_block(job[start:end], "env", 8)
self.assertIsNotNone(environment)
self.assertIn(" RUSTFS_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}", environment)
with (
mock.patch.dict(os.environ, {"GITHUB_REPOSITORY": "rustfs/rustfs", "GH_TOKEN": "test-token"}, clear=True),
mock.patch.object(sys, "argv", ["checker", "--report", "unused.md"]),
mock.patch("sys.stderr", new=io.StringIO()) as stderr,
self.assertRaises(SystemExit) as error,
):
main()
self.assertEqual(error.exception.code, 2)
self.assertIn("RUSTFS_DEFAULT_BRANCH", stderr.getvalue())
def main() -> int: def main() -> int:
@@ -318,11 +501,14 @@ def main() -> int:
repository = os.environ.get("GITHUB_REPOSITORY", "") repository = os.environ.get("GITHUB_REPOSITORY", "")
token = os.environ.get("GH_TOKEN", "") token = os.environ.get("GH_TOKEN", "")
api_url = os.environ.get("GITHUB_API_URL", "https://api.github.com") api_url = os.environ.get("GITHUB_API_URL", "https://api.github.com")
default_branch = os.environ.get("RUSTFS_DEFAULT_BRANCH", "")
if not re.fullmatch(r"[^/\s]+/[^/\s]+", repository): if not re.fullmatch(r"[^/\s]+/[^/\s]+", repository):
parser.error("GITHUB_REPOSITORY must be owner/repository") parser.error("GITHUB_REPOSITORY must be owner/repository")
if not token: if not token:
parser.error("GH_TOKEN is required") parser.error("GH_TOKEN is required")
return check_freshness(args.config, args.report, repository, token, api_url) if not default_branch or any(character.isspace() for character in default_branch):
parser.error("RUSTFS_DEFAULT_BRANCH is required and must name the repository default branch")
return check_freshness(args.config, args.report, repository, token, api_url, default_branch)
if __name__ == "__main__": if __name__ == "__main__":
+197
View File
@@ -0,0 +1,197 @@
#!/usr/bin/env python3
"""Run the security workflow's evidence and result steps without remote VMs."""
from __future__ import annotations
import os
import re
import subprocess
import tempfile
import unittest
from pathlib import Path
from check_test_wiring import yaml_block
ROOT = Path(__file__).resolve().parents[1]
WORKFLOW = ROOT / ".github/workflows/rustfs-security-test.yml"
CASE_ROW = "| IAM-101 | user CRUD lifecycle | PASS |"
class SecurityWorkflowTests(unittest.TestCase):
def setUp(self) -> None:
self.source = WORKFLOW.read_text()
self.job = yaml_block(self.source.splitlines(), "security-test", 2)
self.assertIsNotNone(self.job)
starts = [i for i, line in enumerate(self.job) if line.startswith(" - name: ")]
self.steps = {
self.job[start].split(": ", 1)[1].strip('"'): self.job[start:end]
for start, end in zip(starts, starts[1:] + [len(self.job)])
}
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.directory = Path(self.temp.name)
self.context = {
"runner.temp": self.temp.name,
"github.server_url": "https://github.com",
"github.repository": "rustfs/rustfs",
"github.run_id": "314159",
"github.run_attempt": "2",
"github.sha": "0123456789abcdef0123456789abcdef01234567",
"github.event_name": "workflow_dispatch",
"github.workspace": self.temp.name,
"inputs.package_url": "",
"inputs.rustfs_version": "test-version",
"inputs.topology": "all",
"inputs.oidc_live": "false",
"steps.evidence.outcome": "skipped",
"steps.test.outcome": "skipped",
"steps.report.outcome": "skipped",
}
self.env = {
**os.environ, "GITHUB_STEP_SUMMARY": str(self.directory / "summary.md"),
"GITHUB_ENV": str(self.directory / "github-env"), "RUNNER_TEMP": self.temp.name, "TMPDIR": self.temp.name,
}
for key in ("server_url", "repository", "run_id", "run_attempt", "sha", "event_name"):
self.env[f"GITHUB_{key.upper()}"] = self.context[f"github.{key}"]
self.context["env.SECURITY_ARTIFACTS_DIR"] = ""
self.artifacts = self.directory / "rustfs-security-314159-2"
suite = self.directory / "auto-testing/rustfs-security-test.sh"
suite.parent.mkdir()
suite.write_text(
'#!/usr/bin/env bash\nset -euo pipefail\n'
'log_dir=$(mktemp -d "$TMPDIR/rustfs-security.XXXXXX")\n'
'echo "CURRENT SUITE LOG" > "$log_dir/suite.log"\n'
'case "$FAKE_REPORT" in\n'
f' present) printf "%s\\n" "CURRENT SUITE DIAGNOSTIC" "{CASE_ROW}" > "$REPORT_FILE" ;;\n'
' empty) : > "$REPORT_FILE" ;;\n'
'esac\n'
'echo "UNWRAPPED SUITE SUMMARY" >> "$GITHUB_STEP_SUMMARY"\n'
'exit "$FAKE_EXIT"\n'
)
def render(self, value: str) -> str:
return re.sub(r"\$\{\{\s*(.*?)\s*\}\}", lambda match: self.context[match[1]], value)
def step_env(self, lines: list[str], indent: int = 8) -> dict[str, str]:
result = {}
for line in yaml_block(lines, "env", indent) or []:
if line.strip() and not line.lstrip().startswith("#"):
key, value = line.strip().split(": ", 1)
result[key] = self.render(value.strip("'\""))
return result
def run_step(self, name: str) -> subprocess.CompletedProcess[str]:
lines = self.steps[name]
start = lines.index(" run: |") + 1
shell_lines = []
for line in lines[start:]:
if line.strip() and not line.startswith(" "):
break
shell_lines.append(line[10:])
self.assertTrue(shell_lines, f"missing literal shell body: {name}")
result = subprocess.run(
["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.render("\n".join(shell_lines))],
cwd=self.directory, env={**self.env, **self.step_env(lines)}, capture_output=True, text=True,
)
for line in lines:
if line.startswith(" id: "):
self.context[f"steps.{line.split(': ', 1)[1]}.outcome"] = "failure" if result.returncode else "success"
if Path(self.env["GITHUB_ENV"]).exists():
for line in Path(self.env["GITHUB_ENV"]).read_text().splitlines():
key, value = line.split("=", 1)
self.env[key] = value
self.context[f"env.{key}"] = value
return result
def test_workflow_wiring(self) -> None:
names = list(self.steps)
self.assertLess(names.index("Checkout repository (for the OIDC live gate script)"), names.index("Checkout auto-testing scripts (with retry)"))
self.assertNotIn(" continue-on-error: true", self.job)
self.assertIn(" continue-on-error: true", self.steps["Run security suite"])
for name in ("Initialize security evidence", "Generate report"):
self.assertNotIn(" continue-on-error: true", self.steps[name])
self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", self.steps["Generate report"])
self.assertNotIn("/tmp/rustfs-security", self.source)
for name in ("Upload functional report to dashboard", "File failure issue in rustfs/backlog"):
report = next(line for line in self.steps[name] if line.strip().startswith("REPORT_FILE:"))
self.assertIn("${{ env.SECURITY_ARTIFACTS_DIR }}/report.md", report)
for name in ("Upload functional report to dashboard", "Upload report and logs"):
self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", self.steps[name])
artifact_settings = yaml_block(self.steps["Upload report and logs"], "with", 8)
self.assertIn(" path: ${{ env.SECURITY_ARTIFACTS_DIR }}/", artifact_settings)
self.assertIn(" if-no-files-found: error", artifact_settings)
def test_suite_report_and_result_matrix(self) -> None:
for outcome, mode, exit_code in (
("success", "present", 0), ("failure", "present", 7), ("failure", "missing", 7),
("success", "missing", 0), ("success", "empty", 0),
("skipped", "missing", 0), ("skipped", "present", 0),
("cancelled", "missing", 0), ("cancelled", "present", 0),
):
with self.subTest(outcome=outcome, report=mode):
self.setUp()
initialized = self.run_step("Initialize security evidence")
self.assertEqual(initialized.returncode, 0, initialized.stderr)
self.assertEqual(self.env["SECURITY_ARTIFACTS_DIR"], str(self.artifacts))
self.env.update(FAKE_REPORT=mode, FAKE_EXIT=str(exit_code))
if outcome != "skipped" or mode == "present":
suite = self.run_step("Run security suite")
self.assertEqual(suite.returncode, exit_code, suite.stderr)
logs = list(self.artifacts.glob("rustfs-security.*/suite.log"))
self.assertEqual(len(logs), 1)
self.assertEqual(logs[0].read_text(), "CURRENT SUITE LOG\n")
self.context["steps.test.outcome"] = outcome
report = self.run_step("Generate report")
success = outcome == "success" and mode == "present"
self.assertEqual(report.returncode == 0, success, report.stderr)
contents = (self.artifacts / "report.md").read_text()
for expected in (
"https://github.com/rustfs/rustfs/actions/runs/314159", "Attempt: 2",
f"Workflow Commit: {self.context['github.sha']}", "Trigger: workflow_dispatch",
f"Test Step Outcome: {'success' if success else 'failure'}", f"Suite Step Outcome: {outcome}",
):
self.assertIn(expected, contents)
self.assertEqual(CASE_ROW in contents, success)
self.assertEqual("CURRENT SUITE DIAGNOSTIC" in contents, success)
if mode == "present":
raw = (self.artifacts / "suite-report.md").read_text()
self.assertEqual(raw, f"CURRENT SUITE DIAGNOSTIC\n{CASE_ROW}\n")
summary = Path(self.env["GITHUB_STEP_SUMMARY"]).read_text()
self.assertEqual(summary, contents)
self.assertNotIn("UNWRAPPED SUITE SUMMARY", summary)
def test_existing_evidence_directory_is_rejected(self) -> None:
self.artifacts.mkdir()
stale = self.artifacts / "suite-report.md"
stale.write_text("OLD RUN REPORT")
self.assertNotEqual(self.run_step("Initialize security evidence").returncode, 0)
self.assertEqual(stale.read_text(), "OLD RUN REPORT")
self.assertFalse(Path(self.env["GITHUB_ENV"]).exists())
(self.artifacts / "report.md").write_text("OLD RUN REPORT")
self.context.update({
"env.SECURITY_ARTIFACTS_DIR": str(self.artifacts), "secrets.PF_TESTING_GH_TOKEN": "fake-local-token",
})
fake_bin = self.directory / "bin"
fake_bin.mkdir()
gh = fake_bin / "gh"
gh.write_text(
'#!/usr/bin/env bash\nset -euo pipefail\n'
'if [ "$1 $2" = "issue create" ]; then\n'
' while [ "$#" -gt 0 ]; do\n'
' if [ "$1" = "--body-file" ]; then cat "$2" > "$CAPTURE_BODY"; fi\n'
' shift\n'
' done\n'
'fi\n'
)
gh.chmod(0o755)
body = self.directory / "issue-body.md"
self.env.update(PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", CAPTURE_BODY=str(body))
result = self.run_step("File failure issue in rustfs/backlog")
self.assertEqual(result.returncode, 0, result.stderr)
self.assertNotIn("OLD RUN REPORT", body.read_text())
self.assertIn("https://github.com/rustfs/rustfs/actions/runs/314159", body.read_text())
if __name__ == "__main__":
unittest.main()