Compare commits

..

2 Commits

Author SHA1 Message Date
overtrue 635746534b test(ci): allow equal timestamps for parallel scanner probes 2026-11-21 06:48:16 +08:00
overtrue 6a94097b97 test(ci): make storage fixtures wait for their prerequisites 2026-11-21 06:37:09 +08:00
7 changed files with 21 additions and 168 deletions
@@ -5082,7 +5082,14 @@ mod tests {
let cert = generate_simple_self_signed(vec!["localhost".to_string()]).expect("certificate should generate");
let (port, server) = spawn_https_server(&cert, 2);
let endpoint = format!("https://localhost:{port}");
let strict = target_client_for_test("arn:strict", endpoint.clone(), s3_client_for_test(port, None));
let strict_client = s3_client_for_test(port, None);
// A failed trust check must consume just one of the two fixture connections.
let strict_config = strict_client
.config()
.to_builder()
.retry_config(aws_sdk_s3::config::retry::RetryConfig::disabled())
.build();
let strict = target_client_for_test("arn:strict", endpoint.clone(), S3Client::from_conf(strict_config));
let insecure = target_client_for_test(
"arn:insecure",
endpoint,
-88
View File
@@ -1635,94 +1635,6 @@ async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() {
global_metrics().set_cycle(None).await;
}
#[tokio::test]
#[serial]
async fn single_drive_cycle_persists_usage_and_advances_with_healing_marker() {
use crate::scanner_io::{clear_dirty_usage_buckets_for_tests, tests::wait_for_namespace_commit_tails};
use crate::storage_api::scan::{HEALING_MARKER_PATH, MakeBucketOptions};
clear_dirty_usage_buckets_for_tests();
let temp_dir = tempfile::tempdir().expect("scanner cycle test directory should be created");
let store = setup_scanner_cycle_store_at_path_with_layout(temp_dir.path(), true, 1, 1, 1).await;
let bucket = "marked-drive-bucket";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("fixture bucket should be created");
let mut reader = PutObjReader::from_vec(vec![7; 16]);
store
.put_object(bucket, "object", &mut reader, &ObjectOptions::default())
.await
.expect("fixture object should persist");
wait_for_namespace_commit_tails(store.as_ref()).await;
let marker = temp_dir
.path()
.join("pool0/disk0")
.join(RUSTFS_META_BUCKET)
.join(HEALING_MARKER_PATH);
let marker_contents = b"leftover marker";
tokio::fs::write(&marker, marker_contents)
.await
.expect("fixture healing marker should persist");
let set = &store.pools[0].disk_set[0];
tokio::time::timeout(Duration::from_secs(10), async {
while !set.get_online_disks_with_healing(true).await.1 {
tokio::time::sleep(Duration::from_millis(100)).await;
}
})
.await
.expect("disk info should observe the marker after its cache expires");
let ctx = CancellationToken::new();
let mut cycle_info = CurrentCycle {
next: 1,
..Default::default()
};
let mut revision = DataUsageCacheRevision::Missing;
let leader_epoch = 11;
let outcome = tokio::time::timeout(
Duration::from_secs(60),
run_data_scanner_cycle(&ctx, &store, &mut cycle_info, &mut revision, leader_epoch),
)
.await
.expect("scanner cycle should finish");
assert!(
matches!(
outcome,
ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance
),
"unexpected cycle outcome: {outcome:?}",
);
let mut set_snapshot = DataUsageCache::default();
set_snapshot
.load_with_revisions(Arc::clone(set), DATA_USAGE_CACHE_NAME)
.await
.expect("set usage snapshot should be readable");
assert!(set_snapshot.info.snapshot_complete);
assert_eq!(set_snapshot.info.next_cycle, 1);
let usage = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("global usage snapshot should be durable");
let usage: DataUsageInfo = serde_json::from_slice(&usage).expect("global usage snapshot should decode");
assert!(usage.usage_snapshot_complete);
assert!(usage.last_update.is_some());
assert_eq!(usage.buckets_usage[bucket].objects_count, 1);
let persisted_cycle = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
.await
.expect("scanner cycle cursor should be durable");
let (persisted_cycle, persisted_epoch) =
decode_scanner_cycle_state(&persisted_cycle).expect("scanner cycle cursor should decode");
assert_eq!(persisted_epoch, leader_epoch);
assert_eq!(persisted_cycle.next, 2);
assert_eq!(tokio::fs::read(marker).await.expect("healing marker should remain"), marker_contents);
clear_dirty_usage_buckets_for_tests();
global_metrics().set_cycle(None).await;
}
#[tokio::test]
#[serial]
async fn coordinator_publishes_authority_during_pending_put() {
+7 -20
View File
@@ -620,10 +620,7 @@ impl ScannerIOCache for SetDisks {
.ok_or_else(|| StorageError::other("failed to persist empty scanner set scope"));
}
let single_drive = self.set_drive_count == 1;
// A single-drive set has no alternate scan source. A heal marker
// still disables scanner heals, but must not exclude its only usage walk.
let (disks, healing) = self.get_online_disks_with_healing(single_drive).await;
let (disks, healing) = self.get_online_disks_with_healing(false).await;
#[cfg(test)]
let disks = super::tests::usage_remote_drain::walker_disks(&self, &buckets, disks);
if disks.is_empty() {
@@ -760,18 +757,7 @@ impl ScannerIOCache for SetDisks {
let _ = updates.send(incomplete_scope).await;
return Ok(ScannerSourceTurnReceipt::default());
}
// Coordinator workers need the full inventory only for cross-drive
// healing. Remote workers own their disk inventory on the peer.
let heal_disk_inventory = if !single_drive
&& !healing
&& workers
.iter()
.any(|(_, mode)| *mode == NamespaceScannerWorkerMode::Coordinator)
{
Some(Arc::new(scanner_set_disk_inventory(self.as_ref()).await))
} else {
None
};
let set_disk_inventory = Arc::new(scanner_set_disk_inventory(self.as_ref()).await);
if unsupported_remote_disks > 0 {
debug!(
target: "rustfs::scanner::io",
@@ -1024,6 +1010,7 @@ impl ScannerIOCache for SetDisks {
let store_clone_clone = self.clone();
let bucket_result_tx_clone = bucket_result_tx.clone();
let bucket_observations_clone = bucket_observations.clone();
let set_disk_inventory_clone = set_disk_inventory.clone();
let disk_scan_semaphore_clone = disk_scan_semaphore.clone();
let queued_disk_bucket_scans_clone = queued_disk_bucket_scans.clone();
let active_disk_bucket_scans_clone = active_disk_bucket_scans.clone();
@@ -1039,9 +1026,9 @@ impl ScannerIOCache for SetDisks {
let worker_retired_buckets = retired_buckets.clone();
#[cfg(feature = "e2e-test-hooks")]
let worker_incarnations = current_bucket_incarnations.clone();
let (remote_capabilities, heal_disk_inventory) = match worker_mode {
NamespaceScannerWorkerMode::Coordinator => (None, heal_disk_inventory.clone()),
NamespaceScannerWorkerMode::RemoteV4(capabilities) => (Some(capabilities), None),
let remote_capabilities = match worker_mode {
NamespaceScannerWorkerMode::RemoteV4(capabilities) => Some(capabilities),
NamespaceScannerWorkerMode::Coordinator => None,
};
#[cfg(test)]
let worker_index = futs.len();
@@ -1684,7 +1671,7 @@ impl ScannerIOCache for SetDisks {
let scan = crate::workload_admission::with_scanner_node_requests(node_tx, disk.clone().nsscanner_disk(
scan_ctx.clone(),
bucket_budget.clone(),
heal_disk_inventory.as_deref().cloned().unwrap_or_default(),
set_disk_inventory_clone.as_ref().clone(),
cache,
None,
ScannerDiskScanOptions {
-2
View File
@@ -422,8 +422,6 @@ pub(crate) mod scan {
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::acquire_scanner_bucket_incarnation_fence as ecstore_scanner_ack_fence;
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::disk::HEALING_MARKER_PATH;
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::notification::new_global_notification_sys;
}
@@ -175,6 +175,7 @@ fn lock_write_rejects_bucket_replacement(write: LockWrite) {
bucket::{BucketOperations, DeleteBucketOptions, MakeBucketOptions},
object::{ObjectIO, ObjectOperations},
};
use crate::storage::storage_api::ecstore_object::WriteCompletion;
use crate::storage::storage_api::{StorageObjectOptions, StoragePutObjReader};
use rustfs_scanner::{SegmentInvalidationProducerIdentity, set_scanner_dirty_usage_mutation_observer};
use std::sync::{Arc, Mutex};
@@ -205,6 +206,8 @@ fn lock_write_rejects_bucket_replacement(write: LockWrite) {
&mut StoragePutObjReader::from_vec(body.to_vec()),
&StorageObjectOptions {
versioned: true,
// The test snapshots every disk immediately after this write.
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
@@ -144,59 +144,6 @@ async fn diagnostic_handlers_enforce_advertised_runtime_contract() {
assert_eq!(value["measured"], true);
assert_eq!(value["rx_bytes"], 7);
// Exercise the largest declared body allowed by the legacy claimed-route profile.
let payload = Bytes::from(vec![b'x'; 1024 * 1024]);
let size = payload.len();
let response = signed_admin_request(
&client,
&endpoint,
Method::POST,
"/minio/admin/v3/speedtest/client/devnull",
server.access_key(),
server.secret_key(),
&sha256_hex(&payload),
)
.body(payload)
.send()
.await
.expect("signed streaming upload through the MinIO alias");
let status = response.status();
let body = response.text().await.expect("streaming response");
assert!(status.is_success(), "unexpected upload status {status}: {body}");
let value: serde_json::Value = serde_json::from_str(&body).expect("streaming upload JSON");
assert_eq!(value["rx_bytes"], size);
let anonymous = client
.post(format!("{endpoint}/rustfs/admin/v3/speedtest/client/devnull"))
.header("content-length", 1)
.body(reqwest::Body::wrap_stream(stream::pending::<Result<Bytes, std::io::Error>>()))
.send()
.await
.expect("anonymous upload must be rejected before reading the body");
assert_eq!(anonymous.status(), reqwest::StatusCode::FORBIDDEN);
let bad_digest = signed_admin_request(
&client,
&endpoint,
Method::POST,
"/rustfs/admin/v3/speedtest/client/devnull",
server.access_key(),
server.secret_key(),
&sha256_hex(b"y"),
)
.body("x")
.send()
.await
.expect("mismatched body must receive a response");
assert_eq!(bad_digest.status(), reqwest::StatusCode::BAD_REQUEST);
assert!(
bad_digest
.text()
.await
.expect("digest error body")
.contains("XAmzContentSHA256Mismatch")
);
let oversized = signed_admin_request(
&client,
&endpoint,
@@ -227,11 +174,11 @@ async fn diagnostic_handlers_enforce_advertised_runtime_contract() {
DEVNULL_PATH,
server.access_key(),
server.secret_key(),
&sha256_hex(b"xy"),
&sha256_hex(b"x"),
)
.build()
.expect("signed stalled upload headers");
let mut head = format!("POST {DEVNULL_PATH} HTTP/1.1\r\nContent-Length: 2\r\nExpect: 100-continue\r\n");
let mut head = format!("POST {DEVNULL_PATH} HTTP/1.1\r\nContent-Length: 1\r\nExpect: 100-continue\r\n");
for (name, value) in request.headers() {
write!(head, "{name}: {}\r\n", value.to_str().expect("ASCII signing header")).expect("write request head");
}
@@ -248,7 +195,6 @@ async fn diagnostic_handlers_enforce_advertised_runtime_contract() {
let mut status = String::new();
socket.read_line(&mut status).await.expect("read stalled upload status");
assert_eq!(status, "HTTP/1.1 100 Continue\r\n");
socket.write_all(b"x").await.expect("send partial upload body");
stalled.push(socket);
}
stalled
+1 -1
View File
@@ -333,7 +333,7 @@ root = Path(sys.argv[1])
records = [(path, json.loads(path.read_text())) for path in (root / "requests").glob("*.timing.json")]
assert len(records) == 5, records
assert len({row["clock_id"] for _, row in records}) == 1
assert len({row["start_ns"] for _, row in records}) == 5
# Concurrent requests can start in the same monotonic clock tick.
for _, row in records:
assert 0 < row["start_ns"] < row["end_ns"]
assert row["start_unix_ns"] > 0 and row["end_unix_ns"] > 0