Compare commits

...

9 Commits

Author SHA1 Message Date
houseme bde1356024 Merge remote-tracking branch 'refs/remotes/origin/main' into copilot/fix-scanner-cycle-issue
# Conflicts:
#	rustfs/tests/admin_diagnostic_capability_e2e.rs
#	rustfs/tests/admin_rebalance_status.rs

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
2026-11-20 06:27:03 +00:00
Hauser Zhi 2b63e4104f chore(merge): synchronize with main
Use main’s overlapping gateway and admin fixes, retain the HTTP regressions, and match the structured rebalance failure result.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-20 08:59:43 +08:00
Hauser Zhi 2e3555f0eb fix(ci): restore admin gateway runtime checks
Stream diagnostic uploads through the existing admission gate and retain rebalance metadata error diagnostics. Align the authorization regressions with the current implementation and remove the unused POST form test helper.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-20 08:34:55 +08:00
Hauser Zhi 212cb2a47e refactor(scanner): scope heal inventory to coordinator workers
Build the inventory only for coordinator-owned cross-drive healing and expose the regression marker directly through the scan storage domain.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-20 01:24:53 +08:00
houseme 3c0a0cf7f0 fix(scanner): use scan domain for test marker import
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
2026-11-19 16:30:48 +00:00
Hauser Zhi 7f3ea0c73c perf(scanner): skip unused heal disk inventory
Avoid building and cloning the cross-drive inventory for single-drive scans and scans with healing suppressed. Exercise durable usage publication and cursor advancement through the runtime cycle.

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-20 00:24:56 +08:00
houseme f9cafcb8a2 fix(scanner): route healing marker through storage boundary
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
2026-11-19 15:58:58 +00:00
houseme 156defd1f1 fix(scanner): scan online single drives marked as healing
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
2026-11-19 14:45:52 +00:00
houseme 4398882cc9 Initial plan 2026-11-19 14:37:06 +00:00
4 changed files with 166 additions and 9 deletions
+88
View File
@@ -1635,6 +1635,94 @@ async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() {
global_metrics().set_cycle(None).await;
}
#[tokio::test]
#[serial]
async fn single_drive_cycle_persists_usage_and_advances_with_healing_marker() {
use crate::scanner_io::{clear_dirty_usage_buckets_for_tests, tests::wait_for_namespace_commit_tails};
use crate::storage_api::scan::{HEALING_MARKER_PATH, MakeBucketOptions};
clear_dirty_usage_buckets_for_tests();
let temp_dir = tempfile::tempdir().expect("scanner cycle test directory should be created");
let store = setup_scanner_cycle_store_at_path_with_layout(temp_dir.path(), true, 1, 1, 1).await;
let bucket = "marked-drive-bucket";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("fixture bucket should be created");
let mut reader = PutObjReader::from_vec(vec![7; 16]);
store
.put_object(bucket, "object", &mut reader, &ObjectOptions::default())
.await
.expect("fixture object should persist");
wait_for_namespace_commit_tails(store.as_ref()).await;
let marker = temp_dir
.path()
.join("pool0/disk0")
.join(RUSTFS_META_BUCKET)
.join(HEALING_MARKER_PATH);
let marker_contents = b"leftover marker";
tokio::fs::write(&marker, marker_contents)
.await
.expect("fixture healing marker should persist");
let set = &store.pools[0].disk_set[0];
tokio::time::timeout(Duration::from_secs(10), async {
while !set.get_online_disks_with_healing(true).await.1 {
tokio::time::sleep(Duration::from_millis(100)).await;
}
})
.await
.expect("disk info should observe the marker after its cache expires");
let ctx = CancellationToken::new();
let mut cycle_info = CurrentCycle {
next: 1,
..Default::default()
};
let mut revision = DataUsageCacheRevision::Missing;
let leader_epoch = 11;
let outcome = tokio::time::timeout(
Duration::from_secs(60),
run_data_scanner_cycle(&ctx, &store, &mut cycle_info, &mut revision, leader_epoch),
)
.await
.expect("scanner cycle should finish");
assert!(
matches!(
outcome,
ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance
),
"unexpected cycle outcome: {outcome:?}",
);
let mut set_snapshot = DataUsageCache::default();
set_snapshot
.load_with_revisions(Arc::clone(set), DATA_USAGE_CACHE_NAME)
.await
.expect("set usage snapshot should be readable");
assert!(set_snapshot.info.snapshot_complete);
assert_eq!(set_snapshot.info.next_cycle, 1);
let usage = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("global usage snapshot should be durable");
let usage: DataUsageInfo = serde_json::from_slice(&usage).expect("global usage snapshot should decode");
assert!(usage.usage_snapshot_complete);
assert!(usage.last_update.is_some());
assert_eq!(usage.buckets_usage[bucket].objects_count, 1);
let persisted_cycle = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
.await
.expect("scanner cycle cursor should be durable");
let (persisted_cycle, persisted_epoch) =
decode_scanner_cycle_state(&persisted_cycle).expect("scanner cycle cursor should decode");
assert_eq!(persisted_epoch, leader_epoch);
assert_eq!(persisted_cycle.next, 2);
assert_eq!(tokio::fs::read(marker).await.expect("healing marker should remain"), marker_contents);
clear_dirty_usage_buckets_for_tests();
global_metrics().set_cycle(None).await;
}
#[tokio::test]
#[serial]
async fn coordinator_publishes_authority_during_pending_put() {
+20 -7
View File
@@ -620,7 +620,10 @@ impl ScannerIOCache for SetDisks {
.ok_or_else(|| StorageError::other("failed to persist empty scanner set scope"));
}
let (disks, healing) = self.get_online_disks_with_healing(false).await;
let single_drive = self.set_drive_count == 1;
// A single-drive set has no alternate scan source. A heal marker
// still disables scanner heals, but must not exclude its only usage walk.
let (disks, healing) = self.get_online_disks_with_healing(single_drive).await;
#[cfg(test)]
let disks = super::tests::usage_remote_drain::walker_disks(&self, &buckets, disks);
if disks.is_empty() {
@@ -757,7 +760,18 @@ impl ScannerIOCache for SetDisks {
let _ = updates.send(incomplete_scope).await;
return Ok(ScannerSourceTurnReceipt::default());
}
let set_disk_inventory = Arc::new(scanner_set_disk_inventory(self.as_ref()).await);
// Coordinator workers need the full inventory only for cross-drive
// healing. Remote workers own their disk inventory on the peer.
let heal_disk_inventory = if !single_drive
&& !healing
&& workers
.iter()
.any(|(_, mode)| *mode == NamespaceScannerWorkerMode::Coordinator)
{
Some(Arc::new(scanner_set_disk_inventory(self.as_ref()).await))
} else {
None
};
if unsupported_remote_disks > 0 {
debug!(
target: "rustfs::scanner::io",
@@ -1010,7 +1024,6 @@ impl ScannerIOCache for SetDisks {
let store_clone_clone = self.clone();
let bucket_result_tx_clone = bucket_result_tx.clone();
let bucket_observations_clone = bucket_observations.clone();
let set_disk_inventory_clone = set_disk_inventory.clone();
let disk_scan_semaphore_clone = disk_scan_semaphore.clone();
let queued_disk_bucket_scans_clone = queued_disk_bucket_scans.clone();
let active_disk_bucket_scans_clone = active_disk_bucket_scans.clone();
@@ -1026,9 +1039,9 @@ impl ScannerIOCache for SetDisks {
let worker_retired_buckets = retired_buckets.clone();
#[cfg(feature = "e2e-test-hooks")]
let worker_incarnations = current_bucket_incarnations.clone();
let remote_capabilities = match worker_mode {
NamespaceScannerWorkerMode::RemoteV4(capabilities) => Some(capabilities),
NamespaceScannerWorkerMode::Coordinator => None,
let (remote_capabilities, heal_disk_inventory) = match worker_mode {
NamespaceScannerWorkerMode::Coordinator => (None, heal_disk_inventory.clone()),
NamespaceScannerWorkerMode::RemoteV4(capabilities) => (Some(capabilities), None),
};
#[cfg(test)]
let worker_index = futs.len();
@@ -1671,7 +1684,7 @@ impl ScannerIOCache for SetDisks {
let scan = crate::workload_admission::with_scanner_node_requests(node_tx, disk.clone().nsscanner_disk(
scan_ctx.clone(),
bucket_budget.clone(),
set_disk_inventory_clone.as_ref().clone(),
heal_disk_inventory.as_deref().cloned().unwrap_or_default(),
cache,
None,
ScannerDiskScanOptions {
+2
View File
@@ -422,6 +422,8 @@ pub(crate) mod scan {
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::acquire_scanner_bucket_incarnation_fence as ecstore_scanner_ack_fence;
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::disk::HEALING_MARKER_PATH;
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::notification::new_global_notification_sys;
}
@@ -144,6 +144,59 @@ async fn diagnostic_handlers_enforce_advertised_runtime_contract() {
assert_eq!(value["measured"], true);
assert_eq!(value["rx_bytes"], 7);
// Exercise the largest declared body allowed by the legacy claimed-route profile.
let payload = Bytes::from(vec![b'x'; 1024 * 1024]);
let size = payload.len();
let response = signed_admin_request(
&client,
&endpoint,
Method::POST,
"/minio/admin/v3/speedtest/client/devnull",
server.access_key(),
server.secret_key(),
&sha256_hex(&payload),
)
.body(payload)
.send()
.await
.expect("signed streaming upload through the MinIO alias");
let status = response.status();
let body = response.text().await.expect("streaming response");
assert!(status.is_success(), "unexpected upload status {status}: {body}");
let value: serde_json::Value = serde_json::from_str(&body).expect("streaming upload JSON");
assert_eq!(value["rx_bytes"], size);
let anonymous = client
.post(format!("{endpoint}/rustfs/admin/v3/speedtest/client/devnull"))
.header("content-length", 1)
.body(reqwest::Body::wrap_stream(stream::pending::<Result<Bytes, std::io::Error>>()))
.send()
.await
.expect("anonymous upload must be rejected before reading the body");
assert_eq!(anonymous.status(), reqwest::StatusCode::FORBIDDEN);
let bad_digest = signed_admin_request(
&client,
&endpoint,
Method::POST,
"/rustfs/admin/v3/speedtest/client/devnull",
server.access_key(),
server.secret_key(),
&sha256_hex(b"y"),
)
.body("x")
.send()
.await
.expect("mismatched body must receive a response");
assert_eq!(bad_digest.status(), reqwest::StatusCode::BAD_REQUEST);
assert!(
bad_digest
.text()
.await
.expect("digest error body")
.contains("XAmzContentSHA256Mismatch")
);
let oversized = signed_admin_request(
&client,
&endpoint,
@@ -174,11 +227,11 @@ async fn diagnostic_handlers_enforce_advertised_runtime_contract() {
DEVNULL_PATH,
server.access_key(),
server.secret_key(),
&sha256_hex(b"x"),
&sha256_hex(b"xy"),
)
.build()
.expect("signed stalled upload headers");
let mut head = format!("POST {DEVNULL_PATH} HTTP/1.1\r\nContent-Length: 1\r\nExpect: 100-continue\r\n");
let mut head = format!("POST {DEVNULL_PATH} HTTP/1.1\r\nContent-Length: 2\r\nExpect: 100-continue\r\n");
for (name, value) in request.headers() {
write!(head, "{name}: {}\r\n", value.to_str().expect("ASCII signing header")).expect("write request head");
}
@@ -195,6 +248,7 @@ async fn diagnostic_handlers_enforce_advertised_runtime_contract() {
let mut status = String::new();
socket.read_line(&mut status).await.expect("read stalled upload status");
assert_eq!(status, "HTTP/1.1 100 Continue\r\n");
socket.write_all(b"x").await.expect("send partial upload body");
stalled.push(socket);
}
stalled