Merge branch 'main' into cxymds/fix-1926-owner-selection

crates/ecstore/AGENTS.md

@@ -49,4 +49,3 @@ Applies to `crates/ecstore/`.
 ## Suggested Validation
 
 - `cargo test -p rustfs-ecstore`
-- Full gate before commit: `make pre-commit`

crates/ecstore/src/bucket/bucket_target_sys.rs

@@ -866,7 +866,7 @@ impl BucketTargetSys {
             return Some(cli);
         }
 
-        // TODO: spawn a task to reload the target
+        // TODO(backlog): spawn an async task to proactively reload the replication target
         if self.is_reloading_target(bucket, arn).await {
             return None;
         }

crates/ecstore/src/cluster/rpc/peer_s3_client.rs

@@ -454,7 +454,7 @@ impl S3PeerSys {
                 }
             }
             topology_complete &= bucket_map.values().all(|count| *count >= quorum);
-            // TODO: MRF
+            // TODO(backlog): integrate MRF backlog stats into scanner bucket listing
         }
 
         let mut buckets: Vec<BucketInfo> = result_map.into_values().collect();

crates/ecstore/src/cluster/rpc/remote_disk.rs

@@ -2406,7 +2406,7 @@ impl DiskAPI for RemoteDisk {
             return errors;
         }
 
-        // TODO: use Error not string
+        // TODO(backlog): replace string errors with typed `StorageError` variants
 
         let result = self
             .execute_with_timeout(

crates/ecstore/src/core/sets.rs

@@ -249,7 +249,7 @@ impl Sets {
 
         self.connect_disks().await;
 
-        // TODO: config interval
+        // TODO(backlog): make monitor_and_connect interval configurable instead of hardcoded 15s
         let mut interval = tokio::time::interval(Duration::from_secs(15));
         loop {
             tokio::select! {

crates/ecstore/src/disk/local.rs

@@ -5215,8 +5215,8 @@ impl LocalDisk {
 
         let cache = Cache::new(update_fn, Duration::from_secs(1), Opts::default());
 
-        // TODO: DIRECT support
-        // TODD: DiskInfo
+        // TODO(backlog): add O_DIRECT I/O support for performance-critical paths
+        // TODO(backlog): populate DiskInfo in constructor
         let mut disk = Self {
             root: root.clone(),
             publication_root,
@@ -5751,7 +5751,7 @@ impl LocalDisk {
 
         // return Ok(());
 
-        // TODO: async notifications for disk space checks and trash cleanup
+        // TODO(backlog): make disk space checks and trash cleanup event-driven instead of poll-based
 
         let trash_path = self.io_get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?;
         // if let Some(parent) = trash_path.parent() {
@@ -5997,7 +5997,7 @@ impl LocalDisk {
 
     #[hotpath::measure(impl_type = "LocalDisk")]
     async fn read_all_data(&self, volume: &str, volume_dir: impl AsRef<Path>, file_path: impl AsRef<Path>) -> Result<Vec<u8>> {
-        // TODO: timeout support
+        // TODO(backlog): add configurable timeout for read_all_data operations
         let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir, file_path).await?;
         Ok(data)
     }
@@ -6674,7 +6674,7 @@ impl LocalDisk {
             return Ok(());
         }
 
-        // TODO: add lock
+        // TODO(backlog): add directory listing lock to prevent concurrent enumeration
 
         let stall = opts.stall_timeout_duration();
 
@@ -8796,7 +8796,7 @@ impl DiskAPI for LocalDisk {
         Ok(entries)
     }
 
-    // FIXME: TODO: io.writer TODO cancel
+    // TODO(backlog): support io.writer cancellation and early termination in walk_dir
     #[tracing::instrument(level = "trace", skip_all)]
     async fn walk_dir<W: AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
         self.wait_for_startup_cleanup().await;
@@ -9880,7 +9880,7 @@ impl DiskAPI for LocalDisk {
                 );
                 return Err(e);
             }
-            // TODO: health check
+            // TODO(backlog): add post-setup disk health verification
         }
         Ok(())
     }

crates/ecstore/src/disk/os.rs

@@ -315,7 +315,7 @@ pub async fn fsync_dir(dir: impl AsRef<Path>) -> io::Result<()> {
     #[cfg(unix)]
     {
         let dir = dir.as_ref().to_path_buf();
-        tokio::task::spawn_blocking(move || fsync_dir_std(dir)).await?
+        fsync_spawn_blocking(move || fsync_dir_std(dir)).await?
     }
 
     #[cfg(not(unix))]
@@ -683,7 +683,7 @@ async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup) -> io::Result<()> {
     #[cfg(test)]
     let dir = group.dir.clone();
     let dir_file = group.dir_file.clone();
-    tokio::task::spawn_blocking(move || {
+    fsync_spawn_blocking(move || {
         #[cfg(test)]
         {
             if let Some(kind) = fsync_dir_recorder::take_grouped_failure(&dir) {
@@ -1080,6 +1080,44 @@ const TEST_GLOBAL_FILE_SYNCS: usize = 64;
 
 static FILE_SYNC_PERMITS: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(global_file_sync_limit()));
 static DISK_FILE_SYNC_LIMITERS: LazyLock<Mutex<HashMap<PathBuf, Weak<Semaphore>>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
+
+/// Dedicated tokio runtime for fsync/fdatasync blocking operations. When
+/// configured with >1 threads, isolates device-bound fsync from the main
+/// blocking pool so reads (pread/stat/open) are not starved. `None` means
+/// fall back to the main runtime (zero behavior change).
+static FSYNC_RUNTIME: LazyLock<Option<tokio::runtime::Runtime>> = LazyLock::new(|| {
+    let threads =
+        rustfs_utils::get_env_usize(rustfs_config::ENV_FSYNC_BLOCKING_THREADS, rustfs_config::DEFAULT_FSYNC_BLOCKING_THREADS);
+    if threads <= 1 {
+        return None;
+    }
+    let mut builder = tokio::runtime::Builder::new_multi_thread();
+    builder
+        .worker_threads(num_cpus::get().min(8))
+        .max_blocking_threads(threads)
+        .thread_name("rustfs-fsync")
+        .thread_stack_size(512 * 1024)
+        .enable_all();
+    match builder.build() {
+        Ok(rt) => {
+            tracing::info!(threads, "fsync dedicated blocking pool enabled");
+            Some(rt)
+        }
+        Err(err) => {
+            tracing::warn!(%err, "failed to build fsync runtime, falling back to main pool");
+            None
+        }
+    }
+});
+
+/// Spawn a blocking task on the fsync-dedicated runtime if configured,
+/// otherwise fall back to the main tokio blocking pool.
+fn fsync_spawn_blocking<T: Send + 'static>(f: impl FnOnce() -> T + Send + 'static) -> tokio::task::JoinHandle<T> {
+    match FSYNC_RUNTIME.as_ref() {
+        Some(rt) => rt.spawn_blocking(f),
+        None => tokio::task::spawn_blocking(f),
+    }
+}
 static DISK_VOLUME_MUTATION_LOCKS: LazyLock<Mutex<HashMap<PathBuf, Weak<RwLock<()>>>>> =
     LazyLock::new(|| Mutex::new(HashMap::new()));
 type NamespaceMutationLock = AsyncMutex<()>;
@@ -1217,7 +1255,7 @@ where
     F: FnOnce() -> io::Result<T> + Send + 'static,
 {
     let (disk_permit, global_permit) = acquire_file_sync_permits(disk_permits).await?;
-    let result = tokio::task::spawn_blocking(move || {
+    let result = fsync_spawn_blocking(move || {
         let _disk_permit = disk_permit;
         work()
     })
@@ -2146,7 +2184,7 @@ async fn run_blocking_namespace_file_sync_operation_with_global<T: Send + 'stati
         wait_started,
     );
     let disk_permit = admission.disk_permit.clone();
-    let result = tokio::task::spawn_blocking(move || {
+    let result = fsync_spawn_blocking(move || {
         let _lease = lease;
         let _disk_permit = disk_permit;
         operation()

crates/ecstore/src/layout/endpoints.rs

@@ -249,7 +249,7 @@ impl PoolEndpointList {
             endpoint.set_set_index(0);
             endpoint.set_disk_index(0);
 
-            // TODO Check for cross device mounts if any.
+            // TODO(backlog): check for cross-device mounts in single-drive setup
 
             return Ok(Self {
                 inner: vec![Endpoints::from(vec![endpoint])],
@@ -264,7 +264,7 @@ impl PoolEndpointList {
                 // Convert args to endpoints
                 let mut eps = Endpoints::try_from(set_layout.as_slice())?;
 
-                // TODO Check for cross device mounts if any.
+                // TODO(backlog): check for cross-device mounts in multi-pool setup
 
                 for (disk_idx, ep) in eps.as_mut().iter_mut().enumerate() {
                     ep.set_pool_index(pool_idx);

crates/ecstore/src/object_api/types.rs

@@ -1091,7 +1091,7 @@ impl ObjectInfo {
                     }
                 };
 
-                // TODO:VersionPurgeStatus
+                // TODO(backlog): handle VersionPurgeStatus in object listing
                 let versioned = vcfg.clone().map(|v| v.0.versioned(&entry.name)).unwrap_or_default();
                 objects.push(ObjectInfo::from_file_info(&fi, bucket, &entry.name, versioned));
 

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -1575,7 +1575,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
         let parts_metadata = vec![fi.clone(); disks.len()];
 
         if !user_defined.contains_key("content-type") {
-            // TODO: get content-type
+            // TODO(backlog): detect content-type from part data when header is missing
         }
 
         if let Some(sc) = user_defined.get(AMZ_STORAGE_CLASS)
@@ -1971,7 +1971,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
                 return Err(Error::InvalidPart(p.part_num, ext_part.etag.clone(), p.etag.clone().unwrap_or_default()));
             }
 
-            // TODO: crypto
+            // TODO(backlog): integrate encryption verification during complete multipart
 
             if (i < uploaded_parts.len() - 1)
                 && !(opts.data_movement && ext_part.actual_size < 0)

crates/ecstore/src/set_disk/ops/object.rs

@@ -6161,7 +6161,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
 
         join_all(rollback_futures).await;
 
-        // TODO: add_partial
+        // TODO(backlog): support partial object deletion for multi-part objects
 
         if let Some(api) = opts.tier_delete_journal_api.as_ref() {
             for (idx, je) in persisted_journal_entries {
@@ -6371,7 +6371,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
             }
         }
 
-        // TODO: Lifecycle
+        // TODO(backlog): integrate lifecycle evaluation before object deletion
 
         let mut version_found = true;
         // delete_object_version below derives its own majority quorum from the
@@ -6465,7 +6465,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                 mark_deleted: mark_delete,
                 mod_time: Some(mod_time),
                 replication_state_internal: opts.delete_replication.as_ref().map(replication_state_to_filemeta),
-                ..Default::default() // TODO: Transition
+                ..Default::default() // TODO(backlog): populate transition state on delete markers
             };
 
             fi.set_tier_free_version_id(&find_vid.to_string());

crates/ecstore/src/store/bucket.rs

@@ -601,7 +601,7 @@ impl ECStore {
 
     #[instrument(skip(self))]
     pub(super) async fn handle_list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
-        // TODO: opts.cached
+        // TODO(backlog): support cached bucket listing via opts.cached
 
         let mut buckets = self.peer_sys.list_bucket(opts).await?;
 

crates/ecstore/src/store/list_objects.rs

@@ -4673,7 +4673,7 @@ async fn gather_results(
             entry.name = entry.name.replace("\\", "/");
         }
 
-        // TODO: rx.recv()
+        // TODO(backlog): integrate rx.recv() for incremental listing results
 
         if let Some(marker) = &opts.marker
             && ((!opts.include_marker && &entry.name <= marker) || (opts.include_marker && &entry.name < marker))
@@ -4703,7 +4703,7 @@ async fn gather_results(
             continue;
         }
 
-        // TODO: Lifecycle
+        // TODO(backlog): integrate lifecycle evaluation during object listing
 
         entries.push(Some(entry));
         candidate_entries += 1;

crates/ecstore/src/store/multipart.rs

@@ -332,7 +332,7 @@ impl ECStore {
         let expected_incarnation_id = opts.expected_bucket_incarnation_id;
 
         if request.prefix.is_empty() {
-            // TODO: return from cache
+            // TODO(backlog): return cached multipart listing when prefix is empty
         }
 
         if self.single_pool() {
@@ -610,7 +610,7 @@ impl ECStore {
         let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
         let opts = &opts;
 
-        // TODO: defer DeleteUploadID
+        // TODO(backlog): defer DeleteUploadID to background for faster abort response
 
         if self.single_pool() {
             return self.pools[0].abort_multipart_upload(bucket, object, upload_id, opts).await;

crates/ecstore/src/store/rebalance.rs

@@ -385,7 +385,7 @@ impl ECStore {
     }
 
     pub(super) async fn is_suspended(&self, idx: usize) -> bool {
-        // TODO: LOCK
+        // TODO(backlog): acquire pool metadata lock for consistent suspension check
 
         let pool_meta = self.pool_meta.read().await;