feat(scanner): expose prefix-level bucket usage via admin API (HS-08) (#6171)

feat(scanner): expose prefix-level bucket usage via admin API

The scanner's per-bucket, per-set usage caches already hold a path-keyed
prefix tree, but dui() flattened it only to bucket names — consoles and
operators had no way to ask "what does this prefix hold" without an S3
listing sweep (rustfs/backlog#1872, MinIO loadPrefixUsageFromBackend
parity).

Add:

- data-usage: prefix_usage_in_cache — a shared aggregation over the
  entry map (arbitrary prefix, full counters, one-level sub-prefix
  breakdown with names recovered from the literal-path cache keys),
  hardened like the scanner's checked flatten: cycles, dangling child
  links, over-deep trees, and overflowing counters yield None rather
  than unbounded recursion or wrapped totals.
- ecstore: ECStore::all_set_disks — iterate every erasure set so a
  query can read each set's own cache copy; the hash-routed store path
  would always land on one set.
- scanner: bucket_prefix_usage — per-set loads (5s budget each, a slow
  set degrades to not-reporting instead of stalling the caller),
  merged across sets with partial/compacted/truncated flags, served
  from a bounded 30s cache (128 entries, hard-capped) that bucket
  writes invalidate through the dirty-usage hook.
- admin: GET /rustfs/admin/v3/usage/{bucket}?prefix=&max-entries=
  behind the same any-of gate as datausageinfo (DataUsageInfoAdminAction
  OR ListBucketAction), rejecting unknown query parameters and
  clamping max-entries to 1..=10000. Route registered in the policy
  table (deferred MultipleActions, matching datausageinfo) and the
  route matrix test.

Closes rustfs/backlog#1872.

Co-authored-by: heihutu <heihutu@gmail.com>

rustfs/src/admin/handlers/mod.rs

@@ -64,6 +64,7 @@ mod target_descriptor;
 pub mod tier;
 pub mod tls_debug;
 pub mod trace;
+pub mod usage_prefix;
 pub mod user;
 pub mod user_iam;
 pub mod user_lifecycle;

rustfs/src/admin/handlers/system.rs

@@ -1158,10 +1158,10 @@ impl Operation for RuntimeCapabilitiesHandler {
     }
 }
 
-/// Authorization gate for GET datausageinfo: any-of the dedicated admin action
-/// OR the bucket listing action. Pinned by a unit test so the gate cannot
-/// silently narrow or widen (rustfs/backlog#1306).
-fn data_usage_info_gate_actions() -> Vec<Action> {
+/// Authorization gate for GET datausageinfo (and prefix usage): any-of the
+/// dedicated admin action OR the bucket listing action. Pinned by a unit test
+/// so the gate cannot silently narrow or widen (rustfs/backlog#1306).
+pub(crate) fn data_usage_info_gate_actions() -> Vec<Action> {
     vec![
         Action::AdminAction(AdminAction::DataUsageInfoAdminAction),
         Action::S3Action(S3Action::ListBucketAction),

rustfs/src/admin/handlers/usage_prefix.rs

@@ -0,0 +1,142 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Prefix-level bucket usage admin handler (rustfs/backlog#1872).
+//!
+//! `GET /rustfs/admin/v3/usage/{bucket}?prefix=&max-entries=` answers
+//! "what does this bucket / this prefix hold" from the scanner's per-set
+//! usage caches, with a one-level sub-prefix breakdown — the data console
+//! buckets view MinIO serves from `loadPrefixUsageFromBackend`.
+
+use crate::admin::auth::validate_admin_request;
+use crate::admin::handlers::system::data_usage_info_gate_actions;
+use crate::admin::router::{AdminOperation, Operation, S3Router};
+use crate::auth::{check_key_valid, get_session_token};
+use crate::server::{ADMIN_PREFIX, RemoteAddr};
+use http::{HeaderMap, HeaderValue, StatusCode};
+use hyper::Method;
+use matchit::Params;
+use s3s::header::CONTENT_TYPE;
+use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
+
+const JSON_CONTENT_TYPE: &str = "application/json";
+const DEFAULT_MAX_ENTRIES: usize = 1000;
+const MAX_ENTRIES_LIMIT: usize = 10_000;
+
+pub struct BucketPrefixUsageHandler {}
+
+pub fn register_usage_prefix_route(r: &mut S3Router<AdminOperation>) -> std::io::Result<()> {
+    r.insert(
+        Method::GET,
+        format!("{}{}", ADMIN_PREFIX, "/v3/usage/{bucket}").as_str(),
+        AdminOperation(&BucketPrefixUsageHandler {}),
+    )?;
+    Ok(())
+}
+
+/// Parse `prefix` and `max-entries` from the query string. Unknown keys are
+/// rejected so a typo'd parameter cannot silently change the answer's shape.
+fn parse_usage_prefix_query(query: Option<&str>) -> S3Result<(String, usize)> {
+    let mut prefix: Option<String> = None;
+    let mut max_entries: Option<usize> = None;
+    for (key, value) in url::form_urlencoded::parse(query.unwrap_or_default().as_bytes()) {
+        match key.as_ref() {
+            "prefix" => prefix = Some(value.into_owned()),
+            "max-entries" => {
+                max_entries = Some(
+                    value
+                        .parse::<usize>()
+                        .map_err(|_| s3_error!(InvalidArgument, "max-entries must be a positive integer"))?,
+                );
+            }
+            other => return Err(s3_error!(InvalidArgument, "unknown query parameter: {other}")),
+        }
+    }
+    let max_entries = max_entries.unwrap_or(DEFAULT_MAX_ENTRIES).clamp(1, MAX_ENTRIES_LIMIT);
+    Ok((prefix.unwrap_or_default(), max_entries))
+}
+
+#[async_trait::async_trait]
+impl Operation for BucketPrefixUsageHandler {
+    async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
+        let Some(input_cred) = req.credentials else {
+            return Err(s3_error!(InvalidRequest, "get cred failed"));
+        };
+
+        let (cred, owner) =
+            check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
+
+        let remote_addr = req.extensions.get::<Option<RemoteAddr>>().and_then(|opt| opt.map(|a| a.0));
+        validate_admin_request(&req.headers, &cred, owner, false, data_usage_info_gate_actions(), remote_addr).await?;
+
+        let bucket = params.get("bucket").unwrap_or_default().to_string();
+        if bucket.is_empty() {
+            return Err(s3_error!(InvalidRequest, "bucket path parameter is required"));
+        }
+        let (prefix, max_entries) = parse_usage_prefix_query(req.uri.query())?;
+
+        // Authorization is bucket-scoped by the same any-of gate as the
+        // datausageinfo route; the bucket name itself is validated by the
+        // scanner layer, which rejects reserved/invalid names.
+        let response = rustfs_scanner::bucket_prefix_usage(&bucket, &prefix, max_entries)
+            .await
+            .map_err(|err| s3_error!(InvalidArgument, "{}", err))?;
+
+        let data = serde_json::to_vec(&response)
+            .map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "parse prefix usage failed"))?;
+        let mut header = HeaderMap::new();
+        header.insert(CONTENT_TYPE, HeaderValue::from_static(JSON_CONTENT_TYPE));
+
+        Ok(S3Response::with_headers((StatusCode::OK, Body::from(data)), header))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{DEFAULT_MAX_ENTRIES, MAX_ENTRIES_LIMIT, parse_usage_prefix_query};
+    use s3s::S3Error;
+
+    fn query(raw: &str) -> Result<(String, usize), S3Error> {
+        parse_usage_prefix_query(Some(raw))
+    }
+
+    #[test]
+    fn defaults_apply_when_no_query_is_given() {
+        assert_eq!(parse_usage_prefix_query(None).unwrap(), (String::new(), DEFAULT_MAX_ENTRIES));
+        assert_eq!(query("").unwrap(), (String::new(), DEFAULT_MAX_ENTRIES));
+    }
+
+    #[test]
+    fn prefix_round_trips_url_encoded_characters() {
+        let (prefix, _) = query("prefix=pre%2Ffix%20name").unwrap();
+        assert_eq!(prefix, "pre/fix name");
+    }
+
+    #[test]
+    fn max_entries_parses_and_clamps_to_documented_bounds() {
+        assert_eq!(query("max-entries=5").unwrap().1, 5);
+        assert_eq!(query("max-entries=0").unwrap().1, 1, "zero must clamp up, not mean unlimited");
+        assert_eq!(query("max-entries=99999999").unwrap().1, MAX_ENTRIES_LIMIT);
+        assert!(query("max-entries=-3").is_err());
+        assert!(query("max-entries=abc").is_err());
+    }
+
+    #[test]
+    fn unknown_parameters_are_rejected_not_ignored() {
+        assert!(
+            query("prefixes=x").is_err(),
+            "a typo'd parameter must fail the request, not widen the query"
+        );
+    }
+}

rustfs/src/admin/mod.rs

@@ -40,7 +40,8 @@ use handlers::{
     audit, batch_job, bucket_meta, cluster_snapshot, config_admin, diagnostics, durability as durability_handler, extensions,
     heal, health, idp_compat, ilm_transition, inspect_archive, kms, module_switch, object_data_cache, object_zip_download, oidc,
     plugins_catalog, plugins_instances, pools, profile_admin, quota as quota_handler, rebalance,
-    replication as replication_handler, scanner, site_replication, sts, system, table_catalog, tier, tls_debug, user,
+    replication as replication_handler, scanner, site_replication, sts, system, table_catalog, tier, tls_debug, usage_prefix,
+    user,
 };
 use router::{AdminOperation, S3Router};
 use s3s::route::S3Route;
@@ -80,6 +81,7 @@ fn register_admin_routes(r: &mut S3Router<AdminOperation>) -> std::io::Result<()
     bucket_meta::register_bucket_meta_route(r)?;
     config_admin::register_config_route(r)?;
     scanner::register_scanner_route(r)?;
+    usage_prefix::register_usage_prefix_route(r)?;
     ilm_transition::register_ilm_transition_route(r)?;
     object_data_cache::register_object_data_cache_route(r)?;
     audit::register_audit_target_route(r)?;

rustfs/src/admin/route_policy.rs

@@ -1558,6 +1558,11 @@ pub const DEFERRED_ADMIN_ROUTE_POLICIES: &[DeferredAdminRoutePolicy] = &[
         "/rustfs/admin/v3/datausageinfo",
         DeferredRoutePolicyReason::MultipleActions,
     ),
+    deferred(
+        HttpMethod::Get,
+        "/rustfs/admin/v3/usage/{bucket}",
+        DeferredRoutePolicyReason::MultipleActions,
+    ),
     deferred(
         HttpMethod::Post,
         "/rustfs/admin/v3/object-zip-downloads",

rustfs/src/admin/route_registration_test.rs

@@ -172,6 +172,7 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
         admin_route(Method::POST, "/v4/inspect/archive"),
         admin_route(Method::GET, "/v3/storageinfo"),
         admin_route(Method::GET, "/v3/datausageinfo"),
+        admin_route_sample(Method::GET, "/v3/usage/{bucket}", "/v3/usage/test-bucket"),
         admin_route(Method::GET, "/v3/metrics"),
         admin_route(Method::GET, "/v3/object-data-cache/stats"),
         admin_route(Method::POST, "/v3/object-data-cache/flush"),