feat(backend,frontend): implement prefix and recursive substring search for bucket objects (#89)

* feat(search): implement recursive substring search for bucket objects

* fix: update key formatting to remove trailing slashes in ObjectsTable

* feat(search): implement debounced search functionality and improve UI text clarity

* test(objects): add test for handling search error response
This commit is contained in:
Noste
2026-07-11 17:09:27 +02:00
committed by GitHub
parent b28e975a56
commit 3098e474f2
12 changed files with 494 additions and 56 deletions
+22 -6
View File
@@ -20,13 +20,13 @@ import (
// served from the same origin as the API, any uploader could otherwise plant
// stored XSS by uploading a file with one of these Content-Types.
var unsafeInlineContentTypes = map[string]struct{}{
"text/html": {},
"application/xhtml+xml": {},
"image/svg+xml": {},
"application/xml": {},
"text/xml": {},
"text/html": {},
"application/xhtml+xml": {},
"image/svg+xml": {},
"application/xml": {},
"text/xml": {},
"application/javascript": {},
"text/javascript": {},
"text/javascript": {},
}
// safeContentType rewrites Content-Types that the browser would treat as
@@ -89,6 +89,7 @@ func NewObjectHandler(s3Service services.S3Storage) *ObjectHandler {
// @Produce json
// @Param bucket path string true "Name of the bucket to list objects from"
// @Param prefix query string false "Filter objects by prefix"
// @Param search query string false "Recursively search object keys under prefix by case-insensitive substring (best-effort; bypasses max_keys and continuation_token)"
// @Param max_keys query int false "Maximum number of objects to return (default: 100)"
// @Param continuation_token query string false "Token for pagination to retrieve next page of results"
// @Success 200 {object} models.APIResponse{data=models.ObjectListResponse} "Successfully retrieved list of objects and prefixes"
@@ -109,6 +110,21 @@ func (h *ObjectHandler) ListObjects(c fiber.Ctx) error {
// Get query parameters for filtering and pagination
prefix := c.Query("prefix", "")
// Search mode: a recursive, best-effort substring search across the whole
// subtree under prefix. S3/Garage has no server-side substring search, so
// the backend scans and filters. This bypasses page-token pagination and
// max_keys, see S3Service.SearchObjects -> pagination is handled frontend side.
if search := c.Query("search", ""); search != "" {
results, err := h.s3Service.SearchObjects(ctx, bucketName, prefix, search)
if err != nil {
return c.Status(fiber.StatusInternalServerError).JSON(
models.ErrorResponse(models.ErrCodeListFailed, "Failed to search objects: "+err.Error()),
)
}
return c.JSON(models.SuccessResponse(results))
}
continuationToken := c.Query("continuation_token", "")
maxKeysStr := c.Query("max_keys", "100")
+46
View File
@@ -120,6 +120,52 @@ func TestListObjects_ServiceError500(t *testing.T) {
}
}
func TestListObjects_SearchRoutesToSearchObjects(t *testing.T) {
app, s3 := newObjectsTestApp(t)
// Intentionally leave ListObjectsFn unset: if the handler wrongly falls
// through to a normal listing, the mock returns an error and this fails.
s3.SearchObjectsFn = func(_ context.Context, bucket, prefix, search string) (*models.ObjectListResponse, error) {
if bucket != "b1" || prefix != "docs/" || search != "target" {
t.Errorf("args = (%q, %q, %q)", bucket, prefix, search)
}
return &models.ObjectListResponse{
Bucket: bucket, Count: 1,
Objects: []models.ObjectInfo{{Key: "docs/target.pdf", Size: 20}},
}, nil
}
req := httptest.NewRequest(http.MethodGet, "/buckets/b1/objects?prefix=docs/&search=target", nil)
resp, err := app.Test(req)
if err != nil {
t.Fatalf("app.Test: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("status = %d", resp.StatusCode)
}
var body struct {
Data models.ObjectListResponse `json:"data"`
}
decodeJSON(t, resp.Body, &body)
if body.Data.Count != 1 || len(body.Data.Objects) != 1 || body.Data.Objects[0].Key != "docs/target.pdf" {
t.Errorf("unexpected search results: %+v", body.Data)
}
}
func TestListObjects_SearchError500(t *testing.T) {
app, s3 := newObjectsTestApp(t)
s3.SearchObjectsFn = func(_ context.Context, _, _, _ string) (*models.ObjectListResponse, error) {
return nil, errors.New("boom")
}
resp, err := app.Test(httptest.NewRequest(http.MethodGet, "/buckets/b1/objects?search=target", nil))
if err != nil {
t.Fatalf("app.Test: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusInternalServerError {
t.Fatalf("status = %d, want 500", resp.StatusCode)
}
}
// --- GetObjectMetadata ---
func TestGetObjectMetadata_Success(t *testing.T) {
+1
View File
@@ -48,6 +48,7 @@ type AdminService interface {
// GetBucketStatistics) are intentionally excluded.
type S3Storage interface {
ListObjects(ctx context.Context, bucketName, prefix string, maxKeys int, continuationToken string) (*models.ObjectListResponse, error)
SearchObjects(ctx context.Context, bucketName, prefix, search string) (*models.ObjectListResponse, error)
UploadObject(ctx context.Context, bucketName, key string, body io.Reader, contentType string) (*models.ObjectUploadResponse, error)
CreateDirectoryMarker(ctx context.Context, bucketName, key string) (*models.ObjectUploadResponse, error)
GetObject(ctx context.Context, bucketName, key string) (io.ReadCloser, *models.ObjectInfo, error)
+16 -7
View File
@@ -23,14 +23,15 @@ var _ services.S3Storage = (*S3Mock)(nil)
// per-method function fields they care about; unset methods return
// s3NotConfigured.
type S3Mock struct {
ListObjectsFn func(ctx context.Context, bucketName, prefix string, maxKeys int, continuationToken string) (*models.ObjectListResponse, error)
UploadObjectFn func(ctx context.Context, bucketName, key string, body io.Reader, contentType string) (*models.ObjectUploadResponse, error)
ListObjectsFn func(ctx context.Context, bucketName, prefix string, maxKeys int, continuationToken string) (*models.ObjectListResponse, error)
SearchObjectsFn func(ctx context.Context, bucketName, prefix, search string) (*models.ObjectListResponse, error)
UploadObjectFn func(ctx context.Context, bucketName, key string, body io.Reader, contentType string) (*models.ObjectUploadResponse, error)
CreateDirectoryMarkerFn func(ctx context.Context, bucketName, key string) (*models.ObjectUploadResponse, error)
GetObjectFn func(ctx context.Context, bucketName, key string) (io.ReadCloser, *models.ObjectInfo, error)
ObjectExistsFn func(ctx context.Context, bucketName, key string) (bool, error)
DeleteObjectFn func(ctx context.Context, bucketName, key string) error
GetObjectMetadataFn func(ctx context.Context, bucketName, key string) (*models.ObjectInfo, error)
GetPresignedURLFn func(ctx context.Context, bucketName, key string, expiresIn time.Duration) (string, error)
GetObjectFn func(ctx context.Context, bucketName, key string) (io.ReadCloser, *models.ObjectInfo, error)
ObjectExistsFn func(ctx context.Context, bucketName, key string) (bool, error)
DeleteObjectFn func(ctx context.Context, bucketName, key string) error
GetObjectMetadataFn func(ctx context.Context, bucketName, key string) (*models.ObjectInfo, error)
GetPresignedURLFn func(ctx context.Context, bucketName, key string, expiresIn time.Duration) (string, error)
DeleteMultipleObjectsFn func(ctx context.Context, bucketName string, keys []string) error
UploadMultipleObjectsFn func(ctx context.Context, bucketName string, files []struct {
Key string
@@ -55,6 +56,14 @@ func (m *S3Mock) ListObjects(ctx context.Context, bucketName, prefix string, max
return m.ListObjectsFn(ctx, bucketName, prefix, maxKeys, continuationToken)
}
func (m *S3Mock) SearchObjects(ctx context.Context, bucketName, prefix, search string) (*models.ObjectListResponse, error) {
m.record("SearchObjects", bucketName, prefix, search)
if m.SearchObjectsFn == nil {
return nil, s3NotConfigured("SearchObjects")
}
return m.SearchObjectsFn(ctx, bucketName, prefix, search)
}
func (m *S3Mock) UploadObject(ctx context.Context, bucketName, key string, body io.Reader, contentType string) (*models.ObjectUploadResponse, error) {
m.record("UploadObject", bucketName, key, contentType)
if m.UploadObjectFn == nil {
+92 -7
View File
@@ -11,6 +11,7 @@ import (
"Noooste/garage-ui/internal/config"
"Noooste/garage-ui/internal/models"
logpkg "Noooste/garage-ui/pkg/logger"
"Noooste/garage-ui/pkg/utils"
"github.com/minio/minio-go/v7"
@@ -240,17 +241,10 @@ func (s *S3Service) ListObjects(ctx context.Context, bucketName, prefix string,
return nil, fmt.Errorf("failed to list objects in bucket %s: %w", bucketName, err)
}
// Drop directory marker objects (zero-byte keys ending in "/"). Garage
// returns them in Contents, but the UI renders folders from Prefixes — a
// marker shown as both a folder and a file is confusing. Any marker not
// already covered by a CommonPrefix is promoted to Prefixes below.
contents := make([]minio.ObjectInfo, 0, len(result.Contents))
markerKeys := make([]string, 0)
for _, obj := range result.Contents {
if strings.HasSuffix(obj.Key, "/") && obj.Size == 0 {
// A marker whose key equals the current listing prefix is the
// folder itself — drop it entirely so it doesn't render as a
// nameless child of itself.
if obj.Key != prefix {
markerKeys = append(markerKeys, obj.Key)
}
@@ -331,6 +325,97 @@ func (s *S3Service) ListObjects(ctx context.Context, bucketName, prefix string,
}, nil
}
const (
searchMaxScan = 10000 // stop after scanning this many objects
searchMaxResults = 1000 // stop after collecting this many matches
searchPageSize = 1000 // objects requested per ListObjectsV2 page
)
func objectMatchesSearch(key string, size int64, lowerQuery string) bool {
if strings.HasSuffix(key, "/") && size == 0 {
return false
}
return strings.Contains(strings.ToLower(key), lowerQuery)
}
// SearchObjects performs a recursive, best-effort substring search over object
// keys under the given prefix.
func (s *S3Service) SearchObjects(ctx context.Context, bucketName, prefix, search string) (*models.ObjectListResponse, error) {
client, err := s.getMinioClient(ctx, bucketName, OpRead)
if err != nil {
return nil, fmt.Errorf("failed to get MinIO client for bucket %s: %w", bucketName, err)
}
core := &minio.Core{Client: client}
lowerQuery := strings.ToLower(search)
matches := make([]models.ObjectInfo, 0, 64)
scanned := 0
truncated := false
token := ""
scan:
for {
result, err := core.ListObjectsV2(
bucketName,
prefix,
"",
token,
"",
searchPageSize,
)
if err != nil {
return nil, fmt.Errorf("failed to search objects in bucket %s: %w", bucketName, err)
}
for _, obj := range result.Contents {
scanned++
if objectMatchesSearch(obj.Key, obj.Size, lowerQuery) {
matches = append(matches, models.ObjectInfo{
Key: obj.Key,
Size: obj.Size,
LastModified: obj.LastModified,
ETag: obj.ETag,
StorageClass: obj.StorageClass,
})
if len(matches) >= searchMaxResults {
truncated = true
break scan
}
}
if scanned >= searchMaxScan {
truncated = true
break scan
}
}
if !result.IsTruncated || result.NextContinuationToken == "" {
break
}
token = result.NextContinuationToken
}
if truncated {
logpkg.FromCtx(ctx).Warn().
Str("bucket", bucketName).
Str("prefix", prefix).
Int("scanned", scanned).
Int("matches", len(matches)).
Msg("search hit scan/result cap; results are partial")
}
return &models.ObjectListResponse{
Bucket: bucketName,
Objects: matches,
Prefixes: []string{},
Count: len(matches),
IsTruncated: truncated,
// Search returns all matches up to the cap in one response; there is no
// token-based pagination for search results.
NextContinuationToken: "",
}, nil
}
// UploadObject uploads an object to a bucket
func (s *S3Service) UploadObject(ctx context.Context, bucketName, key string, body io.Reader, contentType string) (*models.ObjectUploadResponse, error) {
// Get bucket-specific MinIO client
+121
View File
@@ -0,0 +1,121 @@
package services
import (
"context"
"io"
"net/http"
"strings"
"testing"
"time"
)
// s3SearchHandler serves a two-page recursive ListObjectsV2 response and counts
// HEAD (StatObject) requests. Page selection is driven by the
// `continuation-token` query parameter, mirroring how the MinIO SDK paginates.
func s3SearchHandler(bucket string, page1, page2 []struct {
Key string
Size int64
LastModified string
ETag string
}) (http.Handler, *int, *string) {
var heads int
var lastDelimiter string
h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodHead {
heads++
w.Header().Set("Content-Length", "0")
w.WriteHeader(http.StatusOK)
return
}
lastDelimiter = r.URL.Query().Get("delimiter")
w.Header().Set("Content-Type", "application/xml")
w.WriteHeader(http.StatusOK)
if r.URL.Query().Get("continuation-token") == "PAGE2" {
_, _ = io.WriteString(w, listBucketResultXML(bucket, false, "", page2, nil))
return
}
_, _ = io.WriteString(w, listBucketResultXML(bucket, true, "PAGE2", page1, nil))
})
return h, &heads, &lastDelimiter
}
// TestS3_SearchObjects_FindsMatchOnLaterPage reproduces issue #87: an object
// that lives on the second page of a listing must still be found by search.
// SearchObjects should page through the whole listing recursively.
func TestS3_SearchObjects_FindsMatchOnLaterPage(t *testing.T) {
bucket := "b-TestS3_SearchObjects_FindsMatchOnLaterPage"
page1 := []struct {
Key string
Size int64
LastModified string
ETag string
}{
{Key: "docs/alpha.txt", Size: 10},
{Key: "docs/beta.txt", Size: 10},
}
page2 := []struct {
Key string
Size int64
LastModified string
ETag string
}{
{Key: "docs/target-report.pdf", Size: 20},
{Key: "docs/gamma.txt", Size: 10},
}
h, heads, lastDelimiter := s3SearchHandler(bucket, page1, page2)
s3 := newS3TestService(t, h)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
got, err := s3.SearchObjects(ctx, bucket, "", "target")
if err != nil {
t.Fatalf("SearchObjects: %v", err)
}
if got.Count != 1 || len(got.Objects) != 1 || got.Objects[0].Key != "docs/target-report.pdf" {
t.Fatalf("expected to find docs/target-report.pdf on page 2, got %+v", got.Objects)
}
// Search must be recursive (no delimiter) so it descends into folders.
if *lastDelimiter != "" {
t.Errorf("delimiter = %q, want empty (recursive listing)", *lastDelimiter)
}
// Search must not fetch ContentType per object — that would be N stat calls.
if *heads != 0 {
t.Errorf("StatObject (HEAD) calls = %d, want 0 during search", *heads)
}
}
// TestS3_SearchObjects_ExcludesDirectoryMarkersAndIsCaseInsensitive verifies
// that zero-byte directory markers never appear as matches and that matching
// is a case-insensitive substring test on the full key.
func TestS3_SearchObjects_ExcludesDirectoryMarkersAndIsCaseInsensitive(t *testing.T) {
bucket := "b-TestS3_SearchObjects_ExcludesDirectoryMarkers"
page1 := []struct {
Key string
Size int64
LastModified string
ETag string
}{
{Key: "Reports/", Size: 0}, // directory marker — must be excluded
{Key: "Reports/Q1-REPORT.csv", Size: 5}, // matches "report" case-insensitively
{Key: "images/logo.png", Size: 5}, // no match
}
h, _, _ := s3SearchHandler(bucket, page1, nil)
s3 := newS3TestService(t, h)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
got, err := s3.SearchObjects(ctx, bucket, "", "report")
if err != nil {
t.Fatalf("SearchObjects: %v", err)
}
if got.Count != 1 || got.Objects[0].Key != "Reports/Q1-REPORT.csv" {
t.Fatalf("expected only Reports/Q1-REPORT.csv, got %+v", got.Objects)
}
for _, o := range got.Objects {
if strings.HasSuffix(o.Key, "/") {
t.Errorf("directory marker leaked into results: %q", o.Key)
}
}
}