mirror of
https://github.com/Noooste/garage-ui.git
synced 2026-07-26 07:48:13 +00:00
feat(backend,frontend): implement prefix and recursive substring search for bucket objects (#89)
* feat(search): implement recursive substring search for bucket objects * fix: update key formatting to remove trailing slashes in ObjectsTable * feat(search): implement debounced search functionality and improve UI text clarity * test(objects): add test for handling search error response
This commit is contained in:
@@ -20,13 +20,13 @@ import (
|
||||
// served from the same origin as the API, any uploader could otherwise plant
|
||||
// stored XSS by uploading a file with one of these Content-Types.
|
||||
var unsafeInlineContentTypes = map[string]struct{}{
|
||||
"text/html": {},
|
||||
"application/xhtml+xml": {},
|
||||
"image/svg+xml": {},
|
||||
"application/xml": {},
|
||||
"text/xml": {},
|
||||
"text/html": {},
|
||||
"application/xhtml+xml": {},
|
||||
"image/svg+xml": {},
|
||||
"application/xml": {},
|
||||
"text/xml": {},
|
||||
"application/javascript": {},
|
||||
"text/javascript": {},
|
||||
"text/javascript": {},
|
||||
}
|
||||
|
||||
// safeContentType rewrites Content-Types that the browser would treat as
|
||||
@@ -89,6 +89,7 @@ func NewObjectHandler(s3Service services.S3Storage) *ObjectHandler {
|
||||
// @Produce json
|
||||
// @Param bucket path string true "Name of the bucket to list objects from"
|
||||
// @Param prefix query string false "Filter objects by prefix"
|
||||
// @Param search query string false "Recursively search object keys under prefix by case-insensitive substring (best-effort; bypasses max_keys and continuation_token)"
|
||||
// @Param max_keys query int false "Maximum number of objects to return (default: 100)"
|
||||
// @Param continuation_token query string false "Token for pagination to retrieve next page of results"
|
||||
// @Success 200 {object} models.APIResponse{data=models.ObjectListResponse} "Successfully retrieved list of objects and prefixes"
|
||||
@@ -109,6 +110,21 @@ func (h *ObjectHandler) ListObjects(c fiber.Ctx) error {
|
||||
|
||||
// Get query parameters for filtering and pagination
|
||||
prefix := c.Query("prefix", "")
|
||||
|
||||
// Search mode: a recursive, best-effort substring search across the whole
|
||||
// subtree under prefix. S3/Garage has no server-side substring search, so
|
||||
// the backend scans and filters. This bypasses page-token pagination and
|
||||
// max_keys, see S3Service.SearchObjects -> pagination is handled frontend side.
|
||||
if search := c.Query("search", ""); search != "" {
|
||||
results, err := h.s3Service.SearchObjects(ctx, bucketName, prefix, search)
|
||||
if err != nil {
|
||||
return c.Status(fiber.StatusInternalServerError).JSON(
|
||||
models.ErrorResponse(models.ErrCodeListFailed, "Failed to search objects: "+err.Error()),
|
||||
)
|
||||
}
|
||||
return c.JSON(models.SuccessResponse(results))
|
||||
}
|
||||
|
||||
continuationToken := c.Query("continuation_token", "")
|
||||
|
||||
maxKeysStr := c.Query("max_keys", "100")
|
||||
|
||||
@@ -120,6 +120,52 @@ func TestListObjects_ServiceError500(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestListObjects_SearchRoutesToSearchObjects(t *testing.T) {
|
||||
app, s3 := newObjectsTestApp(t)
|
||||
// Intentionally leave ListObjectsFn unset: if the handler wrongly falls
|
||||
// through to a normal listing, the mock returns an error and this fails.
|
||||
s3.SearchObjectsFn = func(_ context.Context, bucket, prefix, search string) (*models.ObjectListResponse, error) {
|
||||
if bucket != "b1" || prefix != "docs/" || search != "target" {
|
||||
t.Errorf("args = (%q, %q, %q)", bucket, prefix, search)
|
||||
}
|
||||
return &models.ObjectListResponse{
|
||||
Bucket: bucket, Count: 1,
|
||||
Objects: []models.ObjectInfo{{Key: "docs/target.pdf", Size: 20}},
|
||||
}, nil
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodGet, "/buckets/b1/objects?prefix=docs/&search=target", nil)
|
||||
resp, err := app.Test(req)
|
||||
if err != nil {
|
||||
t.Fatalf("app.Test: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("status = %d", resp.StatusCode)
|
||||
}
|
||||
var body struct {
|
||||
Data models.ObjectListResponse `json:"data"`
|
||||
}
|
||||
decodeJSON(t, resp.Body, &body)
|
||||
if body.Data.Count != 1 || len(body.Data.Objects) != 1 || body.Data.Objects[0].Key != "docs/target.pdf" {
|
||||
t.Errorf("unexpected search results: %+v", body.Data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListObjects_SearchError500(t *testing.T) {
|
||||
app, s3 := newObjectsTestApp(t)
|
||||
s3.SearchObjectsFn = func(_ context.Context, _, _, _ string) (*models.ObjectListResponse, error) {
|
||||
return nil, errors.New("boom")
|
||||
}
|
||||
resp, err := app.Test(httptest.NewRequest(http.MethodGet, "/buckets/b1/objects?search=target", nil))
|
||||
if err != nil {
|
||||
t.Fatalf("app.Test: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusInternalServerError {
|
||||
t.Fatalf("status = %d, want 500", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
// --- GetObjectMetadata ---
|
||||
|
||||
func TestGetObjectMetadata_Success(t *testing.T) {
|
||||
|
||||
@@ -48,6 +48,7 @@ type AdminService interface {
|
||||
// GetBucketStatistics) are intentionally excluded.
|
||||
type S3Storage interface {
|
||||
ListObjects(ctx context.Context, bucketName, prefix string, maxKeys int, continuationToken string) (*models.ObjectListResponse, error)
|
||||
SearchObjects(ctx context.Context, bucketName, prefix, search string) (*models.ObjectListResponse, error)
|
||||
UploadObject(ctx context.Context, bucketName, key string, body io.Reader, contentType string) (*models.ObjectUploadResponse, error)
|
||||
CreateDirectoryMarker(ctx context.Context, bucketName, key string) (*models.ObjectUploadResponse, error)
|
||||
GetObject(ctx context.Context, bucketName, key string) (io.ReadCloser, *models.ObjectInfo, error)
|
||||
|
||||
@@ -23,14 +23,15 @@ var _ services.S3Storage = (*S3Mock)(nil)
|
||||
// per-method function fields they care about; unset methods return
|
||||
// s3NotConfigured.
|
||||
type S3Mock struct {
|
||||
ListObjectsFn func(ctx context.Context, bucketName, prefix string, maxKeys int, continuationToken string) (*models.ObjectListResponse, error)
|
||||
UploadObjectFn func(ctx context.Context, bucketName, key string, body io.Reader, contentType string) (*models.ObjectUploadResponse, error)
|
||||
ListObjectsFn func(ctx context.Context, bucketName, prefix string, maxKeys int, continuationToken string) (*models.ObjectListResponse, error)
|
||||
SearchObjectsFn func(ctx context.Context, bucketName, prefix, search string) (*models.ObjectListResponse, error)
|
||||
UploadObjectFn func(ctx context.Context, bucketName, key string, body io.Reader, contentType string) (*models.ObjectUploadResponse, error)
|
||||
CreateDirectoryMarkerFn func(ctx context.Context, bucketName, key string) (*models.ObjectUploadResponse, error)
|
||||
GetObjectFn func(ctx context.Context, bucketName, key string) (io.ReadCloser, *models.ObjectInfo, error)
|
||||
ObjectExistsFn func(ctx context.Context, bucketName, key string) (bool, error)
|
||||
DeleteObjectFn func(ctx context.Context, bucketName, key string) error
|
||||
GetObjectMetadataFn func(ctx context.Context, bucketName, key string) (*models.ObjectInfo, error)
|
||||
GetPresignedURLFn func(ctx context.Context, bucketName, key string, expiresIn time.Duration) (string, error)
|
||||
GetObjectFn func(ctx context.Context, bucketName, key string) (io.ReadCloser, *models.ObjectInfo, error)
|
||||
ObjectExistsFn func(ctx context.Context, bucketName, key string) (bool, error)
|
||||
DeleteObjectFn func(ctx context.Context, bucketName, key string) error
|
||||
GetObjectMetadataFn func(ctx context.Context, bucketName, key string) (*models.ObjectInfo, error)
|
||||
GetPresignedURLFn func(ctx context.Context, bucketName, key string, expiresIn time.Duration) (string, error)
|
||||
DeleteMultipleObjectsFn func(ctx context.Context, bucketName string, keys []string) error
|
||||
UploadMultipleObjectsFn func(ctx context.Context, bucketName string, files []struct {
|
||||
Key string
|
||||
@@ -55,6 +56,14 @@ func (m *S3Mock) ListObjects(ctx context.Context, bucketName, prefix string, max
|
||||
return m.ListObjectsFn(ctx, bucketName, prefix, maxKeys, continuationToken)
|
||||
}
|
||||
|
||||
func (m *S3Mock) SearchObjects(ctx context.Context, bucketName, prefix, search string) (*models.ObjectListResponse, error) {
|
||||
m.record("SearchObjects", bucketName, prefix, search)
|
||||
if m.SearchObjectsFn == nil {
|
||||
return nil, s3NotConfigured("SearchObjects")
|
||||
}
|
||||
return m.SearchObjectsFn(ctx, bucketName, prefix, search)
|
||||
}
|
||||
|
||||
func (m *S3Mock) UploadObject(ctx context.Context, bucketName, key string, body io.Reader, contentType string) (*models.ObjectUploadResponse, error) {
|
||||
m.record("UploadObject", bucketName, key, contentType)
|
||||
if m.UploadObjectFn == nil {
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
"Noooste/garage-ui/internal/config"
|
||||
"Noooste/garage-ui/internal/models"
|
||||
logpkg "Noooste/garage-ui/pkg/logger"
|
||||
"Noooste/garage-ui/pkg/utils"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
@@ -240,17 +241,10 @@ func (s *S3Service) ListObjects(ctx context.Context, bucketName, prefix string,
|
||||
return nil, fmt.Errorf("failed to list objects in bucket %s: %w", bucketName, err)
|
||||
}
|
||||
|
||||
// Drop directory marker objects (zero-byte keys ending in "/"). Garage
|
||||
// returns them in Contents, but the UI renders folders from Prefixes — a
|
||||
// marker shown as both a folder and a file is confusing. Any marker not
|
||||
// already covered by a CommonPrefix is promoted to Prefixes below.
|
||||
contents := make([]minio.ObjectInfo, 0, len(result.Contents))
|
||||
markerKeys := make([]string, 0)
|
||||
for _, obj := range result.Contents {
|
||||
if strings.HasSuffix(obj.Key, "/") && obj.Size == 0 {
|
||||
// A marker whose key equals the current listing prefix is the
|
||||
// folder itself — drop it entirely so it doesn't render as a
|
||||
// nameless child of itself.
|
||||
if obj.Key != prefix {
|
||||
markerKeys = append(markerKeys, obj.Key)
|
||||
}
|
||||
@@ -331,6 +325,97 @@ func (s *S3Service) ListObjects(ctx context.Context, bucketName, prefix string,
|
||||
}, nil
|
||||
}
|
||||
|
||||
const (
|
||||
searchMaxScan = 10000 // stop after scanning this many objects
|
||||
searchMaxResults = 1000 // stop after collecting this many matches
|
||||
searchPageSize = 1000 // objects requested per ListObjectsV2 page
|
||||
)
|
||||
|
||||
func objectMatchesSearch(key string, size int64, lowerQuery string) bool {
|
||||
if strings.HasSuffix(key, "/") && size == 0 {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(strings.ToLower(key), lowerQuery)
|
||||
}
|
||||
|
||||
// SearchObjects performs a recursive, best-effort substring search over object
|
||||
// keys under the given prefix.
|
||||
func (s *S3Service) SearchObjects(ctx context.Context, bucketName, prefix, search string) (*models.ObjectListResponse, error) {
|
||||
client, err := s.getMinioClient(ctx, bucketName, OpRead)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get MinIO client for bucket %s: %w", bucketName, err)
|
||||
}
|
||||
|
||||
core := &minio.Core{Client: client}
|
||||
lowerQuery := strings.ToLower(search)
|
||||
|
||||
matches := make([]models.ObjectInfo, 0, 64)
|
||||
scanned := 0
|
||||
truncated := false
|
||||
token := ""
|
||||
|
||||
scan:
|
||||
for {
|
||||
result, err := core.ListObjectsV2(
|
||||
bucketName,
|
||||
prefix,
|
||||
"",
|
||||
token,
|
||||
"",
|
||||
searchPageSize,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to search objects in bucket %s: %w", bucketName, err)
|
||||
}
|
||||
|
||||
for _, obj := range result.Contents {
|
||||
scanned++
|
||||
if objectMatchesSearch(obj.Key, obj.Size, lowerQuery) {
|
||||
matches = append(matches, models.ObjectInfo{
|
||||
Key: obj.Key,
|
||||
Size: obj.Size,
|
||||
LastModified: obj.LastModified,
|
||||
ETag: obj.ETag,
|
||||
StorageClass: obj.StorageClass,
|
||||
})
|
||||
if len(matches) >= searchMaxResults {
|
||||
truncated = true
|
||||
break scan
|
||||
}
|
||||
}
|
||||
if scanned >= searchMaxScan {
|
||||
truncated = true
|
||||
break scan
|
||||
}
|
||||
}
|
||||
|
||||
if !result.IsTruncated || result.NextContinuationToken == "" {
|
||||
break
|
||||
}
|
||||
token = result.NextContinuationToken
|
||||
}
|
||||
|
||||
if truncated {
|
||||
logpkg.FromCtx(ctx).Warn().
|
||||
Str("bucket", bucketName).
|
||||
Str("prefix", prefix).
|
||||
Int("scanned", scanned).
|
||||
Int("matches", len(matches)).
|
||||
Msg("search hit scan/result cap; results are partial")
|
||||
}
|
||||
|
||||
return &models.ObjectListResponse{
|
||||
Bucket: bucketName,
|
||||
Objects: matches,
|
||||
Prefixes: []string{},
|
||||
Count: len(matches),
|
||||
IsTruncated: truncated,
|
||||
// Search returns all matches up to the cap in one response; there is no
|
||||
// token-based pagination for search results.
|
||||
NextContinuationToken: "",
|
||||
}, nil
|
||||
}
|
||||
|
||||
// UploadObject uploads an object to a bucket
|
||||
func (s *S3Service) UploadObject(ctx context.Context, bucketName, key string, body io.Reader, contentType string) (*models.ObjectUploadResponse, error) {
|
||||
// Get bucket-specific MinIO client
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// s3SearchHandler serves a two-page recursive ListObjectsV2 response and counts
|
||||
// HEAD (StatObject) requests. Page selection is driven by the
|
||||
// `continuation-token` query parameter, mirroring how the MinIO SDK paginates.
|
||||
func s3SearchHandler(bucket string, page1, page2 []struct {
|
||||
Key string
|
||||
Size int64
|
||||
LastModified string
|
||||
ETag string
|
||||
}) (http.Handler, *int, *string) {
|
||||
var heads int
|
||||
var lastDelimiter string
|
||||
h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodHead {
|
||||
heads++
|
||||
w.Header().Set("Content-Length", "0")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
lastDelimiter = r.URL.Query().Get("delimiter")
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if r.URL.Query().Get("continuation-token") == "PAGE2" {
|
||||
_, _ = io.WriteString(w, listBucketResultXML(bucket, false, "", page2, nil))
|
||||
return
|
||||
}
|
||||
_, _ = io.WriteString(w, listBucketResultXML(bucket, true, "PAGE2", page1, nil))
|
||||
})
|
||||
return h, &heads, &lastDelimiter
|
||||
}
|
||||
|
||||
// TestS3_SearchObjects_FindsMatchOnLaterPage reproduces issue #87: an object
|
||||
// that lives on the second page of a listing must still be found by search.
|
||||
// SearchObjects should page through the whole listing recursively.
|
||||
func TestS3_SearchObjects_FindsMatchOnLaterPage(t *testing.T) {
|
||||
bucket := "b-TestS3_SearchObjects_FindsMatchOnLaterPage"
|
||||
page1 := []struct {
|
||||
Key string
|
||||
Size int64
|
||||
LastModified string
|
||||
ETag string
|
||||
}{
|
||||
{Key: "docs/alpha.txt", Size: 10},
|
||||
{Key: "docs/beta.txt", Size: 10},
|
||||
}
|
||||
page2 := []struct {
|
||||
Key string
|
||||
Size int64
|
||||
LastModified string
|
||||
ETag string
|
||||
}{
|
||||
{Key: "docs/target-report.pdf", Size: 20},
|
||||
{Key: "docs/gamma.txt", Size: 10},
|
||||
}
|
||||
h, heads, lastDelimiter := s3SearchHandler(bucket, page1, page2)
|
||||
s3 := newS3TestService(t, h)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
got, err := s3.SearchObjects(ctx, bucket, "", "target")
|
||||
if err != nil {
|
||||
t.Fatalf("SearchObjects: %v", err)
|
||||
}
|
||||
if got.Count != 1 || len(got.Objects) != 1 || got.Objects[0].Key != "docs/target-report.pdf" {
|
||||
t.Fatalf("expected to find docs/target-report.pdf on page 2, got %+v", got.Objects)
|
||||
}
|
||||
// Search must be recursive (no delimiter) so it descends into folders.
|
||||
if *lastDelimiter != "" {
|
||||
t.Errorf("delimiter = %q, want empty (recursive listing)", *lastDelimiter)
|
||||
}
|
||||
// Search must not fetch ContentType per object — that would be N stat calls.
|
||||
if *heads != 0 {
|
||||
t.Errorf("StatObject (HEAD) calls = %d, want 0 during search", *heads)
|
||||
}
|
||||
}
|
||||
|
||||
// TestS3_SearchObjects_ExcludesDirectoryMarkersAndIsCaseInsensitive verifies
|
||||
// that zero-byte directory markers never appear as matches and that matching
|
||||
// is a case-insensitive substring test on the full key.
|
||||
func TestS3_SearchObjects_ExcludesDirectoryMarkersAndIsCaseInsensitive(t *testing.T) {
|
||||
bucket := "b-TestS3_SearchObjects_ExcludesDirectoryMarkers"
|
||||
page1 := []struct {
|
||||
Key string
|
||||
Size int64
|
||||
LastModified string
|
||||
ETag string
|
||||
}{
|
||||
{Key: "Reports/", Size: 0}, // directory marker — must be excluded
|
||||
{Key: "Reports/Q1-REPORT.csv", Size: 5}, // matches "report" case-insensitively
|
||||
{Key: "images/logo.png", Size: 5}, // no match
|
||||
}
|
||||
h, _, _ := s3SearchHandler(bucket, page1, nil)
|
||||
s3 := newS3TestService(t, h)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
got, err := s3.SearchObjects(ctx, bucket, "", "report")
|
||||
if err != nil {
|
||||
t.Fatalf("SearchObjects: %v", err)
|
||||
}
|
||||
if got.Count != 1 || got.Objects[0].Key != "Reports/Q1-REPORT.csv" {
|
||||
t.Fatalf("expected only Reports/Q1-REPORT.csv, got %+v", got.Objects)
|
||||
}
|
||||
for _, o := range got.Objects {
|
||||
if strings.HasSuffix(o.Key, "/") {
|
||||
t.Errorf("directory marker leaked into results: %q", o.Key)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user