mirror of
https://github.com/Noooste/garage-ui.git
synced 2026-07-26 07:48:13 +00:00
Noste
573 lines
17 KiB
Go
573 lines
17 KiB
Go
package services
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
|
|
"Noooste/garage-ui/internal/config"
|
|
"Noooste/garage-ui/internal/models"
|
|
"Noooste/garage-ui/pkg/utils"
|
|
|
|
"github.com/minio/minio-go/v7"
|
|
"github.com/minio/minio-go/v7/pkg/credentials"
|
|
)
|
|
|
|
// S3Service handles all S3 operations with Garage using MinIO SDK
|
|
type S3Service struct {
|
|
client *minio.Client
|
|
config *config.GarageConfig
|
|
adminService *GarageAdminService
|
|
}
|
|
|
|
// NewS3Service creates a new S3 service instance using MinIO SDK
|
|
func NewS3Service(cfg *config.GarageConfig, adminService *GarageAdminService) *S3Service {
|
|
// Create MinIO client for Garage
|
|
// trim http or https from endpoint
|
|
if strings.HasPrefix(cfg.Endpoint, "http://") {
|
|
cfg.Endpoint = strings.TrimPrefix(cfg.Endpoint, "http://")
|
|
}
|
|
|
|
if strings.HasPrefix(cfg.Endpoint, "https://") {
|
|
cfg.Endpoint = strings.TrimPrefix(cfg.Endpoint, "https://")
|
|
cfg.UseSSL = true
|
|
}
|
|
|
|
client, err := minio.New(cfg.Endpoint, &minio.Options{
|
|
//Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""),
|
|
Secure: cfg.UseSSL,
|
|
Region: cfg.Region,
|
|
})
|
|
if err != nil {
|
|
panic(fmt.Errorf("failed to create MinIO client: %w", err))
|
|
}
|
|
|
|
return &S3Service{
|
|
client: client,
|
|
config: cfg,
|
|
adminService: adminService,
|
|
}
|
|
}
|
|
|
|
func (s *S3Service) getBucketCredentials(ctx context.Context, bucketName string) (*credentials.Credentials, error) {
|
|
cacheKey := fmt.Sprintf("key:%s", bucketName)
|
|
cacheData := utils.GlobalCache.Get(cacheKey)
|
|
|
|
if cacheData != nil {
|
|
return cacheData.(*credentials.Credentials), nil
|
|
}
|
|
|
|
// Get bucket info from Garage Admin API
|
|
bucketInfo, err := s.adminService.GetBucketInfoByAlias(ctx, bucketName)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to get bucket info: %w", err)
|
|
}
|
|
|
|
// Find a key with read and write permissions
|
|
var accessKeyID, secretAccessKey string
|
|
for _, keyInfo := range bucketInfo.Keys {
|
|
if !keyInfo.Permissions.Read || !keyInfo.Permissions.Write {
|
|
continue
|
|
}
|
|
|
|
// Get key details with secret
|
|
keyDetails, err := s.adminService.GetKeyInfo(ctx, keyInfo.AccessKeyID, true)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to get key info: %w", err)
|
|
}
|
|
|
|
if keyDetails.SecretAccessKey != nil {
|
|
accessKeyID = keyDetails.AccessKeyID
|
|
secretAccessKey = *keyDetails.SecretAccessKey
|
|
break
|
|
}
|
|
}
|
|
|
|
if accessKeyID == "" || secretAccessKey == "" {
|
|
return nil, fmt.Errorf("no valid credentials found for bucket %s", bucketName)
|
|
}
|
|
|
|
// Create credentials
|
|
creds := credentials.NewStaticV4(accessKeyID, secretAccessKey, "")
|
|
|
|
// Cache credentials for 1 hour
|
|
utils.GlobalCache.Set(cacheKey, creds, time.Hour)
|
|
|
|
return creds, nil
|
|
}
|
|
|
|
// getMinioClient creates a MinIO client for a specific bucket with dynamic credentials
|
|
func (s *S3Service) getMinioClient(ctx context.Context, bucketName string) (*minio.Client, error) {
|
|
creds, err := s.getBucketCredentials(ctx, bucketName)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot get credentials for bucket %s: %w", bucketName, err)
|
|
}
|
|
|
|
// Create MinIO client with bucket-specific credentials
|
|
client, err := minio.New(s.config.Endpoint, &minio.Options{
|
|
Creds: creds,
|
|
Secure: s.config.UseSSL,
|
|
Region: s.config.Region,
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create MinIO client for bucket %s: %w", bucketName, err)
|
|
}
|
|
|
|
return client, nil
|
|
}
|
|
|
|
// ListBuckets retrieves all buckets from Garage
|
|
func (s *S3Service) ListBuckets(ctx context.Context) (*models.BucketListResponse, error) {
|
|
var bucketInfos []minio.BucketInfo
|
|
|
|
// Call MinIO ListBuckets API with retry logic
|
|
retryConfig := utils.DefaultRetryConfig()
|
|
err := utils.RetryWithBackoff(ctx, retryConfig, func() error {
|
|
var listErr error
|
|
bucketInfos, listErr = s.client.ListBuckets(ctx)
|
|
return listErr
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to list buckets: %w", err)
|
|
}
|
|
|
|
// Convert MinIO buckets to our model
|
|
buckets := make([]models.BucketInfo, 0, len(bucketInfos))
|
|
for _, bucket := range bucketInfos {
|
|
buckets = append(buckets, models.BucketInfo{
|
|
Name: bucket.Name,
|
|
CreationDate: bucket.CreationDate,
|
|
})
|
|
}
|
|
|
|
return &models.BucketListResponse{
|
|
Buckets: buckets,
|
|
Count: len(buckets),
|
|
}, nil
|
|
}
|
|
|
|
// CreateBucket creates a new bucket in Garage
|
|
func (s *S3Service) CreateBucket(ctx context.Context, bucketName string) error {
|
|
client, err := s.getMinioClient(ctx, bucketName)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to get MinIO client for bucket %s: %w", bucketName, err)
|
|
}
|
|
|
|
// Call MinIO MakeBucket API with retry logic
|
|
retryConfig := utils.DefaultRetryConfig()
|
|
err = utils.RetryWithBackoff(ctx, retryConfig, func() error {
|
|
return client.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{
|
|
Region: s.config.Region,
|
|
})
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create bucket %s: %w", bucketName, err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// DeleteBucket deletes a bucket from Garage
|
|
func (s *S3Service) DeleteBucket(ctx context.Context, bucketName string) error {
|
|
client, err := s.getMinioClient(ctx, bucketName)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to get MinIO client for bucket %s: %w", bucketName, err)
|
|
}
|
|
|
|
// Call MinIO RemoveBucket API with retry logic
|
|
retryConfig := utils.DefaultRetryConfig()
|
|
err = utils.RetryWithBackoff(ctx, retryConfig, func() error {
|
|
return client.RemoveBucket(ctx, bucketName)
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("failed to delete bucket %s: %w", bucketName, err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// ListObjects lists objects in a bucket with optional prefix filter and pagination
|
|
func (s *S3Service) ListObjects(ctx context.Context, bucketName, prefix string, maxKeys int, continuationToken string) (*models.ObjectListResponse, error) {
|
|
// Get bucket-specific MinIO client
|
|
client, err := s.getMinioClient(ctx, bucketName)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to get MinIO client for bucket %s: %w", bucketName, err)
|
|
}
|
|
|
|
// Set default max keys if not specified
|
|
if maxKeys <= 0 {
|
|
maxKeys = 1000
|
|
}
|
|
|
|
// Create Core client for low-level API access
|
|
core := &minio.Core{Client: client}
|
|
|
|
// Use Core.ListObjectsV2 for proper pagination with continuation tokens
|
|
result, err := core.ListObjectsV2(
|
|
bucketName,
|
|
prefix, // objectPrefix
|
|
"", // startAfter (empty when using continuationToken)
|
|
continuationToken, // continuationToken (proper S3 token)
|
|
"/", // delimiter (for folder listing)
|
|
maxKeys, // maxkeys
|
|
)
|
|
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to list objects in bucket %s: %w", bucketName, err)
|
|
}
|
|
|
|
// Process objects from result.Contents
|
|
// Note: ListObjectsV2 doesn't return ContentType, so we need to fetch it separately
|
|
objects := make([]models.ObjectInfo, len(result.Contents))
|
|
|
|
// Use goroutines to fetch ContentType concurrently for better performance
|
|
type statResult struct {
|
|
index int
|
|
contentType string
|
|
err error
|
|
}
|
|
|
|
statChan := make(chan statResult, len(result.Contents))
|
|
|
|
for i, obj := range result.Contents {
|
|
go func(idx int, objKey string) {
|
|
// Fetch object metadata to get ContentType
|
|
stat, err := client.StatObject(ctx, bucketName, objKey, minio.StatObjectOptions{})
|
|
if err != nil {
|
|
// If StatObject fails, we still include the object but without ContentType
|
|
statChan <- statResult{index: idx, contentType: "", err: err}
|
|
return
|
|
}
|
|
statChan <- statResult{index: idx, contentType: stat.ContentType, err: nil}
|
|
}(i, obj.Key)
|
|
|
|
// Initialize the object with basic info from ListObjectsV2
|
|
objects[i] = models.ObjectInfo{
|
|
Key: obj.Key,
|
|
Size: obj.Size,
|
|
LastModified: obj.LastModified,
|
|
ETag: obj.ETag,
|
|
StorageClass: obj.StorageClass,
|
|
}
|
|
}
|
|
|
|
// Collect results from goroutines
|
|
for range result.Contents {
|
|
res := <-statChan
|
|
if res.err == nil {
|
|
objects[res.index].ContentType = res.contentType
|
|
}
|
|
// If there was an error, ContentType remains empty, which is acceptable
|
|
}
|
|
close(statChan)
|
|
|
|
// Process folders from result.CommonPrefixes
|
|
prefixList := make([]string, 0, len(result.CommonPrefixes))
|
|
for _, p := range result.CommonPrefixes {
|
|
prefixList = append(prefixList, p.Prefix)
|
|
}
|
|
|
|
return &models.ObjectListResponse{
|
|
Bucket: bucketName,
|
|
Objects: objects,
|
|
Prefixes: prefixList,
|
|
Count: len(objects),
|
|
IsTruncated: result.IsTruncated,
|
|
NextContinuationToken: result.NextContinuationToken,
|
|
}, nil
|
|
}
|
|
|
|
// UploadObject uploads an object to a bucket
|
|
func (s *S3Service) UploadObject(ctx context.Context, bucketName, key string, body io.Reader, contentType string) (*models.ObjectUploadResponse, error) {
|
|
// Get bucket-specific MinIO client
|
|
client, err := s.getMinioClient(ctx, bucketName)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to get MinIO client for bucket %s: %w", bucketName, err)
|
|
}
|
|
|
|
// Upload options
|
|
opts := minio.PutObjectOptions{
|
|
ContentType: contentType,
|
|
}
|
|
|
|
var info minio.UploadInfo
|
|
|
|
// Call MinIO PutObject API with retry logic
|
|
retryConfig := utils.DefaultRetryConfig()
|
|
err = utils.RetryWithBackoff(ctx, retryConfig, func() error {
|
|
var uploadErr error
|
|
info, uploadErr = client.PutObject(ctx, bucketName, key, body, -1, opts)
|
|
return uploadErr
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to upload object %s to bucket %s: %w", key, bucketName, err)
|
|
}
|
|
|
|
return &models.ObjectUploadResponse{
|
|
Bucket: bucketName,
|
|
Key: key,
|
|
ETag: info.ETag,
|
|
Size: info.Size,
|
|
ContentType: contentType,
|
|
}, nil
|
|
}
|
|
|
|
// GetObject retrieves an object from a bucket
|
|
func (s *S3Service) GetObject(ctx context.Context, bucketName, key string) (io.ReadCloser, *models.ObjectInfo, error) {
|
|
var object *minio.Object
|
|
|
|
// Call MinIO GetObject API with retry logic
|
|
retryConfig := utils.DefaultRetryConfig()
|
|
err := utils.RetryWithBackoff(ctx, retryConfig, func() error {
|
|
var getErr error
|
|
object, getErr = s.client.GetObject(ctx, bucketName, key, minio.GetObjectOptions{})
|
|
return getErr
|
|
})
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to get object %s from bucket %s: %w", key, bucketName, err)
|
|
}
|
|
|
|
// Get object info
|
|
stat, err := object.Stat()
|
|
if err != nil {
|
|
object.Close()
|
|
return nil, nil, fmt.Errorf("failed to get object info for %s in bucket %s: %w", key, bucketName, err)
|
|
}
|
|
|
|
// Create object info
|
|
objectInfo := &models.ObjectInfo{
|
|
Key: key,
|
|
Size: stat.Size,
|
|
LastModified: stat.LastModified,
|
|
ETag: stat.ETag,
|
|
ContentType: stat.ContentType,
|
|
}
|
|
|
|
return object, objectInfo, nil
|
|
}
|
|
|
|
// DeleteObject deletes an object from a bucket
|
|
func (s *S3Service) DeleteObject(ctx context.Context, bucketName, key string) error {
|
|
// Call MinIO RemoveObject API with retry logic
|
|
retryConfig := utils.DefaultRetryConfig()
|
|
err := utils.RetryWithBackoff(ctx, retryConfig, func() error {
|
|
return s.client.RemoveObject(ctx, bucketName, key, minio.RemoveObjectOptions{})
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("failed to delete object %s from bucket %s: %w", key, bucketName, err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// ObjectExists checks if an object exists in a bucket
|
|
func (s *S3Service) ObjectExists(ctx context.Context, bucketName, key string) (bool, error) {
|
|
// Get bucket-specific MinIO client
|
|
client, err := s.getMinioClient(ctx, bucketName)
|
|
if err != nil {
|
|
return false, fmt.Errorf("failed to get MinIO client for bucket %s: %w", bucketName, err)
|
|
}
|
|
|
|
var statErr error
|
|
|
|
// Call MinIO StatObject API with retry logic
|
|
retryConfig := utils.DefaultRetryConfig()
|
|
err = utils.RetryWithBackoff(ctx, retryConfig, func() error {
|
|
_, statErr = client.StatObject(ctx, bucketName, key, minio.StatObjectOptions{})
|
|
return statErr
|
|
})
|
|
|
|
if err != nil {
|
|
// Check if error is "object not found"
|
|
errResponse := minio.ToErrorResponse(err)
|
|
if errResponse.Code == "NoSuchKey" {
|
|
return false, nil
|
|
}
|
|
return false, fmt.Errorf("failed to check if object exists: %w", err)
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
// GetObjectMetadata retrieves metadata for an object without downloading it
|
|
func (s *S3Service) GetObjectMetadata(ctx context.Context, bucketName, key string) (*models.ObjectInfo, error) {
|
|
// Get bucket-specific MinIO client
|
|
client, err := s.getMinioClient(ctx, bucketName)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to get MinIO client for bucket %s: %w", bucketName, err)
|
|
}
|
|
|
|
var stat minio.ObjectInfo
|
|
|
|
// Call MinIO StatObject API with retry logic
|
|
retryConfig := utils.DefaultRetryConfig()
|
|
err = utils.RetryWithBackoff(ctx, retryConfig, func() error {
|
|
var statErr error
|
|
stat, statErr = client.StatObject(ctx, bucketName, key, minio.StatObjectOptions{})
|
|
return statErr
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to get metadata for object %s in bucket %s: %w", key, bucketName, err)
|
|
}
|
|
|
|
return &models.ObjectInfo{
|
|
Key: key,
|
|
Size: stat.Size,
|
|
LastModified: stat.LastModified,
|
|
ETag: stat.ETag,
|
|
ContentType: stat.ContentType,
|
|
StorageClass: stat.StorageClass,
|
|
Metadata: stat.UserMetadata,
|
|
}, nil
|
|
}
|
|
|
|
// DeleteMultipleObjects deletes multiple objects from a bucket
|
|
func (s *S3Service) DeleteMultipleObjects(ctx context.Context, bucketName string, keys []string) error {
|
|
if len(keys) == 0 {
|
|
return nil
|
|
}
|
|
|
|
// Get bucket-specific MinIO client
|
|
client, err := s.getMinioClient(ctx, bucketName)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to get MinIO client for bucket %s: %w", bucketName, err)
|
|
}
|
|
|
|
// Create channel for objects to delete
|
|
objectsCh := make(chan minio.ObjectInfo)
|
|
|
|
// Send objects to delete in a goroutine
|
|
go func() {
|
|
defer close(objectsCh)
|
|
for _, key := range keys {
|
|
objectsCh <- minio.ObjectInfo{
|
|
Key: key,
|
|
}
|
|
}
|
|
}()
|
|
|
|
// Call MinIO RemoveObjects API (batch delete)
|
|
errorCh := client.RemoveObjects(ctx, bucketName, objectsCh, minio.RemoveObjectsOptions{})
|
|
|
|
// Check for errors
|
|
for err := range errorCh {
|
|
if err.Err != nil {
|
|
return fmt.Errorf("failed to delete object %s from bucket %s: %w", err.ObjectName, bucketName, err.Err)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// GetPresignedURL generates a pre-signed URL for temporary access to an object
|
|
// This is useful for sharing files without exposing credentials
|
|
func (s *S3Service) GetPresignedURL(ctx context.Context, bucketName, key string, expiresIn time.Duration) (string, error) {
|
|
// Get bucket-specific MinIO client
|
|
client, err := s.getMinioClient(ctx, bucketName)
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to get MinIO client for bucket %s: %w", bucketName, err)
|
|
}
|
|
|
|
var presignedURL *url.URL
|
|
|
|
// Generate presigned GET URL with retry logic
|
|
retryConfig := utils.DefaultRetryConfig()
|
|
err = utils.RetryWithBackoff(ctx, retryConfig, func() error {
|
|
var presignErr error
|
|
presignedURL, presignErr = client.PresignedGetObject(ctx, bucketName, key, expiresIn, nil)
|
|
return presignErr
|
|
})
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to generate presigned URL for %s/%s: %w", bucketName, key, err)
|
|
}
|
|
|
|
return presignedURL.String(), nil
|
|
}
|
|
|
|
// UploadResult represents the result of a single file upload
|
|
type UploadResult struct {
|
|
Key string
|
|
Success bool
|
|
Error error
|
|
ETag string
|
|
Size int64
|
|
ContentType string
|
|
}
|
|
|
|
func (s *S3Service) UploadMultipleObjects(ctx context.Context, bucketName string, files []struct {
|
|
Key string
|
|
Body io.Reader
|
|
ContentType string
|
|
}) []UploadResult {
|
|
results := make([]UploadResult, len(files))
|
|
|
|
// Get bucket-specific MinIO client once for all uploads
|
|
client, err := s.getMinioClient(ctx, bucketName)
|
|
if err != nil {
|
|
// If we can't get the client, all uploads fail
|
|
for i := range files {
|
|
results[i] = UploadResult{
|
|
Key: files[i].Key,
|
|
Success: false,
|
|
Error: fmt.Errorf("failed to get MinIO client for bucket %s: %w", bucketName, err),
|
|
}
|
|
}
|
|
return results
|
|
}
|
|
|
|
// Upload each file
|
|
for i, file := range files {
|
|
// Upload options
|
|
opts := minio.PutObjectOptions{
|
|
ContentType: file.ContentType,
|
|
}
|
|
|
|
// Attempt upload
|
|
info, err := client.PutObject(ctx, bucketName, file.Key, file.Body, -1, opts)
|
|
if err != nil {
|
|
results[i] = UploadResult{
|
|
Key: file.Key,
|
|
Success: false,
|
|
Error: fmt.Errorf("failed to upload object %s: %w", file.Key, err),
|
|
ContentType: file.ContentType,
|
|
}
|
|
continue
|
|
}
|
|
|
|
results[i] = UploadResult{
|
|
Key: file.Key,
|
|
Success: true,
|
|
Error: nil,
|
|
ETag: info.ETag,
|
|
Size: info.Size,
|
|
ContentType: file.ContentType,
|
|
}
|
|
}
|
|
|
|
return results
|
|
}
|
|
|
|
// BucketStatistics holds statistical information about a bucket
|
|
type BucketStatistics struct {
|
|
ObjectCount int64
|
|
TotalSize int64
|
|
}
|
|
|
|
// GetBucketStatistics retrieves bucket statistics from Garage Admin API
|
|
// This is much more efficient than iterating through all objects
|
|
func (s *S3Service) GetBucketStatistics(ctx context.Context, bucketName string) (*BucketStatistics, error) {
|
|
// Get bucket info from Garage Admin API which includes object count and size
|
|
bucketInfo, err := s.adminService.GetBucketInfoByAlias(ctx, bucketName)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to get bucket info for %s: %w", bucketName, err)
|
|
}
|
|
|
|
// Return statistics from Admin API
|
|
return &BucketStatistics{
|
|
ObjectCount: bucketInfo.Objects,
|
|
TotalSize: bucketInfo.Bytes,
|
|
}, nil
|
|
}
|