batch upload impl
This commit is contained in:
@@ -0,0 +1,84 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/google/uuid"
|
||||
|
||||
"queryorchestration/internal/serviceconfig/objectstore"
|
||||
)
|
||||
|
||||
type Service struct {
|
||||
cfg objectstore.ConfigProvider
|
||||
}
|
||||
|
||||
type StorageResult struct {
|
||||
Bucket string
|
||||
Key string
|
||||
SizeBytes int64
|
||||
}
|
||||
|
||||
func New(cfg objectstore.ConfigProvider) *Service {
|
||||
return &Service{cfg: cfg}
|
||||
}
|
||||
|
||||
// StoreZIPFile uploads ZIP file to S3 and returns storage metadata
|
||||
func (s *Service) StoreZIPFile(ctx context.Context, clientID string, filename string, content io.Reader) (*StorageResult, error) {
|
||||
// Create unique S3 key with timestamp and UUID
|
||||
timestamp := time.Now().Format("2006/01/02/15")
|
||||
batchID := uuid.New()
|
||||
key := fmt.Sprintf("batches/%s/%s/%s-%s", clientID, timestamp, batchID.String(), filename)
|
||||
|
||||
// Get bucket from existing configuration
|
||||
bucketName := s.cfg.GetBucket()
|
||||
|
||||
// Read content to get size (required for S3 PutObject)
|
||||
buf := &bytes.Buffer{}
|
||||
size, err := io.Copy(buf, content)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read file content: %w", err)
|
||||
}
|
||||
|
||||
// Upload to S3 using the store client
|
||||
_, err = s.cfg.GetStoreClient().PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: &bucketName,
|
||||
Key: &key,
|
||||
Body: bytes.NewReader(buf.Bytes()),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to upload ZIP to S3: %w", err)
|
||||
}
|
||||
|
||||
return &StorageResult{
|
||||
Bucket: bucketName,
|
||||
Key: key,
|
||||
SizeBytes: size,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ValidateZIPFile performs basic ZIP file validation
|
||||
func (s *Service) ValidateZIPFile(filename string, sizeBytes int64) error {
|
||||
// Check file extension
|
||||
ext := filepath.Ext(filename)
|
||||
if ext != ".zip" {
|
||||
return fmt.Errorf("invalid file type: expected .zip, got %s", ext)
|
||||
}
|
||||
|
||||
// Check file size (100MB limit for Part 1)
|
||||
const maxSizeBytes = 100 * 1024 * 1024 // 100MB
|
||||
if sizeBytes > maxSizeBytes {
|
||||
return fmt.Errorf("file too large: %d bytes exceeds limit of %d bytes", sizeBytes, maxSizeBytes)
|
||||
}
|
||||
|
||||
if sizeBytes == 0 {
|
||||
return fmt.Errorf("empty file not allowed")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user