Merged in feature/doc_import (pull request #177)
integration of background processor * integration part 1 * feature working * fix mimetype issue
This commit is contained in:
Jay Brown
@@ -13,6 +13,14 @@ import (
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
)
|
||||
|
||||
// Batch status constants
|
||||
const (
|
||||
StatusProcessing = "processing"
|
||||
StatusCompleted = "completed"
|
||||
StatusFailed = "failed"
|
||||
StatusCanceled = "canceled"
|
||||
)
|
||||
|
||||
// BatchUploadSummary represents a summary of a batch upload
|
||||
type BatchUploadSummary struct {
|
||||
ID uuid.UUID `json:"batch_id"`
|
||||
@@ -150,20 +158,37 @@ func (s *Service) UpdateProgress(ctx context.Context, clientID string, batchID u
|
||||
})
|
||||
}
|
||||
|
||||
// MarkCompleted marks a batch as completed
|
||||
func (s *Service) MarkCompleted(ctx context.Context, batchID uuid.UUID) error {
|
||||
// UpdateStatus updates the status of a batch
|
||||
func (s *Service) UpdateStatus(ctx context.Context, batchID uuid.UUID, status string) error {
|
||||
// Convert string status to repository.BatchStatus
|
||||
var batchStatus repository.BatchStatus
|
||||
switch status {
|
||||
case StatusProcessing:
|
||||
batchStatus = repository.BatchStatusProcessing
|
||||
case StatusCompleted:
|
||||
batchStatus = repository.BatchStatusCompleted
|
||||
case StatusFailed:
|
||||
batchStatus = repository.BatchStatusFailed
|
||||
case StatusCanceled:
|
||||
batchStatus = repository.BatchStatusCancelled
|
||||
default:
|
||||
return fmt.Errorf("invalid status: %s", status)
|
||||
}
|
||||
|
||||
return s.cfg.GetDBQueries().UpdateBatchStatus(ctx, &repository.UpdateBatchStatusParams{
|
||||
ID: batchID,
|
||||
Column2: repository.BatchStatusCompleted,
|
||||
Column2: batchStatus,
|
||||
})
|
||||
}
|
||||
|
||||
// MarkCompleted marks a batch as completed
|
||||
func (s *Service) MarkCompleted(ctx context.Context, batchID uuid.UUID) error {
|
||||
return s.UpdateStatus(ctx, batchID, StatusCompleted)
|
||||
}
|
||||
|
||||
// MarkFailed marks a batch as failed
|
||||
func (s *Service) MarkFailed(ctx context.Context, batchID uuid.UUID) error {
|
||||
return s.cfg.GetDBQueries().UpdateBatchStatus(ctx, &repository.UpdateBatchStatusParams{
|
||||
ID: batchID,
|
||||
Column2: repository.BatchStatusFailed,
|
||||
})
|
||||
return s.UpdateStatus(ctx, batchID, StatusFailed)
|
||||
}
|
||||
|
||||
// AddFailedFilename adds a failed filename to the batch
|
||||
@@ -254,3 +279,34 @@ func (s *Service) convertToDetailsWithStorage(batch *repository.GetBatchUploadWi
|
||||
|
||||
return details
|
||||
}
|
||||
|
||||
// ListUnprocessed retrieves all unprocessed batch uploads across all clients
|
||||
func (s *Service) ListUnprocessed(ctx context.Context) ([]*BatchUploadSummary, error) {
|
||||
batches, err := s.cfg.GetDBQueries().GetUnprocessedBatches(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query unprocessed batches: %w", err)
|
||||
}
|
||||
|
||||
summaries := make([]*BatchUploadSummary, len(batches))
|
||||
for i, batch := range batches {
|
||||
summaries[i] = &BatchUploadSummary{
|
||||
ID: batch.ID,
|
||||
ClientID: batch.ClientID,
|
||||
OriginalFilename: batch.OriginalFilename,
|
||||
Status: string(batch.Status),
|
||||
TotalDocuments: batch.TotalDocuments,
|
||||
ProcessedDocuments: batch.ProcessedDocuments,
|
||||
FailedDocuments: batch.FailedDocuments,
|
||||
InvalidTypeDocuments: batch.InvalidTypeDocuments,
|
||||
ProgressPercent: batch.ProgressPercent,
|
||||
CreatedAt: s.pgTimestampToTime(batch.CreatedAt),
|
||||
}
|
||||
|
||||
if batch.CompletedAt.Valid {
|
||||
t := s.pgTimestampToTime(batch.CompletedAt)
|
||||
summaries[i].CompletedAt = &t
|
||||
}
|
||||
}
|
||||
|
||||
return summaries, nil
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package batch_test
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"testing"
|
||||
@@ -225,7 +226,7 @@ func TestCreateWithStorage_Success(t *testing.T) {
|
||||
batch, err := service.Get(t.Context(), clientID, batchID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "test.zip", batch.OriginalFilename)
|
||||
assert.Equal(t, "processing", batch.Status)
|
||||
assert.Equal(t, documentbatch.StatusProcessing, batch.Status)
|
||||
assert.Equal(t, "test-bucket", batch.ArchiveBucket)
|
||||
assert.Equal(t, "batches/test-client/2025/01/01/test.zip", batch.ArchiveKey)
|
||||
assert.Equal(t, int64(1024*1024), batch.FileSizeBytes)
|
||||
@@ -420,3 +421,141 @@ func createTestZIPContent(t *testing.T, numFiles int) io.Reader {
|
||||
|
||||
return bytes.NewReader(buf.Bytes())
|
||||
}
|
||||
|
||||
func TestListUnprocessed(t *testing.T) {
|
||||
cfg := &TestConfig{}
|
||||
test.CreateDB(t, cfg)
|
||||
|
||||
service := documentbatch.New(cfg)
|
||||
ctx := t.Context()
|
||||
|
||||
// Create multiple test clients
|
||||
clientID1 := "test_client_unprocessed1"
|
||||
clientID2 := "test_client_unprocessed2"
|
||||
|
||||
for _, clientID := range []string{clientID1, clientID2} {
|
||||
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
|
||||
Clientid: clientID,
|
||||
Name: fmt.Sprintf("Test Client %s", clientID),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Create processing batches (should be returned)
|
||||
var expectedBatchIDs []string
|
||||
|
||||
// Client 1: 2 processing batches
|
||||
for i := 0; i < 2; i++ {
|
||||
batchID, err := service.Create(ctx, clientID1, fmt.Sprintf("processing%d.zip", i), 10)
|
||||
require.NoError(t, err)
|
||||
expectedBatchIDs = append(expectedBatchIDs, batchID.String())
|
||||
}
|
||||
|
||||
// Client 2: 1 processing batch
|
||||
batchID, err := service.Create(ctx, clientID2, "processing_client2.zip", 5)
|
||||
require.NoError(t, err)
|
||||
expectedBatchIDs = append(expectedBatchIDs, batchID.String())
|
||||
|
||||
// Create completed batch (should NOT be returned)
|
||||
completedBatchID, err := service.Create(ctx, clientID1, "completed.zip", 8)
|
||||
require.NoError(t, err)
|
||||
err = service.MarkCompleted(ctx, completedBatchID)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create failed batch (should NOT be returned)
|
||||
failedBatchID, err := service.Create(ctx, clientID2, "failed.zip", 3)
|
||||
require.NoError(t, err)
|
||||
err = service.MarkFailed(ctx, failedBatchID)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Test: Get unprocessed batches
|
||||
unprocessedBatches, err := service.ListUnprocessed(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify: Should return exactly 3 processing batches
|
||||
assert.Len(t, unprocessedBatches, 3)
|
||||
|
||||
// Verify: All returned batches have processing status
|
||||
actualBatchIDs := make([]string, len(unprocessedBatches))
|
||||
for i, batch := range unprocessedBatches {
|
||||
assert.Equal(t, documentbatch.StatusProcessing, batch.Status)
|
||||
actualBatchIDs[i] = batch.ID.String()
|
||||
}
|
||||
|
||||
// Verify: All processing batches are returned
|
||||
for _, expectedID := range expectedBatchIDs {
|
||||
assert.Contains(t, actualBatchIDs, expectedID)
|
||||
}
|
||||
|
||||
// Verify: Completed and failed batches are NOT returned
|
||||
assert.NotContains(t, actualBatchIDs, completedBatchID.String())
|
||||
assert.NotContains(t, actualBatchIDs, failedBatchID.String())
|
||||
|
||||
// Verify: Batches from both clients are returned
|
||||
clientIDs := make(map[string]bool)
|
||||
for _, batch := range unprocessedBatches {
|
||||
clientIDs[batch.ClientID] = true
|
||||
}
|
||||
assert.True(t, clientIDs[clientID1], "should include batches from client1")
|
||||
assert.True(t, clientIDs[clientID2], "should include batches from client2")
|
||||
|
||||
// Verify: Batches are ordered by created_at ASC (oldest first)
|
||||
if len(unprocessedBatches) > 1 {
|
||||
for i := 0; i < len(unprocessedBatches)-1; i++ {
|
||||
assert.True(t,
|
||||
unprocessedBatches[i].CreatedAt.Before(unprocessedBatches[i+1].CreatedAt) ||
|
||||
unprocessedBatches[i].CreatedAt.Equal(unprocessedBatches[i+1].CreatedAt),
|
||||
"batches should be ordered by created_at ASC")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestListUnprocessed_EmptyResult(t *testing.T) {
|
||||
cfg := &TestConfig{}
|
||||
test.CreateDB(t, cfg)
|
||||
|
||||
service := documentbatch.New(cfg)
|
||||
ctx := t.Context()
|
||||
|
||||
// Create test client
|
||||
clientID := "test_client_empty"
|
||||
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
|
||||
Clientid: clientID,
|
||||
Name: "Test Client Empty",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create only completed batches (none processing)
|
||||
for i := 0; i < 2; i++ {
|
||||
batchID, err := service.Create(ctx, clientID, fmt.Sprintf("completed%d.zip", i), 10)
|
||||
require.NoError(t, err)
|
||||
err = service.MarkCompleted(ctx, batchID)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Test: Get unprocessed batches when none exist
|
||||
unprocessedBatches, err := service.ListUnprocessed(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify: Should return empty list
|
||||
assert.Empty(t, unprocessedBatches)
|
||||
}
|
||||
|
||||
func TestListUnprocessed_DatabaseError(t *testing.T) {
|
||||
cfg := &TestConfig{}
|
||||
test.CreateDB(t, cfg)
|
||||
|
||||
service := documentbatch.New(cfg)
|
||||
|
||||
// Use a canceled context to simulate database error
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
cancel() // Cancel immediately
|
||||
|
||||
// Test: Try to get unprocessed batches with canceled context
|
||||
unprocessedBatches, err := service.ListUnprocessed(ctx)
|
||||
|
||||
// Verify: Should return error
|
||||
require.Error(t, err)
|
||||
assert.Nil(t, unprocessedBatches)
|
||||
assert.Contains(t, err.Error(), "failed to query unprocessed batches")
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
@@ -28,6 +29,17 @@ func New(cfg objectstore.ConfigProvider) *Service {
|
||||
return &Service{cfg: cfg}
|
||||
}
|
||||
|
||||
// detectContentType determines the MIME type based on file extension
|
||||
func detectContentType(filename string) string {
|
||||
// Use Go's standard library to detect MIME type from extension
|
||||
contentType := mime.TypeByExtension(filepath.Ext(filename))
|
||||
if contentType == "" {
|
||||
// Default to binary if extension is unknown
|
||||
return "application/octet-stream"
|
||||
}
|
||||
return contentType
|
||||
}
|
||||
|
||||
// StoreZIPFile uploads ZIP file to S3 and returns storage metadata
|
||||
func (s *Service) StoreZIPFile(ctx context.Context, clientID string, filename string, content io.Reader) (*StorageResult, error) {
|
||||
// Create unique S3 key with timestamp and UUID
|
||||
@@ -45,11 +57,15 @@ func (s *Service) StoreZIPFile(ctx context.Context, clientID string, filename st
|
||||
return nil, fmt.Errorf("failed to read file content: %w", err)
|
||||
}
|
||||
|
||||
// Detect content type from filename
|
||||
contentType := detectContentType(filename)
|
||||
|
||||
// Upload to S3 using the store client
|
||||
_, err = s.cfg.GetStoreClient().PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: &bucketName,
|
||||
Key: &key,
|
||||
Body: bytes.NewReader(buf.Bytes()),
|
||||
Bucket: &bucketName,
|
||||
Key: &key,
|
||||
Body: bytes.NewReader(buf.Bytes()),
|
||||
ContentType: &contentType,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to upload ZIP to S3: %w", err)
|
||||
|
||||
Reference in New Issue
Block a user