Merged in feature/batch-status (pull request #215)
Implement and test the batch status feature * working
This commit is contained in:
@@ -0,0 +1,87 @@
|
||||
// Package outcome provides a shared reporter that runners use to write
|
||||
// pipeline stage outcomes to the batch_document_outcomes table.
|
||||
// For non-batch documents (no outcome row exists), updates are a natural
|
||||
// no-op (0 rows affected).
|
||||
package outcome
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"queryorchestration/internal/database/repository"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// Reporter writes pipeline stage outcomes to batch_document_outcomes.
|
||||
// Used by runners to report their processing results for batch-uploaded documents.
|
||||
// For non-batch documents (no outcome row exists), updates are a no-op (0 rows affected).
|
||||
//
|
||||
// Fields:
|
||||
// - queries: SQLC-generated database queries
|
||||
type Reporter struct {
|
||||
queries *repository.Queries
|
||||
}
|
||||
|
||||
// New creates an outcome Reporter.
|
||||
//
|
||||
// Parameters:
|
||||
// - q: SQLC-generated queries instance (from cfg.GetDBQueries())
|
||||
//
|
||||
// Returns:
|
||||
// - *Reporter: the reporter instance
|
||||
func New(q *repository.Queries) *Reporter {
|
||||
return &Reporter{queries: q}
|
||||
}
|
||||
|
||||
// Resolve sets the document_id on a batch outcome row and updates the outcome.
|
||||
// Called by docInitRunner after creating or finding the document.
|
||||
// Matches on batch_id + filename since document_id is NULL before init.
|
||||
//
|
||||
// Parameters:
|
||||
// - ctx: request context
|
||||
// - batchID: the batch this file belongs to (from the document's batch_id)
|
||||
// - filename: original filename from the ZIP (from the document record)
|
||||
// - documentID: the created or found document ID
|
||||
// - outcomeStatus: the init-stage outcome ("init_complete" or "init_duplicate")
|
||||
// - errorDetail: optional error message (nil for success)
|
||||
//
|
||||
// Returns:
|
||||
// - error: database error, or nil
|
||||
//
|
||||
// If batchID is nil, this is a no-op (document was not from a batch upload).
|
||||
func (r *Reporter) Resolve(ctx context.Context, batchID *uuid.UUID, filename string,
|
||||
documentID uuid.UUID, outcomeStatus repository.BatchOutcomeStatus, errorDetail *string) error {
|
||||
if batchID == nil {
|
||||
return nil
|
||||
}
|
||||
return r.queries.ResolveBatchDocumentOutcome(ctx, &repository.ResolveBatchDocumentOutcomeParams{
|
||||
BatchID: *batchID,
|
||||
Filename: filename,
|
||||
DocumentID: &documentID,
|
||||
Column4: outcomeStatus,
|
||||
ErrorDetail: errorDetail,
|
||||
})
|
||||
}
|
||||
|
||||
// Update updates the outcome for a document that already has document_id set.
|
||||
// Called by docSyncRunner and docCleanRunner after processing.
|
||||
//
|
||||
// Parameters:
|
||||
// - ctx: request context
|
||||
// - documentID: the document being processed
|
||||
// - outcomeStatus: the pipeline outcome (e.g. "sync_complete", "clean_passed")
|
||||
// - errorDetail: optional error message (nil for success)
|
||||
//
|
||||
// Returns:
|
||||
// - error: database error, or nil
|
||||
//
|
||||
// If no outcome row exists for this document_id (non-batch document), this
|
||||
// affects 0 rows and is a no-op.
|
||||
func (r *Reporter) Update(ctx context.Context, documentID uuid.UUID,
|
||||
outcomeStatus repository.BatchOutcomeStatus, errorDetail *string) error {
|
||||
return r.queries.UpdateBatchDocumentOutcomeByDocumentID(ctx, &repository.UpdateBatchDocumentOutcomeByDocumentIDParams{
|
||||
DocumentID: &documentID,
|
||||
Column2: outcomeStatus,
|
||||
ErrorDetail: errorDetail,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
// Package outcome_test contains integration tests for the batch outcome reporter.
|
||||
// Tests use real database via testcontainers (no mocks).
|
||||
package outcome_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"queryorchestration/internal/database/repository"
|
||||
"queryorchestration/internal/document/batch/outcome"
|
||||
"queryorchestration/internal/serviceconfig"
|
||||
"queryorchestration/internal/test"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type TestConfig struct {
|
||||
serviceconfig.BaseConfig
|
||||
}
|
||||
|
||||
// createBatchWithOutcome is a helper that creates a client, batch, and inserts
|
||||
// a submitted outcome row. Returns the batchID and the unique clientID used.
|
||||
func createBatchWithOutcome(t *testing.T, cfg *TestConfig, clientPrefix, filename string) (uuid.UUID, string) {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
|
||||
// Use a unique clientID to avoid collisions when tests share the DB
|
||||
clientID := clientPrefix + "_" + uuid.New().String()[:8]
|
||||
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
|
||||
Clientid: clientID,
|
||||
Name: "Test Client " + clientID,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
batchID, err := cfg.GetDBQueries().CreateBatchUpload(ctx, &repository.CreateBatchUploadParams{
|
||||
ClientID: clientID,
|
||||
OriginalFilename: "test.zip",
|
||||
TotalDocuments: 5,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
err = cfg.GetDBQueries().InsertBatchDocumentOutcome(ctx, &repository.InsertBatchDocumentOutcomeParams{
|
||||
BatchID: batchID,
|
||||
Filename: filename,
|
||||
Column3: repository.BatchOutcomeStatusSubmitted,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
return batchID, clientID
|
||||
}
|
||||
|
||||
// TestResolve_SetsDocumentIDAndOutcome inserts a submitted row, resolves it
|
||||
// with a document_id, and verifies the update took effect.
|
||||
func TestResolve_SetsDocumentIDAndOutcome(t *testing.T) {
|
||||
cfg := &TestConfig{}
|
||||
test.CreateDB(t, cfg)
|
||||
ctx := t.Context()
|
||||
|
||||
filename := "invoice.pdf"
|
||||
batchID, clientID := createBatchWithOutcome(t, cfg, "test_resolve_sets", filename)
|
||||
|
||||
// Create a real document so FK constraint is satisfied
|
||||
docID, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
|
||||
Clientid: clientID,
|
||||
Hash: "hash_resolve_test",
|
||||
BatchID: &batchID,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
reporter := outcome.New(cfg.GetDBQueries())
|
||||
|
||||
err = reporter.Resolve(ctx, &batchID, filename, docID,
|
||||
repository.BatchOutcomeStatusInitComplete, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify the row was updated
|
||||
rows, err := cfg.GetDBQueries().ListBatchDocumentOutcomes(ctx, batchID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 1)
|
||||
|
||||
assert.Equal(t, repository.BatchOutcomeStatusInitComplete, rows[0].Outcome)
|
||||
require.NotNil(t, rows[0].DocumentID)
|
||||
assert.Equal(t, docID, *rows[0].DocumentID)
|
||||
}
|
||||
|
||||
// TestResolve_NoOpForNilBatchID verifies that calling Resolve with a nil batchID
|
||||
// returns no error and makes no DB changes.
|
||||
func TestResolve_NoOpForNilBatchID(t *testing.T) {
|
||||
cfg := &TestConfig{}
|
||||
test.CreateDB(t, cfg)
|
||||
ctx := t.Context()
|
||||
|
||||
reporter := outcome.New(cfg.GetDBQueries())
|
||||
|
||||
// Call with nil batchID -- should be a no-op
|
||||
err := reporter.Resolve(ctx, nil, "anything.pdf", uuid.New(),
|
||||
repository.BatchOutcomeStatusInitComplete, nil)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// TestUpdate_UpdatesOutcomeByDocumentID inserts a resolved row (with document_id),
|
||||
// then updates it to sync_complete and verifies.
|
||||
func TestUpdate_UpdatesOutcomeByDocumentID(t *testing.T) {
|
||||
cfg := &TestConfig{}
|
||||
test.CreateDB(t, cfg)
|
||||
ctx := t.Context()
|
||||
|
||||
filename := "receipt.pdf"
|
||||
batchID, clientID := createBatchWithOutcome(t, cfg, "test_update_docid", filename)
|
||||
|
||||
// Create a real document so FK constraint is satisfied
|
||||
docID, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
|
||||
Clientid: clientID,
|
||||
Hash: "hash_update_test",
|
||||
BatchID: &batchID,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
reporter := outcome.New(cfg.GetDBQueries())
|
||||
|
||||
// Resolve first (set document_id and advance to init_complete)
|
||||
err = reporter.Resolve(ctx, &batchID, filename, docID,
|
||||
repository.BatchOutcomeStatusInitComplete, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Now update to sync_complete
|
||||
err = reporter.Update(ctx, docID, repository.BatchOutcomeStatusSyncComplete, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify the outcome was advanced
|
||||
rows, err := cfg.GetDBQueries().ListBatchDocumentOutcomes(ctx, batchID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, rows, 1)
|
||||
assert.Equal(t, repository.BatchOutcomeStatusSyncComplete, rows[0].Outcome)
|
||||
}
|
||||
|
||||
// TestUpdate_NoOpForNonBatchDocument verifies that calling Update for a
|
||||
// document_id that has no outcome row produces no error (0 rows affected).
|
||||
func TestUpdate_NoOpForNonBatchDocument(t *testing.T) {
|
||||
cfg := &TestConfig{}
|
||||
test.CreateDB(t, cfg)
|
||||
ctx := t.Context()
|
||||
|
||||
reporter := outcome.New(cfg.GetDBQueries())
|
||||
|
||||
// Call Update with a random document_id that has no outcome row
|
||||
err := reporter.Update(ctx, uuid.New(), repository.BatchOutcomeStatusSyncComplete, nil)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
@@ -2,7 +2,9 @@ package batch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
@@ -36,13 +38,40 @@ type BatchUploadSummary struct {
|
||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||
}
|
||||
|
||||
// BatchUploadDetails includes full details with failed filenames
|
||||
// BatchDocumentOutcome represents the per-file result from batch extraction
|
||||
// and pipeline processing. This is the single source of truth for a file's
|
||||
// status within a batch.
|
||||
//
|
||||
// Fields:
|
||||
// - ID: unique outcome record ID
|
||||
// - BatchID: the batch this file belongs to
|
||||
// - Filename: original filename from the ZIP
|
||||
// - Outcome: current status (extraction outcome or pipeline stage)
|
||||
// - ErrorDetail: human-readable error/failure message, nil for success states
|
||||
// - DocumentID: resolved document ID (nil if document not yet created)
|
||||
// - CleanFail: specific clean failure reason from documentCleans table (nil if N/A)
|
||||
// - CreatedAt: when the outcome row was created (extraction time)
|
||||
// - UpdatedAt: when the outcome was last changed (pipeline progression)
|
||||
type BatchDocumentOutcome struct {
|
||||
ID uuid.UUID
|
||||
BatchID uuid.UUID
|
||||
Filename string
|
||||
Outcome string
|
||||
ErrorDetail *string
|
||||
DocumentID *uuid.UUID
|
||||
CleanFail *string
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
// BatchUploadDetails includes full details with failed filenames and document outcomes
|
||||
type BatchUploadDetails struct {
|
||||
BatchUploadSummary
|
||||
FailedFilenames []string `json:"failed_filenames"`
|
||||
ArchiveBucket string `json:"archive_bucket,omitempty"`
|
||||
ArchiveKey string `json:"archive_key,omitempty"`
|
||||
FileSizeBytes int64 `json:"file_size_bytes,omitempty"`
|
||||
FailedFilenames []string `json:"failed_filenames"`
|
||||
DocumentOutcomes []*BatchDocumentOutcome `json:"document_outcomes,omitempty"`
|
||||
ArchiveBucket string `json:"archive_bucket,omitempty"`
|
||||
ArchiveKey string `json:"archive_key,omitempty"`
|
||||
FileSizeBytes int64 `json:"file_size_bytes,omitempty"`
|
||||
}
|
||||
|
||||
// Service handles batch upload operations
|
||||
@@ -84,9 +113,18 @@ func (s *Service) CreateWithStorage(ctx context.Context, clientID string, filena
|
||||
return result.ID, nil
|
||||
}
|
||||
|
||||
// Get retrieves batch upload details
|
||||
// Get retrieves batch upload details including per-file document outcomes.
|
||||
//
|
||||
// Parameters:
|
||||
// - ctx: request context
|
||||
// - clientID: the client identifier
|
||||
// - batchID: the batch UUID
|
||||
//
|
||||
// Returns:
|
||||
// - *BatchUploadDetails: full batch details with document outcomes
|
||||
// - error: database error, or nil
|
||||
func (s *Service) Get(ctx context.Context, clientID string, batchID uuid.UUID) (*BatchUploadDetails, error) {
|
||||
batch, err := s.cfg.GetDBQueries().GetBatchUploadWithStorage(ctx, &repository.GetBatchUploadWithStorageParams{
|
||||
batchRow, err := s.cfg.GetDBQueries().GetBatchUploadWithStorage(ctx, &repository.GetBatchUploadWithStorageParams{
|
||||
ID: batchID,
|
||||
ClientID: clientID,
|
||||
})
|
||||
@@ -94,7 +132,16 @@ func (s *Service) Get(ctx context.Context, clientID string, batchID uuid.UUID) (
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return s.convertToDetailsWithStorage(batch), nil
|
||||
details := s.convertToDetailsWithStorage(batchRow)
|
||||
|
||||
// Populate document outcomes
|
||||
outcomes, err := s.ListOutcomes(ctx, batchID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list batch outcomes: %w", err)
|
||||
}
|
||||
details.DocumentOutcomes = outcomes
|
||||
|
||||
return details, nil
|
||||
}
|
||||
|
||||
// List retrieves all batch uploads for a client with pagination
|
||||
@@ -272,6 +319,94 @@ func (s *Service) convertToDetailsWithStorage(batch *repository.GetBatchUploadWi
|
||||
return details
|
||||
}
|
||||
|
||||
// RecordOutcome inserts or upserts a per-file outcome row for a batch document.
|
||||
// Called by the batch worker during ZIP extraction to record the initial outcome
|
||||
// for each file.
|
||||
//
|
||||
// Parameters:
|
||||
// - ctx: request context
|
||||
// - batchID: the batch this file belongs to
|
||||
// - filename: original filename from the ZIP
|
||||
// - outcome: the typed outcome status (e.g. repository.BatchOutcomeStatusSubmitted)
|
||||
// - errorDetail: optional error message for failure outcomes
|
||||
// - documentID: optional document ID (set for duplicates)
|
||||
//
|
||||
// Returns:
|
||||
// - error: database error, or nil
|
||||
func (s *Service) RecordOutcome(ctx context.Context, batchID uuid.UUID, filename string,
|
||||
outcome repository.BatchOutcomeStatus, errorDetail *string, documentID *uuid.UUID) error {
|
||||
return s.cfg.GetDBQueries().InsertBatchDocumentOutcome(ctx, &repository.InsertBatchDocumentOutcomeParams{
|
||||
BatchID: batchID,
|
||||
Filename: filename,
|
||||
Column3: outcome,
|
||||
ErrorDetail: errorDetail,
|
||||
DocumentID: documentID,
|
||||
})
|
||||
}
|
||||
|
||||
// ListOutcomes retrieves all per-file outcome rows for a batch, with clean
|
||||
// failure detail joined from the currentCleanEntries view.
|
||||
//
|
||||
// Parameters:
|
||||
// - ctx: request context
|
||||
// - batchID: the batch UUID
|
||||
//
|
||||
// Returns:
|
||||
// - []*BatchDocumentOutcome: list of outcome rows ordered by creation time
|
||||
// - error: database error, or nil
|
||||
func (s *Service) ListOutcomes(ctx context.Context, batchID uuid.UUID) ([]*BatchDocumentOutcome, error) {
|
||||
rows, err := s.cfg.GetDBQueries().ListBatchDocumentOutcomes(ctx, batchID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
outcomes := make([]*BatchDocumentOutcome, len(rows))
|
||||
for i, row := range rows {
|
||||
o := &BatchDocumentOutcome{
|
||||
ID: row.ID,
|
||||
BatchID: row.BatchID,
|
||||
Filename: row.Filename,
|
||||
Outcome: string(row.Outcome),
|
||||
ErrorDetail: row.ErrorDetail,
|
||||
DocumentID: row.DocumentID,
|
||||
CreatedAt: s.pgTimestampToTime(row.CreatedAt),
|
||||
UpdatedAt: s.pgTimestampToTime(row.UpdatedAt),
|
||||
}
|
||||
if row.CleanFail.Valid {
|
||||
failStr := string(row.CleanFail.Cleanfailtype)
|
||||
o.CleanFail = &failStr
|
||||
}
|
||||
outcomes[i] = o
|
||||
}
|
||||
|
||||
return outcomes, nil
|
||||
}
|
||||
|
||||
// CheckDuplicate checks if a document with the given hash already exists for
|
||||
// the specified client. Returns the existing document's UUID if found, nil otherwise.
|
||||
//
|
||||
// Parameters:
|
||||
// - ctx: request context
|
||||
// - clientID: the client identifier
|
||||
// - hash: the SHA-256 hash of the file content
|
||||
//
|
||||
// Returns:
|
||||
// - *uuid.UUID: existing document ID if a duplicate exists, nil otherwise
|
||||
// - error: database error (other than not-found), or nil
|
||||
func (s *Service) CheckDuplicate(ctx context.Context, clientID string, hash string) (*uuid.UUID, error) {
|
||||
id, err := s.cfg.GetDBQueries().GetDocumentIDByHash(ctx, &repository.GetDocumentIDByHashParams{
|
||||
Clientid: clientID,
|
||||
Hash: hash,
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return &id, nil
|
||||
}
|
||||
|
||||
// ListUnprocessed retrieves all unprocessed batch uploads across all clients
|
||||
func (s *Service) ListUnprocessed(ctx context.Context) ([]*BatchUploadSummary, error) {
|
||||
batches, err := s.cfg.GetDBQueries().GetUnprocessedBatches(ctx)
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"queryorchestration/internal/test"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -559,3 +560,194 @@ func TestListUnprocessed_DatabaseError(t *testing.T) {
|
||||
assert.Nil(t, unprocessedBatches)
|
||||
assert.Contains(t, err.Error(), "failed to query unprocessed batches")
|
||||
}
|
||||
|
||||
// TestRecordOutcome verifies that RecordOutcome inserts a per-file outcome row
|
||||
// and that it can be read back via ListOutcomes.
|
||||
func TestRecordOutcome(t *testing.T) {
|
||||
cfg := &TestConfig{}
|
||||
test.CreateDB(t, cfg)
|
||||
ctx := t.Context()
|
||||
|
||||
service := documentbatch.New(cfg)
|
||||
|
||||
clientID := "test_record_outcome_" + uuid.New().String()[:8]
|
||||
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
|
||||
Clientid: clientID,
|
||||
Name: "Test Client Record Outcome",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
batchID, err := service.Create(ctx, clientID, "outcomes.zip", 3)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Insert multiple outcomes
|
||||
err = service.RecordOutcome(ctx, batchID, "good.pdf", repository.BatchOutcomeStatusSubmitted, nil, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
errMsg := "file is not a PDF"
|
||||
err = service.RecordOutcome(ctx, batchID, "notes.txt", repository.BatchOutcomeStatusInvalidType, &errMsg, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify persistence via ListOutcomes
|
||||
outcomes, err := service.ListOutcomes(ctx, batchID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, outcomes, 2)
|
||||
|
||||
// Verify the outcome details
|
||||
found := map[string]string{}
|
||||
for _, o := range outcomes {
|
||||
found[o.Filename] = o.Outcome
|
||||
}
|
||||
assert.Equal(t, "submitted", found["good.pdf"])
|
||||
assert.Equal(t, "invalid_type", found["notes.txt"])
|
||||
}
|
||||
|
||||
// TestListOutcomes inserts outcomes for various states and verifies the listing.
|
||||
func TestListOutcomes(t *testing.T) {
|
||||
cfg := &TestConfig{}
|
||||
test.CreateDB(t, cfg)
|
||||
ctx := t.Context()
|
||||
|
||||
service := documentbatch.New(cfg)
|
||||
|
||||
clientID := "test_list_outcomes_" + uuid.New().String()[:8]
|
||||
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
|
||||
Clientid: clientID,
|
||||
Name: "Test Client List Outcomes",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
batchID, err := service.Create(ctx, clientID, "list.zip", 5)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Insert a variety of outcomes
|
||||
err = service.RecordOutcome(ctx, batchID, "a.pdf", repository.BatchOutcomeStatusSubmitted, nil, nil)
|
||||
require.NoError(t, err)
|
||||
err = service.RecordOutcome(ctx, batchID, "b.pdf", repository.BatchOutcomeStatusSubmitted, nil, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
errOpen := "cannot open"
|
||||
err = service.RecordOutcome(ctx, batchID, "c.pdf", repository.BatchOutcomeStatusFailedOpen, &errOpen, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = service.RecordOutcome(ctx, batchID, "d.txt", repository.BatchOutcomeStatusInvalidType, nil, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
dupID := uuid.New()
|
||||
// Create a real document for the duplicate reference
|
||||
realDocID, createErr := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
|
||||
Clientid: clientID,
|
||||
Hash: "hash_dup",
|
||||
BatchID: &batchID,
|
||||
})
|
||||
require.NoError(t, createErr)
|
||||
_ = dupID // not needed since we use the real ID
|
||||
err = service.RecordOutcome(ctx, batchID, "e.pdf", repository.BatchOutcomeStatusDuplicate, nil, &realDocID)
|
||||
require.NoError(t, err)
|
||||
|
||||
outcomes, err := service.ListOutcomes(ctx, batchID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, outcomes, 5)
|
||||
|
||||
// Verify ordering is by created_at ASC
|
||||
expectedFilenames := []string{"a.pdf", "b.pdf", "c.pdf", "d.txt", "e.pdf"}
|
||||
for i, o := range outcomes {
|
||||
assert.Equal(t, expectedFilenames[i], o.Filename)
|
||||
}
|
||||
|
||||
// Verify the duplicate has document_id set
|
||||
assert.NotNil(t, outcomes[4].DocumentID)
|
||||
assert.Equal(t, realDocID, *outcomes[4].DocumentID)
|
||||
}
|
||||
|
||||
// TestListOutcomes_WithCleanFail verifies that the clean_fail column from
|
||||
// the currentCleanEntries view is joined when outcome is clean_failed.
|
||||
func TestListOutcomes_WithCleanFail(t *testing.T) {
|
||||
cfg := &TestConfig{}
|
||||
test.CreateDB(t, cfg)
|
||||
ctx := t.Context()
|
||||
|
||||
service := documentbatch.New(cfg)
|
||||
|
||||
clientID := "test_clean_fail_" + uuid.New().String()[:8]
|
||||
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
|
||||
Clientid: clientID,
|
||||
Name: "Test Client Clean Fail",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
batchID, err := service.Create(ctx, clientID, "cleanfail.zip", 1)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create a real document
|
||||
docID, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
|
||||
Clientid: clientID,
|
||||
Hash: "hash_clean_fail",
|
||||
BatchID: &batchID,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Insert a clean_failed outcome
|
||||
err = service.RecordOutcome(ctx, batchID, "corrupt.pdf", repository.BatchOutcomeStatusCleanFailed, nil, &docID)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create a documentClean with fail type and a documentCleanEntry to populate
|
||||
// the currentCleanEntries view
|
||||
cleanID, err := cfg.GetDBQueries().AddDocumentClean(ctx, &repository.AddDocumentCleanParams{
|
||||
Documentid: docID,
|
||||
Fail: repository.NullCleanfailtype{
|
||||
Cleanfailtype: repository.CleanfailtypeInvalidMimetype,
|
||||
Valid: true,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
err = cfg.GetDBQueries().AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
|
||||
Cleanid: cleanID,
|
||||
Version: 1,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
outcomes, err := service.ListOutcomes(ctx, batchID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, outcomes, 1)
|
||||
|
||||
assert.Equal(t, "clean_failed", outcomes[0].Outcome)
|
||||
require.NotNil(t, outcomes[0].CleanFail)
|
||||
assert.Equal(t, "invalid_mimetype", *outcomes[0].CleanFail)
|
||||
}
|
||||
|
||||
// TestCheckDuplicate creates a document, then verifies that CheckDuplicate
|
||||
// returns its ID when the hash matches.
|
||||
func TestCheckDuplicate(t *testing.T) {
|
||||
cfg := &TestConfig{}
|
||||
test.CreateDB(t, cfg)
|
||||
ctx := t.Context()
|
||||
|
||||
service := documentbatch.New(cfg)
|
||||
|
||||
clientID := "test_check_dup_" + uuid.New().String()[:8]
|
||||
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
|
||||
Clientid: clientID,
|
||||
Name: "Test Client Check Dup",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create a document with a known hash
|
||||
knownHash := "sha256_known_hash_abcdef"
|
||||
docID, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
|
||||
Clientid: clientID,
|
||||
Hash: knownHash,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Check duplicate with the same hash -- should find it
|
||||
existingID, err := service.CheckDuplicate(ctx, clientID, knownHash)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, existingID)
|
||||
assert.Equal(t, docID, *existingID)
|
||||
|
||||
// Check duplicate with a different hash -- should return nil
|
||||
noMatch, err := service.CheckDuplicate(ctx, clientID, "totally_different_hash")
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, noMatch)
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
|
||||
"queryorchestration/internal/database/repository"
|
||||
@@ -82,23 +81,34 @@ func (s *Service) executeCleanTasks(ctx context.Context, params *CleanParams) (*
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Service) clean(ctx context.Context, id uuid.UUID) error {
|
||||
// clean runs the document validation and stores the result. Returns a CleanResult
|
||||
// indicating whether the document passed validation. Infrastructure errors (DB, S3)
|
||||
// are returned as the error value; validation failures are returned via CleanResult.
|
||||
//
|
||||
// Parameters:
|
||||
// - ctx: request context
|
||||
// - id: the document UUID to clean
|
||||
//
|
||||
// Returns:
|
||||
// - *CleanResult: result with Passed=true on success, or Passed=false with FailReason
|
||||
// - error: only for infrastructure errors
|
||||
func (s *Service) clean(ctx context.Context, id uuid.UUID) (*CleanResult, error) {
|
||||
slog.Debug("cleaning document", "id", id.String())
|
||||
docId := id
|
||||
|
||||
doc, err := s.cfg.GetDBQueries().GetDocumentSummary(ctx, docId)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
entry, err := s.cfg.GetDBQueries().GetDocumentEntry(ctx, docId)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
key, err := objectstore.ParseBucketKey(entry.Key)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out, err := s.executeCleanTasks(ctx, &CleanParams{
|
||||
@@ -108,18 +118,31 @@ func (s *Service) clean(ctx context.Context, id uuid.UUID) error {
|
||||
Key: key,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = s.storeClean(ctx, id, out)
|
||||
result, err := s.storeClean(ctx, id, out)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return nil
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *Service) storeClean(ctx context.Context, id uuid.UUID, out *ExecuteCleanResponse) error {
|
||||
// storeClean persists the clean result to the database. Returns a CleanResult
|
||||
// indicating whether the document passed validation. Validation failures are
|
||||
// returned via CleanResult (not as errors) so callers can distinguish them
|
||||
// from infrastructure errors.
|
||||
//
|
||||
// Parameters:
|
||||
// - ctx: request context
|
||||
// - id: the document UUID
|
||||
// - out: the result from executeCleanTasks
|
||||
//
|
||||
// Returns:
|
||||
// - *CleanResult: Passed=true if validation succeeded, Passed=false with FailReason otherwise
|
||||
// - error: only for infrastructure errors (DB transaction failures)
|
||||
func (s *Service) storeClean(ctx context.Context, id uuid.UUID, out *ExecuteCleanResponse) (*CleanResult, error) {
|
||||
docId := id
|
||||
version := build.GetVersionUnixTimestamp()
|
||||
|
||||
@@ -164,14 +187,15 @@ func (s *Service) storeClean(ctx context.Context, id uuid.UUID, out *ExecuteClea
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if out.failReason != nil {
|
||||
return fmt.Errorf("%s", *out.failReason)
|
||||
failStr := string(*out.failReason)
|
||||
return &CleanResult{Passed: false, FailReason: &failStr}, nil
|
||||
}
|
||||
|
||||
return nil
|
||||
return &CleanResult{Passed: true}, nil
|
||||
}
|
||||
|
||||
func (s *Service) isNewClean(lastEntry *repository.GetMostRecentDocumentCleanEntryRow, out *ExecuteCleanResponse) bool {
|
||||
|
||||
@@ -10,21 +10,41 @@ import (
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// CleanResult contains the result of the clean operation.
|
||||
//
|
||||
// Fields:
|
||||
// - Passed: true if the document passed all validation checks
|
||||
// - FailReason: the specific failure reason if Passed is false, nil otherwise
|
||||
type CleanResult struct {
|
||||
Passed bool
|
||||
FailReason *string
|
||||
}
|
||||
|
||||
// Clean processes a document by cleaning it. Previously this also triggered
|
||||
// text extraction, but that feature has been removed.
|
||||
func (s *Service) Clean(ctx context.Context, documentId uuid.UUID) error {
|
||||
//
|
||||
// Parameters:
|
||||
// - ctx: request context
|
||||
// - documentId: the document UUID to clean
|
||||
//
|
||||
// Returns:
|
||||
// - *CleanResult: result indicating whether validation passed and any failure reason
|
||||
// - error: only for infrastructure errors (DB, S3). Validation failures are
|
||||
// returned via CleanResult.Passed=false, not as errors.
|
||||
func (s *Service) Clean(ctx context.Context, documentId uuid.UUID) (*CleanResult, error) {
|
||||
isclean, err := s.cfg.GetDBQueries().HasDocumentCleanEntry(ctx, documentId)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to verify if document has been cleaned: %w", err)
|
||||
return nil, fmt.Errorf("unable to verify if document has been cleaned: %w", err)
|
||||
}
|
||||
|
||||
if !isclean {
|
||||
err = s.clean(ctx, documentId)
|
||||
result, err := s.clean(ctx, documentId)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to clean document: %w", err)
|
||||
return nil, fmt.Errorf("unable to clean document: %w", err)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// Note: Text extraction has been removed. Document pipeline ends here.
|
||||
return nil
|
||||
return &CleanResult{Passed: true}, nil
|
||||
}
|
||||
|
||||
@@ -3,18 +3,26 @@
|
||||
package documentclean_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"queryorchestration/internal/database/repository"
|
||||
documentclean "queryorchestration/internal/document/clean"
|
||||
"queryorchestration/internal/serviceconfig"
|
||||
"queryorchestration/internal/serviceconfig/objectstore"
|
||||
"queryorchestration/internal/test"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type DocCleanConfig struct {
|
||||
serviceconfig.BaseConfig
|
||||
objectstore.ConfigProvider
|
||||
objectstore.ObjectStoreConfig
|
||||
}
|
||||
|
||||
func TestService(t *testing.T) {
|
||||
@@ -22,3 +30,104 @@ func TestService(t *testing.T) {
|
||||
svc := documentclean.New(cfg)
|
||||
assert.NotNil(t, svc)
|
||||
}
|
||||
|
||||
// TestDocumentCleanClean_ReturnsCleanResult verifies that Clean() returns a
|
||||
// *CleanResult with correct Passed and FailReason fields.
|
||||
//
|
||||
// Two scenarios are tested:
|
||||
// 1. Already-cleaned document (HasDocumentCleanEntry=true) returns Passed=true
|
||||
// 2. Non-PDF file returns Passed=false with a non-nil FailReason
|
||||
func TestDocumentCleanClean_ReturnsCleanResult(t *testing.T) {
|
||||
cfg := &DocCleanConfig{}
|
||||
test.CreateDB(t, cfg)
|
||||
test.CreateAWSResources(t, cfg)
|
||||
ctx := t.Context()
|
||||
|
||||
svc := documentclean.New(cfg)
|
||||
|
||||
// Test 1: Already cleaned document returns Passed=true (shortcut path)
|
||||
clientID := "clean_result_" + uuid.New().String()[:8]
|
||||
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
|
||||
Clientid: clientID,
|
||||
Name: "Test Clean Result",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
docID, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
|
||||
Clientid: clientID,
|
||||
Hash: "hash_clean_result_" + uuid.New().String()[:8],
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create a clean entry so HasDocumentCleanEntry returns true
|
||||
bucket := "bucket"
|
||||
key := "key"
|
||||
hash := "hash"
|
||||
cleanID, err := cfg.GetDBQueries().AddDocumentClean(ctx, &repository.AddDocumentCleanParams{
|
||||
Documentid: docID,
|
||||
Bucket: &bucket,
|
||||
Key: &key,
|
||||
Hash: &hash,
|
||||
Mimetype: repository.NullCleanmimetype{
|
||||
Cleanmimetype: repository.CleanmimetypeApplicationPdf,
|
||||
Valid: true,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
err = cfg.GetDBQueries().AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
|
||||
Cleanid: cleanID,
|
||||
Version: 1,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
result, err := svc.Clean(ctx, docID)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
assert.True(t, result.Passed, "already-cleaned document should return Passed=true")
|
||||
assert.Nil(t, result.FailReason, "Passed document should have nil FailReason")
|
||||
|
||||
// Test 2: Non-PDF file returns Passed=false with FailReason
|
||||
// Upload a non-PDF file to S3 first to get its ETag
|
||||
fileContent := []byte("This is NOT a PDF file. It has no PDF signature.")
|
||||
location := objectstore.BucketKey{
|
||||
Location: objectstore.Import,
|
||||
ClientID: clientID,
|
||||
CreatedAt: time.Now().UTC(),
|
||||
EntityID: uuid.New(),
|
||||
}
|
||||
_, err = cfg.GetStoreClient().PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(cfg.GetBucket()),
|
||||
Key: aws.String(location.String()),
|
||||
Body: bytes.NewReader(fileContent),
|
||||
ContentType: aws.String("text/plain"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get ETag to use as the document hash (IfMatch requires it)
|
||||
headResp, err := cfg.GetStoreClient().HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: aws.String(cfg.GetBucket()),
|
||||
Key: aws.String(location.String()),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
etag := *headResp.ETag
|
||||
|
||||
// Create document with hash matching the ETag
|
||||
docID2, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
|
||||
Clientid: clientID,
|
||||
Hash: etag,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
err = cfg.GetDBQueries().AddDocumentEntry(ctx, &repository.AddDocumentEntryParams{
|
||||
Documentid: docID2,
|
||||
Bucket: cfg.GetBucket(),
|
||||
Key: location.String(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
failResult, err := svc.Clean(ctx, docID2)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, failResult)
|
||||
assert.False(t, failResult.Passed, "non-PDF document should fail validation")
|
||||
assert.NotNil(t, failResult.FailReason, "failed document should have a FailReason")
|
||||
}
|
||||
|
||||
@@ -26,15 +26,39 @@ type Create struct {
|
||||
FolderID *uuid.UUID // Optional folder ID if folder hierarchy was pre-created
|
||||
}
|
||||
|
||||
func (s *Service) Create(ctx context.Context, doc *Create) (uuid.UUID, error) {
|
||||
// CreateResult contains the result of document creation.
|
||||
//
|
||||
// Fields:
|
||||
// - ID: the document's UUID (new or existing)
|
||||
// - IsDuplicate: true if a document with the same hash already existed
|
||||
// - BatchID: the batch this document belongs to (nil for non-batch uploads)
|
||||
// - Filename: the original filename (nil for non-batch uploads)
|
||||
type CreateResult struct {
|
||||
ID uuid.UUID
|
||||
IsDuplicate bool
|
||||
BatchID *uuid.UUID
|
||||
Filename *string
|
||||
}
|
||||
|
||||
// Create processes a new document upload by checking for duplicates, creating
|
||||
// database records, and forwarding new documents to the sync queue.
|
||||
//
|
||||
// Parameters:
|
||||
// - ctx: request context
|
||||
// - doc: document creation parameters (bucket, key, hash, filename, etc.)
|
||||
//
|
||||
// Returns:
|
||||
// - *CreateResult: result containing document ID, duplicate status, and batch info
|
||||
// - error: if any infrastructure operation (DB, S3, queue) fails
|
||||
func (s *Service) Create(ctx context.Context, doc *Create) (*CreateResult, error) {
|
||||
params, err := s.getCreateParams(ctx, doc)
|
||||
if err != nil {
|
||||
return uuid.Nil, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
id, err := s.submitCreate(ctx, params)
|
||||
if err != nil {
|
||||
return uuid.Nil, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if params.ID == nil {
|
||||
@@ -45,11 +69,16 @@ func (s *Service) Create(ctx context.Context, doc *Create) (uuid.UUID, error) {
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return uuid.Nil, err
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return id, nil
|
||||
return &CreateResult{
|
||||
ID: id,
|
||||
IsDuplicate: params.ID != nil,
|
||||
BatchID: doc.Key.BatchID,
|
||||
Filename: doc.Filename,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type createDocumentParams struct {
|
||||
|
||||
@@ -6,7 +6,10 @@ import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"queryorchestration/internal/database/repository"
|
||||
"queryorchestration/internal/serviceconfig/objectstore"
|
||||
"queryorchestration/internal/test"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
@@ -105,3 +108,130 @@ func TestMeasureFileSize_Integration_NonExistentKey(t *testing.T) {
|
||||
assert.Error(t, err, "measureFileSize should return an error for non-existent key")
|
||||
assert.Contains(t, err.Error(), "HeadObject failed", "error should indicate HeadObject failure")
|
||||
}
|
||||
|
||||
// TestDocumentInitCreate_ReturnsCreateResult verifies that Create() returns a
|
||||
// *CreateResult with correct IsDuplicate, BatchID, and Filename fields for both
|
||||
// new and duplicate documents.
|
||||
func TestDocumentInitCreate_ReturnsCreateResult(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
|
||||
// Set up DB, S3, and SQS
|
||||
cfg := &DocInitConfig{}
|
||||
test.CreateDB(t, cfg)
|
||||
acfg := test.CreateAWSContainer(t, cfg)
|
||||
|
||||
test.SetQueueClient(t, ctx, cfg, acfg.ExternalEndpoint)
|
||||
cfg.DocumentSyncURL = test.CreateQueue(t, cfg, test.DocSyncRunnerName)
|
||||
|
||||
test.SetStoreClient(t, ctx, cfg, acfg.ExternalEndpoint)
|
||||
test.CreateBucket(t, cfg)
|
||||
|
||||
svc := New(cfg)
|
||||
|
||||
clientID := "create_result_" + uuid.New().String()[:8]
|
||||
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
|
||||
Clientid: clientID,
|
||||
Name: "Test CreateResult",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create a batch
|
||||
batchID, err := cfg.GetDBQueries().CreateBatchUpload(ctx, &repository.CreateBatchUploadParams{
|
||||
ClientID: clientID,
|
||||
OriginalFilename: "test.zip",
|
||||
TotalDocuments: 2,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Upload a file to S3
|
||||
location := objectstore.BucketKey{
|
||||
Location: objectstore.Import,
|
||||
ClientID: clientID,
|
||||
CreatedAt: time.Now().UTC(),
|
||||
EntityID: uuid.New(),
|
||||
BatchID: &batchID,
|
||||
}
|
||||
testContent := []byte("test content for create result")
|
||||
_, err = cfg.GetStoreClient().PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(cfg.GetBucket()),
|
||||
Key: aws.String(location.String()),
|
||||
Body: bytes.NewReader(testContent),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
filename := "invoice.pdf"
|
||||
uniqueHash := "create_result_hash_" + uuid.New().String()[:8]
|
||||
|
||||
// Test 1: New document returns IsDuplicate=false with correct fields
|
||||
result, err := svc.Create(ctx, &Create{
|
||||
Bucket: cfg.GetBucket(),
|
||||
Key: location,
|
||||
Hash: uniqueHash,
|
||||
Filename: &filename,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
|
||||
assert.NotEqual(t, uuid.Nil, result.ID, "new document should have a valid ID")
|
||||
assert.False(t, result.IsDuplicate, "new document should not be a duplicate")
|
||||
assert.NotNil(t, result.BatchID, "batch document should have BatchID set")
|
||||
assert.Equal(t, batchID, *result.BatchID, "BatchID should match the batch")
|
||||
assert.NotNil(t, result.Filename, "Filename should be set")
|
||||
assert.Equal(t, filename, *result.Filename, "Filename should match")
|
||||
|
||||
// Test 2: Duplicate document (same hash) returns IsDuplicate=true
|
||||
location2 := objectstore.BucketKey{
|
||||
Location: objectstore.Import,
|
||||
ClientID: clientID,
|
||||
CreatedAt: time.Now().UTC(),
|
||||
EntityID: uuid.New(),
|
||||
BatchID: &batchID,
|
||||
}
|
||||
_, err = cfg.GetStoreClient().PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(cfg.GetBucket()),
|
||||
Key: aws.String(location2.String()),
|
||||
Body: bytes.NewReader(testContent),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
dupResult, err := svc.Create(ctx, &Create{
|
||||
Bucket: cfg.GetBucket(),
|
||||
Key: location2,
|
||||
Hash: uniqueHash, // Same hash triggers duplicate
|
||||
Filename: &filename,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, dupResult)
|
||||
|
||||
assert.Equal(t, result.ID, dupResult.ID, "duplicate should reference the original document ID")
|
||||
assert.True(t, dupResult.IsDuplicate, "second document with same hash should be duplicate")
|
||||
assert.NotNil(t, dupResult.BatchID, "duplicate should still have BatchID")
|
||||
assert.Equal(t, batchID, *dupResult.BatchID, "duplicate BatchID should match")
|
||||
|
||||
// Test 3: Non-batch document has nil BatchID
|
||||
location3 := objectstore.BucketKey{
|
||||
Location: objectstore.Import,
|
||||
ClientID: clientID,
|
||||
CreatedAt: time.Now().UTC(),
|
||||
EntityID: uuid.New(),
|
||||
// No BatchID
|
||||
}
|
||||
_, err = cfg.GetStoreClient().PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(cfg.GetBucket()),
|
||||
Key: aws.String(location3.String()),
|
||||
Body: bytes.NewReader(testContent),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
nonBatchResult, err := svc.Create(ctx, &Create{
|
||||
Bucket: cfg.GetBucket(),
|
||||
Key: location3,
|
||||
Hash: "nonbatch_hash_" + uuid.New().String()[:8],
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, nonBatchResult)
|
||||
|
||||
assert.False(t, nonBatchResult.IsDuplicate, "non-batch new document should not be duplicate")
|
||||
assert.Nil(t, nonBatchResult.BatchID, "non-batch document should have nil BatchID")
|
||||
assert.Nil(t, nonBatchResult.Filename, "non-batch document should have nil Filename")
|
||||
}
|
||||
|
||||
@@ -2,20 +2,132 @@ package documentsync_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"queryorchestration/internal/client"
|
||||
"queryorchestration/internal/database/repository"
|
||||
"queryorchestration/internal/document"
|
||||
documentsync "queryorchestration/internal/document/sync"
|
||||
"queryorchestration/internal/serviceconfig"
|
||||
"queryorchestration/internal/serviceconfig/objectstore"
|
||||
"queryorchestration/internal/serviceconfig/queue"
|
||||
"queryorchestration/internal/serviceconfig/queue/documentclean"
|
||||
"queryorchestration/internal/test"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type DocSyncConfig struct {
|
||||
serviceconfig.BaseConfig
|
||||
documentclean.DocCleanConfig
|
||||
queue.QueueConfig
|
||||
objectstore.ObjectStoreConfig
|
||||
}
|
||||
|
||||
func TestNewDocumentSyncService(t *testing.T) {
|
||||
svc := documentsync.New(&DocSyncConfig{}, &documentsync.Services{})
|
||||
assert.NotNil(t, svc)
|
||||
}
|
||||
|
||||
// TestDocumentSyncSync_ReturnsSyncResult verifies that Sync() returns a
|
||||
// *SyncResult with correct Synced and Skipped fields based on the client's
|
||||
// can_sync configuration.
|
||||
func TestDocumentSyncSync_ReturnsSyncResult(t *testing.T) {
|
||||
cfg := &DocSyncConfig{}
|
||||
test.CreateDB(t, cfg)
|
||||
acfg := test.CreateAWSContainer(t, cfg)
|
||||
|
||||
test.SetQueueClient(t, t.Context(), cfg, acfg.ExternalEndpoint)
|
||||
cfg.DocumentCleanURL = test.CreateQueue(t, cfg, test.DocCleanRunnerName)
|
||||
|
||||
ctx := t.Context()
|
||||
|
||||
// Test 1: can_sync=true -> Synced=true, Skipped=false
|
||||
syncClientID := "sync_result_true_" + uuid.New().String()[:8]
|
||||
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
|
||||
Clientid: syncClientID,
|
||||
Name: "Test Sync True",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
err = cfg.GetDBQueries().AddClientCanSync(ctx, &repository.AddClientCanSyncParams{
|
||||
Clientid: syncClientID,
|
||||
Cansync: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
docID, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
|
||||
Clientid: syncClientID,
|
||||
Hash: "hash_sync_true_" + uuid.New().String()[:8],
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
part := uint16(1)
|
||||
filetype := "pdf"
|
||||
key := objectstore.BucketKey{
|
||||
CreatedAt: time.Now().UTC(),
|
||||
ClientID: syncClientID,
|
||||
EntityID: uuid.New(),
|
||||
Location: objectstore.Import,
|
||||
Part: &part,
|
||||
FileType: &filetype,
|
||||
}
|
||||
err = cfg.GetDBQueries().AddDocumentEntry(ctx, &repository.AddDocumentEntryParams{
|
||||
Documentid: docID,
|
||||
Bucket: "bucket",
|
||||
Key: key.String(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
svc := documentsync.New(cfg, &documentsync.Services{
|
||||
Document: document.New(cfg),
|
||||
Client: client.New(cfg),
|
||||
})
|
||||
|
||||
result, err := svc.Sync(ctx, docID)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
assert.True(t, result.Synced, "should be synced when can_sync=true")
|
||||
assert.False(t, result.Skipped, "should not be skipped when can_sync=true")
|
||||
|
||||
// Test 2: can_sync=false -> Synced=false, Skipped=true
|
||||
skipClientID := "sync_result_false_" + uuid.New().String()[:8]
|
||||
err = cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
|
||||
Clientid: skipClientID,
|
||||
Name: "Test Sync False",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
err = cfg.GetDBQueries().AddClientCanSync(ctx, &repository.AddClientCanSyncParams{
|
||||
Clientid: skipClientID,
|
||||
Cansync: false,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
skipDocID, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
|
||||
Clientid: skipClientID,
|
||||
Hash: "hash_sync_false_" + uuid.New().String()[:8],
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
key2 := objectstore.BucketKey{
|
||||
CreatedAt: time.Now().UTC(),
|
||||
ClientID: skipClientID,
|
||||
EntityID: uuid.New(),
|
||||
Location: objectstore.Import,
|
||||
Part: &part,
|
||||
FileType: &filetype,
|
||||
}
|
||||
err = cfg.GetDBQueries().AddDocumentEntry(ctx, &repository.AddDocumentEntryParams{
|
||||
Documentid: skipDocID,
|
||||
Bucket: "bucket",
|
||||
Key: key2.String(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
skipResult, err := svc.Sync(ctx, skipDocID)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, skipResult)
|
||||
assert.False(t, skipResult.Synced, "should not be synced when can_sync=false")
|
||||
assert.True(t, skipResult.Skipped, "should be skipped when can_sync=false")
|
||||
}
|
||||
|
||||
@@ -10,20 +10,40 @@ import (
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func (s *Service) Sync(ctx context.Context, id uuid.UUID) error {
|
||||
// SyncResult contains the result of the sync check.
|
||||
//
|
||||
// Fields:
|
||||
// - Synced: true if the document was forwarded to the clean queue
|
||||
// - Skipped: true if the client's can_sync flag is false
|
||||
type SyncResult struct {
|
||||
Synced bool
|
||||
Skipped bool
|
||||
}
|
||||
|
||||
// Sync checks if a document is eligible for syncing based on the client's
|
||||
// can_sync configuration, and forwards eligible documents to the clean queue.
|
||||
//
|
||||
// Parameters:
|
||||
// - ctx: request context
|
||||
// - id: the document UUID to sync
|
||||
//
|
||||
// Returns:
|
||||
// - *SyncResult: result indicating whether sync was performed or skipped
|
||||
// - error: if any infrastructure operation (DB, queue) fails
|
||||
func (s *Service) Sync(ctx context.Context, id uuid.UUID) (*SyncResult, error) {
|
||||
doc, err := s.svc.Document.GetSummary(ctx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
j, err := s.svc.Client.Get(ctx, doc.ClientID)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !j.CanSync {
|
||||
slog.Debug("not syncing document", "id", id.String())
|
||||
return nil
|
||||
return &SyncResult{Skipped: true}, nil
|
||||
}
|
||||
|
||||
slog.Debug("syncing document", "id", id.String())
|
||||
@@ -35,8 +55,8 @@ func (s *Service) Sync(ctx context.Context, id uuid.UUID) error {
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return nil
|
||||
return &SyncResult{Synced: true}, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user