Merged in feature/batch-status (pull request #215)

Implement and test the batch status feature

* working
This commit is contained in:
Jay Brown
2026-03-12 18:53:42 +00:00
parent 8a4029058b
commit aafe7d5b5f
39 changed files with 4252 additions and 516 deletions
@@ -0,0 +1,87 @@
// Package outcome provides a shared reporter that runners use to write
// pipeline stage outcomes to the batch_document_outcomes table.
// For non-batch documents (no outcome row exists), updates are a natural
// no-op (0 rows affected).
package outcome
import (
"context"
"queryorchestration/internal/database/repository"
"github.com/google/uuid"
)
// Reporter writes pipeline stage outcomes to batch_document_outcomes.
// Used by runners to report their processing results for batch-uploaded documents.
// For non-batch documents (no outcome row exists), updates are a no-op (0 rows affected).
//
// Fields:
// - queries: SQLC-generated database queries
type Reporter struct {
queries *repository.Queries
}
// New creates an outcome Reporter.
//
// Parameters:
// - q: SQLC-generated queries instance (from cfg.GetDBQueries())
//
// Returns:
// - *Reporter: the reporter instance
func New(q *repository.Queries) *Reporter {
return &Reporter{queries: q}
}
// Resolve sets the document_id on a batch outcome row and updates the outcome.
// Called by docInitRunner after creating or finding the document.
// Matches on batch_id + filename since document_id is NULL before init.
//
// Parameters:
// - ctx: request context
// - batchID: the batch this file belongs to (from the document's batch_id)
// - filename: original filename from the ZIP (from the document record)
// - documentID: the created or found document ID
// - outcomeStatus: the init-stage outcome ("init_complete" or "init_duplicate")
// - errorDetail: optional error message (nil for success)
//
// Returns:
// - error: database error, or nil
//
// If batchID is nil, this is a no-op (document was not from a batch upload).
func (r *Reporter) Resolve(ctx context.Context, batchID *uuid.UUID, filename string,
documentID uuid.UUID, outcomeStatus repository.BatchOutcomeStatus, errorDetail *string) error {
if batchID == nil {
return nil
}
return r.queries.ResolveBatchDocumentOutcome(ctx, &repository.ResolveBatchDocumentOutcomeParams{
BatchID: *batchID,
Filename: filename,
DocumentID: &documentID,
Column4: outcomeStatus,
ErrorDetail: errorDetail,
})
}
// Update updates the outcome for a document that already has document_id set.
// Called by docSyncRunner and docCleanRunner after processing.
//
// Parameters:
// - ctx: request context
// - documentID: the document being processed
// - outcomeStatus: the pipeline outcome (e.g. "sync_complete", "clean_passed")
// - errorDetail: optional error message (nil for success)
//
// Returns:
// - error: database error, or nil
//
// If no outcome row exists for this document_id (non-batch document), this
// affects 0 rows and is a no-op.
func (r *Reporter) Update(ctx context.Context, documentID uuid.UUID,
outcomeStatus repository.BatchOutcomeStatus, errorDetail *string) error {
return r.queries.UpdateBatchDocumentOutcomeByDocumentID(ctx, &repository.UpdateBatchDocumentOutcomeByDocumentIDParams{
DocumentID: &documentID,
Column2: outcomeStatus,
ErrorDetail: errorDetail,
})
}
@@ -0,0 +1,150 @@
// Package outcome_test contains integration tests for the batch outcome reporter.
// Tests use real database via testcontainers (no mocks).
package outcome_test
import (
"testing"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/document/batch/outcome"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/test"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type TestConfig struct {
serviceconfig.BaseConfig
}
// createBatchWithOutcome is a helper that creates a client, batch, and inserts
// a submitted outcome row. Returns the batchID and the unique clientID used.
func createBatchWithOutcome(t *testing.T, cfg *TestConfig, clientPrefix, filename string) (uuid.UUID, string) {
t.Helper()
ctx := t.Context()
// Use a unique clientID to avoid collisions when tests share the DB
clientID := clientPrefix + "_" + uuid.New().String()[:8]
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Client " + clientID,
})
require.NoError(t, err)
batchID, err := cfg.GetDBQueries().CreateBatchUpload(ctx, &repository.CreateBatchUploadParams{
ClientID: clientID,
OriginalFilename: "test.zip",
TotalDocuments: 5,
})
require.NoError(t, err)
err = cfg.GetDBQueries().InsertBatchDocumentOutcome(ctx, &repository.InsertBatchDocumentOutcomeParams{
BatchID: batchID,
Filename: filename,
Column3: repository.BatchOutcomeStatusSubmitted,
})
require.NoError(t, err)
return batchID, clientID
}
// TestResolve_SetsDocumentIDAndOutcome inserts a submitted row, resolves it
// with a document_id, and verifies the update took effect.
func TestResolve_SetsDocumentIDAndOutcome(t *testing.T) {
cfg := &TestConfig{}
test.CreateDB(t, cfg)
ctx := t.Context()
filename := "invoice.pdf"
batchID, clientID := createBatchWithOutcome(t, cfg, "test_resolve_sets", filename)
// Create a real document so FK constraint is satisfied
docID, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientID,
Hash: "hash_resolve_test",
BatchID: &batchID,
})
require.NoError(t, err)
reporter := outcome.New(cfg.GetDBQueries())
err = reporter.Resolve(ctx, &batchID, filename, docID,
repository.BatchOutcomeStatusInitComplete, nil)
require.NoError(t, err)
// Verify the row was updated
rows, err := cfg.GetDBQueries().ListBatchDocumentOutcomes(ctx, batchID)
require.NoError(t, err)
require.Len(t, rows, 1)
assert.Equal(t, repository.BatchOutcomeStatusInitComplete, rows[0].Outcome)
require.NotNil(t, rows[0].DocumentID)
assert.Equal(t, docID, *rows[0].DocumentID)
}
// TestResolve_NoOpForNilBatchID verifies that calling Resolve with a nil batchID
// returns no error and makes no DB changes.
func TestResolve_NoOpForNilBatchID(t *testing.T) {
cfg := &TestConfig{}
test.CreateDB(t, cfg)
ctx := t.Context()
reporter := outcome.New(cfg.GetDBQueries())
// Call with nil batchID -- should be a no-op
err := reporter.Resolve(ctx, nil, "anything.pdf", uuid.New(),
repository.BatchOutcomeStatusInitComplete, nil)
require.NoError(t, err)
}
// TestUpdate_UpdatesOutcomeByDocumentID inserts a resolved row (with document_id),
// then updates it to sync_complete and verifies.
func TestUpdate_UpdatesOutcomeByDocumentID(t *testing.T) {
cfg := &TestConfig{}
test.CreateDB(t, cfg)
ctx := t.Context()
filename := "receipt.pdf"
batchID, clientID := createBatchWithOutcome(t, cfg, "test_update_docid", filename)
// Create a real document so FK constraint is satisfied
docID, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientID,
Hash: "hash_update_test",
BatchID: &batchID,
})
require.NoError(t, err)
reporter := outcome.New(cfg.GetDBQueries())
// Resolve first (set document_id and advance to init_complete)
err = reporter.Resolve(ctx, &batchID, filename, docID,
repository.BatchOutcomeStatusInitComplete, nil)
require.NoError(t, err)
// Now update to sync_complete
err = reporter.Update(ctx, docID, repository.BatchOutcomeStatusSyncComplete, nil)
require.NoError(t, err)
// Verify the outcome was advanced
rows, err := cfg.GetDBQueries().ListBatchDocumentOutcomes(ctx, batchID)
require.NoError(t, err)
require.Len(t, rows, 1)
assert.Equal(t, repository.BatchOutcomeStatusSyncComplete, rows[0].Outcome)
}
// TestUpdate_NoOpForNonBatchDocument verifies that calling Update for a
// document_id that has no outcome row produces no error (0 rows affected).
func TestUpdate_NoOpForNonBatchDocument(t *testing.T) {
cfg := &TestConfig{}
test.CreateDB(t, cfg)
ctx := t.Context()
reporter := outcome.New(cfg.GetDBQueries())
// Call Update with a random document_id that has no outcome row
err := reporter.Update(ctx, uuid.New(), repository.BatchOutcomeStatusSyncComplete, nil)
require.NoError(t, err)
}
+143 -8
View File
@@ -2,7 +2,9 @@ package batch
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"time"
@@ -36,13 +38,40 @@ type BatchUploadSummary struct {
CompletedAt *time.Time `json:"completed_at,omitempty"`
}
// BatchUploadDetails includes full details with failed filenames
// BatchDocumentOutcome represents the per-file result from batch extraction
// and pipeline processing. This is the single source of truth for a file's
// status within a batch.
//
// Fields:
// - ID: unique outcome record ID
// - BatchID: the batch this file belongs to
// - Filename: original filename from the ZIP
// - Outcome: current status (extraction outcome or pipeline stage)
// - ErrorDetail: human-readable error/failure message, nil for success states
// - DocumentID: resolved document ID (nil if document not yet created)
// - CleanFail: specific clean failure reason from documentCleans table (nil if N/A)
// - CreatedAt: when the outcome row was created (extraction time)
// - UpdatedAt: when the outcome was last changed (pipeline progression)
type BatchDocumentOutcome struct {
ID uuid.UUID
BatchID uuid.UUID
Filename string
Outcome string
ErrorDetail *string
DocumentID *uuid.UUID
CleanFail *string
CreatedAt time.Time
UpdatedAt time.Time
}
// BatchUploadDetails includes full details with failed filenames and document outcomes
type BatchUploadDetails struct {
BatchUploadSummary
FailedFilenames []string `json:"failed_filenames"`
ArchiveBucket string `json:"archive_bucket,omitempty"`
ArchiveKey string `json:"archive_key,omitempty"`
FileSizeBytes int64 `json:"file_size_bytes,omitempty"`
FailedFilenames []string `json:"failed_filenames"`
DocumentOutcomes []*BatchDocumentOutcome `json:"document_outcomes,omitempty"`
ArchiveBucket string `json:"archive_bucket,omitempty"`
ArchiveKey string `json:"archive_key,omitempty"`
FileSizeBytes int64 `json:"file_size_bytes,omitempty"`
}
// Service handles batch upload operations
@@ -84,9 +113,18 @@ func (s *Service) CreateWithStorage(ctx context.Context, clientID string, filena
return result.ID, nil
}
// Get retrieves batch upload details
// Get retrieves batch upload details including per-file document outcomes.
//
// Parameters:
// - ctx: request context
// - clientID: the client identifier
// - batchID: the batch UUID
//
// Returns:
// - *BatchUploadDetails: full batch details with document outcomes
// - error: database error, or nil
func (s *Service) Get(ctx context.Context, clientID string, batchID uuid.UUID) (*BatchUploadDetails, error) {
batch, err := s.cfg.GetDBQueries().GetBatchUploadWithStorage(ctx, &repository.GetBatchUploadWithStorageParams{
batchRow, err := s.cfg.GetDBQueries().GetBatchUploadWithStorage(ctx, &repository.GetBatchUploadWithStorageParams{
ID: batchID,
ClientID: clientID,
})
@@ -94,7 +132,16 @@ func (s *Service) Get(ctx context.Context, clientID string, batchID uuid.UUID) (
return nil, err
}
return s.convertToDetailsWithStorage(batch), nil
details := s.convertToDetailsWithStorage(batchRow)
// Populate document outcomes
outcomes, err := s.ListOutcomes(ctx, batchID)
if err != nil {
return nil, fmt.Errorf("failed to list batch outcomes: %w", err)
}
details.DocumentOutcomes = outcomes
return details, nil
}
// List retrieves all batch uploads for a client with pagination
@@ -272,6 +319,94 @@ func (s *Service) convertToDetailsWithStorage(batch *repository.GetBatchUploadWi
return details
}
// RecordOutcome inserts or upserts a per-file outcome row for a batch document.
// Called by the batch worker during ZIP extraction to record the initial outcome
// for each file.
//
// Parameters:
// - ctx: request context
// - batchID: the batch this file belongs to
// - filename: original filename from the ZIP
// - outcome: the typed outcome status (e.g. repository.BatchOutcomeStatusSubmitted)
// - errorDetail: optional error message for failure outcomes
// - documentID: optional document ID (set for duplicates)
//
// Returns:
// - error: database error, or nil
func (s *Service) RecordOutcome(ctx context.Context, batchID uuid.UUID, filename string,
outcome repository.BatchOutcomeStatus, errorDetail *string, documentID *uuid.UUID) error {
return s.cfg.GetDBQueries().InsertBatchDocumentOutcome(ctx, &repository.InsertBatchDocumentOutcomeParams{
BatchID: batchID,
Filename: filename,
Column3: outcome,
ErrorDetail: errorDetail,
DocumentID: documentID,
})
}
// ListOutcomes retrieves all per-file outcome rows for a batch, with clean
// failure detail joined from the currentCleanEntries view.
//
// Parameters:
// - ctx: request context
// - batchID: the batch UUID
//
// Returns:
// - []*BatchDocumentOutcome: list of outcome rows ordered by creation time
// - error: database error, or nil
func (s *Service) ListOutcomes(ctx context.Context, batchID uuid.UUID) ([]*BatchDocumentOutcome, error) {
rows, err := s.cfg.GetDBQueries().ListBatchDocumentOutcomes(ctx, batchID)
if err != nil {
return nil, err
}
outcomes := make([]*BatchDocumentOutcome, len(rows))
for i, row := range rows {
o := &BatchDocumentOutcome{
ID: row.ID,
BatchID: row.BatchID,
Filename: row.Filename,
Outcome: string(row.Outcome),
ErrorDetail: row.ErrorDetail,
DocumentID: row.DocumentID,
CreatedAt: s.pgTimestampToTime(row.CreatedAt),
UpdatedAt: s.pgTimestampToTime(row.UpdatedAt),
}
if row.CleanFail.Valid {
failStr := string(row.CleanFail.Cleanfailtype)
o.CleanFail = &failStr
}
outcomes[i] = o
}
return outcomes, nil
}
// CheckDuplicate checks if a document with the given hash already exists for
// the specified client. Returns the existing document's UUID if found, nil otherwise.
//
// Parameters:
// - ctx: request context
// - clientID: the client identifier
// - hash: the SHA-256 hash of the file content
//
// Returns:
// - *uuid.UUID: existing document ID if a duplicate exists, nil otherwise
// - error: database error (other than not-found), or nil
func (s *Service) CheckDuplicate(ctx context.Context, clientID string, hash string) (*uuid.UUID, error) {
id, err := s.cfg.GetDBQueries().GetDocumentIDByHash(ctx, &repository.GetDocumentIDByHashParams{
Clientid: clientID,
Hash: hash,
})
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
}
return nil, err
}
return &id, nil
}
// ListUnprocessed retrieves all unprocessed batch uploads across all clients
func (s *Service) ListUnprocessed(ctx context.Context) ([]*BatchUploadSummary, error) {
batches, err := s.cfg.GetDBQueries().GetUnprocessedBatches(ctx)
+192
View File
@@ -16,6 +16,7 @@ import (
"queryorchestration/internal/test"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -559,3 +560,194 @@ func TestListUnprocessed_DatabaseError(t *testing.T) {
assert.Nil(t, unprocessedBatches)
assert.Contains(t, err.Error(), "failed to query unprocessed batches")
}
// TestRecordOutcome verifies that RecordOutcome inserts a per-file outcome row
// and that it can be read back via ListOutcomes.
func TestRecordOutcome(t *testing.T) {
cfg := &TestConfig{}
test.CreateDB(t, cfg)
ctx := t.Context()
service := documentbatch.New(cfg)
clientID := "test_record_outcome_" + uuid.New().String()[:8]
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Client Record Outcome",
})
require.NoError(t, err)
batchID, err := service.Create(ctx, clientID, "outcomes.zip", 3)
require.NoError(t, err)
// Insert multiple outcomes
err = service.RecordOutcome(ctx, batchID, "good.pdf", repository.BatchOutcomeStatusSubmitted, nil, nil)
require.NoError(t, err)
errMsg := "file is not a PDF"
err = service.RecordOutcome(ctx, batchID, "notes.txt", repository.BatchOutcomeStatusInvalidType, &errMsg, nil)
require.NoError(t, err)
// Verify persistence via ListOutcomes
outcomes, err := service.ListOutcomes(ctx, batchID)
require.NoError(t, err)
require.Len(t, outcomes, 2)
// Verify the outcome details
found := map[string]string{}
for _, o := range outcomes {
found[o.Filename] = o.Outcome
}
assert.Equal(t, "submitted", found["good.pdf"])
assert.Equal(t, "invalid_type", found["notes.txt"])
}
// TestListOutcomes inserts outcomes for various states and verifies the listing.
func TestListOutcomes(t *testing.T) {
cfg := &TestConfig{}
test.CreateDB(t, cfg)
ctx := t.Context()
service := documentbatch.New(cfg)
clientID := "test_list_outcomes_" + uuid.New().String()[:8]
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Client List Outcomes",
})
require.NoError(t, err)
batchID, err := service.Create(ctx, clientID, "list.zip", 5)
require.NoError(t, err)
// Insert a variety of outcomes
err = service.RecordOutcome(ctx, batchID, "a.pdf", repository.BatchOutcomeStatusSubmitted, nil, nil)
require.NoError(t, err)
err = service.RecordOutcome(ctx, batchID, "b.pdf", repository.BatchOutcomeStatusSubmitted, nil, nil)
require.NoError(t, err)
errOpen := "cannot open"
err = service.RecordOutcome(ctx, batchID, "c.pdf", repository.BatchOutcomeStatusFailedOpen, &errOpen, nil)
require.NoError(t, err)
err = service.RecordOutcome(ctx, batchID, "d.txt", repository.BatchOutcomeStatusInvalidType, nil, nil)
require.NoError(t, err)
dupID := uuid.New()
// Create a real document for the duplicate reference
realDocID, createErr := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientID,
Hash: "hash_dup",
BatchID: &batchID,
})
require.NoError(t, createErr)
_ = dupID // not needed since we use the real ID
err = service.RecordOutcome(ctx, batchID, "e.pdf", repository.BatchOutcomeStatusDuplicate, nil, &realDocID)
require.NoError(t, err)
outcomes, err := service.ListOutcomes(ctx, batchID)
require.NoError(t, err)
require.Len(t, outcomes, 5)
// Verify ordering is by created_at ASC
expectedFilenames := []string{"a.pdf", "b.pdf", "c.pdf", "d.txt", "e.pdf"}
for i, o := range outcomes {
assert.Equal(t, expectedFilenames[i], o.Filename)
}
// Verify the duplicate has document_id set
assert.NotNil(t, outcomes[4].DocumentID)
assert.Equal(t, realDocID, *outcomes[4].DocumentID)
}
// TestListOutcomes_WithCleanFail verifies that the clean_fail column from
// the currentCleanEntries view is joined when outcome is clean_failed.
func TestListOutcomes_WithCleanFail(t *testing.T) {
cfg := &TestConfig{}
test.CreateDB(t, cfg)
ctx := t.Context()
service := documentbatch.New(cfg)
clientID := "test_clean_fail_" + uuid.New().String()[:8]
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Client Clean Fail",
})
require.NoError(t, err)
batchID, err := service.Create(ctx, clientID, "cleanfail.zip", 1)
require.NoError(t, err)
// Create a real document
docID, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientID,
Hash: "hash_clean_fail",
BatchID: &batchID,
})
require.NoError(t, err)
// Insert a clean_failed outcome
err = service.RecordOutcome(ctx, batchID, "corrupt.pdf", repository.BatchOutcomeStatusCleanFailed, nil, &docID)
require.NoError(t, err)
// Create a documentClean with fail type and a documentCleanEntry to populate
// the currentCleanEntries view
cleanID, err := cfg.GetDBQueries().AddDocumentClean(ctx, &repository.AddDocumentCleanParams{
Documentid: docID,
Fail: repository.NullCleanfailtype{
Cleanfailtype: repository.CleanfailtypeInvalidMimetype,
Valid: true,
},
})
require.NoError(t, err)
err = cfg.GetDBQueries().AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Cleanid: cleanID,
Version: 1,
})
require.NoError(t, err)
outcomes, err := service.ListOutcomes(ctx, batchID)
require.NoError(t, err)
require.Len(t, outcomes, 1)
assert.Equal(t, "clean_failed", outcomes[0].Outcome)
require.NotNil(t, outcomes[0].CleanFail)
assert.Equal(t, "invalid_mimetype", *outcomes[0].CleanFail)
}
// TestCheckDuplicate creates a document, then verifies that CheckDuplicate
// returns its ID when the hash matches.
func TestCheckDuplicate(t *testing.T) {
cfg := &TestConfig{}
test.CreateDB(t, cfg)
ctx := t.Context()
service := documentbatch.New(cfg)
clientID := "test_check_dup_" + uuid.New().String()[:8]
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Client Check Dup",
})
require.NoError(t, err)
// Create a document with a known hash
knownHash := "sha256_known_hash_abcdef"
docID, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientID,
Hash: knownHash,
})
require.NoError(t, err)
// Check duplicate with the same hash -- should find it
existingID, err := service.CheckDuplicate(ctx, clientID, knownHash)
require.NoError(t, err)
require.NotNil(t, existingID)
assert.Equal(t, docID, *existingID)
// Check duplicate with a different hash -- should return nil
noMatch, err := service.CheckDuplicate(ctx, clientID, "totally_different_hash")
require.NoError(t, err)
assert.Nil(t, noMatch)
}
+37 -13
View File
@@ -4,7 +4,6 @@ import (
"context"
"database/sql"
"errors"
"fmt"
"log/slog"
"queryorchestration/internal/database/repository"
@@ -82,23 +81,34 @@ func (s *Service) executeCleanTasks(ctx context.Context, params *CleanParams) (*
}, nil
}
func (s *Service) clean(ctx context.Context, id uuid.UUID) error {
// clean runs the document validation and stores the result. Returns a CleanResult
// indicating whether the document passed validation. Infrastructure errors (DB, S3)
// are returned as the error value; validation failures are returned via CleanResult.
//
// Parameters:
// - ctx: request context
// - id: the document UUID to clean
//
// Returns:
// - *CleanResult: result with Passed=true on success, or Passed=false with FailReason
// - error: only for infrastructure errors
func (s *Service) clean(ctx context.Context, id uuid.UUID) (*CleanResult, error) {
slog.Debug("cleaning document", "id", id.String())
docId := id
doc, err := s.cfg.GetDBQueries().GetDocumentSummary(ctx, docId)
if err != nil {
return err
return nil, err
}
entry, err := s.cfg.GetDBQueries().GetDocumentEntry(ctx, docId)
if err != nil {
return err
return nil, err
}
key, err := objectstore.ParseBucketKey(entry.Key)
if err != nil {
return err
return nil, err
}
out, err := s.executeCleanTasks(ctx, &CleanParams{
@@ -108,18 +118,31 @@ func (s *Service) clean(ctx context.Context, id uuid.UUID) error {
Key: key,
})
if err != nil {
return err
return nil, err
}
err = s.storeClean(ctx, id, out)
result, err := s.storeClean(ctx, id, out)
if err != nil {
return err
return nil, err
}
return nil
return result, nil
}
func (s *Service) storeClean(ctx context.Context, id uuid.UUID, out *ExecuteCleanResponse) error {
// storeClean persists the clean result to the database. Returns a CleanResult
// indicating whether the document passed validation. Validation failures are
// returned via CleanResult (not as errors) so callers can distinguish them
// from infrastructure errors.
//
// Parameters:
// - ctx: request context
// - id: the document UUID
// - out: the result from executeCleanTasks
//
// Returns:
// - *CleanResult: Passed=true if validation succeeded, Passed=false with FailReason otherwise
// - error: only for infrastructure errors (DB transaction failures)
func (s *Service) storeClean(ctx context.Context, id uuid.UUID, out *ExecuteCleanResponse) (*CleanResult, error) {
docId := id
version := build.GetVersionUnixTimestamp()
@@ -164,14 +187,15 @@ func (s *Service) storeClean(ctx context.Context, id uuid.UUID, out *ExecuteClea
return nil
})
if err != nil {
return err
return nil, err
}
if out.failReason != nil {
return fmt.Errorf("%s", *out.failReason)
failStr := string(*out.failReason)
return &CleanResult{Passed: false, FailReason: &failStr}, nil
}
return nil
return &CleanResult{Passed: true}, nil
}
func (s *Service) isNewClean(lastEntry *repository.GetMostRecentDocumentCleanEntryRow, out *ExecuteCleanResponse) bool {
+25 -5
View File
@@ -10,21 +10,41 @@ import (
"github.com/google/uuid"
)
// CleanResult contains the result of the clean operation.
//
// Fields:
// - Passed: true if the document passed all validation checks
// - FailReason: the specific failure reason if Passed is false, nil otherwise
type CleanResult struct {
Passed bool
FailReason *string
}
// Clean processes a document by cleaning it. Previously this also triggered
// text extraction, but that feature has been removed.
func (s *Service) Clean(ctx context.Context, documentId uuid.UUID) error {
//
// Parameters:
// - ctx: request context
// - documentId: the document UUID to clean
//
// Returns:
// - *CleanResult: result indicating whether validation passed and any failure reason
// - error: only for infrastructure errors (DB, S3). Validation failures are
// returned via CleanResult.Passed=false, not as errors.
func (s *Service) Clean(ctx context.Context, documentId uuid.UUID) (*CleanResult, error) {
isclean, err := s.cfg.GetDBQueries().HasDocumentCleanEntry(ctx, documentId)
if err != nil {
return fmt.Errorf("unable to verify if document has been cleaned: %w", err)
return nil, fmt.Errorf("unable to verify if document has been cleaned: %w", err)
}
if !isclean {
err = s.clean(ctx, documentId)
result, err := s.clean(ctx, documentId)
if err != nil {
return fmt.Errorf("unable to clean document: %w", err)
return nil, fmt.Errorf("unable to clean document: %w", err)
}
return result, nil
}
// Note: Text extraction has been removed. Document pipeline ends here.
return nil
return &CleanResult{Passed: true}, nil
}
+110 -1
View File
@@ -3,18 +3,26 @@
package documentclean_test
import (
"bytes"
"testing"
"time"
"queryorchestration/internal/database/repository"
documentclean "queryorchestration/internal/document/clean"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/test"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type DocCleanConfig struct {
serviceconfig.BaseConfig
objectstore.ConfigProvider
objectstore.ObjectStoreConfig
}
func TestService(t *testing.T) {
@@ -22,3 +30,104 @@ func TestService(t *testing.T) {
svc := documentclean.New(cfg)
assert.NotNil(t, svc)
}
// TestDocumentCleanClean_ReturnsCleanResult verifies that Clean() returns a
// *CleanResult with correct Passed and FailReason fields.
//
// Two scenarios are tested:
// 1. Already-cleaned document (HasDocumentCleanEntry=true) returns Passed=true
// 2. Non-PDF file returns Passed=false with a non-nil FailReason
func TestDocumentCleanClean_ReturnsCleanResult(t *testing.T) {
cfg := &DocCleanConfig{}
test.CreateDB(t, cfg)
test.CreateAWSResources(t, cfg)
ctx := t.Context()
svc := documentclean.New(cfg)
// Test 1: Already cleaned document returns Passed=true (shortcut path)
clientID := "clean_result_" + uuid.New().String()[:8]
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Clean Result",
})
require.NoError(t, err)
docID, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientID,
Hash: "hash_clean_result_" + uuid.New().String()[:8],
})
require.NoError(t, err)
// Create a clean entry so HasDocumentCleanEntry returns true
bucket := "bucket"
key := "key"
hash := "hash"
cleanID, err := cfg.GetDBQueries().AddDocumentClean(ctx, &repository.AddDocumentCleanParams{
Documentid: docID,
Bucket: &bucket,
Key: &key,
Hash: &hash,
Mimetype: repository.NullCleanmimetype{
Cleanmimetype: repository.CleanmimetypeApplicationPdf,
Valid: true,
},
})
require.NoError(t, err)
err = cfg.GetDBQueries().AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Cleanid: cleanID,
Version: 1,
})
require.NoError(t, err)
result, err := svc.Clean(ctx, docID)
require.NoError(t, err)
require.NotNil(t, result)
assert.True(t, result.Passed, "already-cleaned document should return Passed=true")
assert.Nil(t, result.FailReason, "Passed document should have nil FailReason")
// Test 2: Non-PDF file returns Passed=false with FailReason
// Upload a non-PDF file to S3 first to get its ETag
fileContent := []byte("This is NOT a PDF file. It has no PDF signature.")
location := objectstore.BucketKey{
Location: objectstore.Import,
ClientID: clientID,
CreatedAt: time.Now().UTC(),
EntityID: uuid.New(),
}
_, err = cfg.GetStoreClient().PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(cfg.GetBucket()),
Key: aws.String(location.String()),
Body: bytes.NewReader(fileContent),
ContentType: aws.String("text/plain"),
})
require.NoError(t, err)
// Get ETag to use as the document hash (IfMatch requires it)
headResp, err := cfg.GetStoreClient().HeadObject(ctx, &s3.HeadObjectInput{
Bucket: aws.String(cfg.GetBucket()),
Key: aws.String(location.String()),
})
require.NoError(t, err)
etag := *headResp.ETag
// Create document with hash matching the ETag
docID2, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientID,
Hash: etag,
})
require.NoError(t, err)
err = cfg.GetDBQueries().AddDocumentEntry(ctx, &repository.AddDocumentEntryParams{
Documentid: docID2,
Bucket: cfg.GetBucket(),
Key: location.String(),
})
require.NoError(t, err)
failResult, err := svc.Clean(ctx, docID2)
require.NoError(t, err)
require.NotNil(t, failResult)
assert.False(t, failResult.Passed, "non-PDF document should fail validation")
assert.NotNil(t, failResult.FailReason, "failed document should have a FailReason")
}
+34 -5
View File
@@ -26,15 +26,39 @@ type Create struct {
FolderID *uuid.UUID // Optional folder ID if folder hierarchy was pre-created
}
func (s *Service) Create(ctx context.Context, doc *Create) (uuid.UUID, error) {
// CreateResult contains the result of document creation.
//
// Fields:
// - ID: the document's UUID (new or existing)
// - IsDuplicate: true if a document with the same hash already existed
// - BatchID: the batch this document belongs to (nil for non-batch uploads)
// - Filename: the original filename (nil for non-batch uploads)
type CreateResult struct {
ID uuid.UUID
IsDuplicate bool
BatchID *uuid.UUID
Filename *string
}
// Create processes a new document upload by checking for duplicates, creating
// database records, and forwarding new documents to the sync queue.
//
// Parameters:
// - ctx: request context
// - doc: document creation parameters (bucket, key, hash, filename, etc.)
//
// Returns:
// - *CreateResult: result containing document ID, duplicate status, and batch info
// - error: if any infrastructure operation (DB, S3, queue) fails
func (s *Service) Create(ctx context.Context, doc *Create) (*CreateResult, error) {
params, err := s.getCreateParams(ctx, doc)
if err != nil {
return uuid.Nil, err
return nil, err
}
id, err := s.submitCreate(ctx, params)
if err != nil {
return uuid.Nil, err
return nil, err
}
if params.ID == nil {
@@ -45,11 +69,16 @@ func (s *Service) Create(ctx context.Context, doc *Create) (uuid.UUID, error) {
},
})
if err != nil {
return uuid.Nil, err
return nil, err
}
}
return id, nil
return &CreateResult{
ID: id,
IsDuplicate: params.ID != nil,
BatchID: doc.Key.BatchID,
Filename: doc.Filename,
}, nil
}
type createDocumentParams struct {
+130
View File
@@ -6,7 +6,10 @@ import (
"bytes"
"fmt"
"testing"
"time"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/test"
"github.com/aws/aws-sdk-go-v2/aws"
@@ -105,3 +108,130 @@ func TestMeasureFileSize_Integration_NonExistentKey(t *testing.T) {
assert.Error(t, err, "measureFileSize should return an error for non-existent key")
assert.Contains(t, err.Error(), "HeadObject failed", "error should indicate HeadObject failure")
}
// TestDocumentInitCreate_ReturnsCreateResult verifies that Create() returns a
// *CreateResult with correct IsDuplicate, BatchID, and Filename fields for both
// new and duplicate documents.
func TestDocumentInitCreate_ReturnsCreateResult(t *testing.T) {
ctx := t.Context()
// Set up DB, S3, and SQS
cfg := &DocInitConfig{}
test.CreateDB(t, cfg)
acfg := test.CreateAWSContainer(t, cfg)
test.SetQueueClient(t, ctx, cfg, acfg.ExternalEndpoint)
cfg.DocumentSyncURL = test.CreateQueue(t, cfg, test.DocSyncRunnerName)
test.SetStoreClient(t, ctx, cfg, acfg.ExternalEndpoint)
test.CreateBucket(t, cfg)
svc := New(cfg)
clientID := "create_result_" + uuid.New().String()[:8]
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: "Test CreateResult",
})
require.NoError(t, err)
// Create a batch
batchID, err := cfg.GetDBQueries().CreateBatchUpload(ctx, &repository.CreateBatchUploadParams{
ClientID: clientID,
OriginalFilename: "test.zip",
TotalDocuments: 2,
})
require.NoError(t, err)
// Upload a file to S3
location := objectstore.BucketKey{
Location: objectstore.Import,
ClientID: clientID,
CreatedAt: time.Now().UTC(),
EntityID: uuid.New(),
BatchID: &batchID,
}
testContent := []byte("test content for create result")
_, err = cfg.GetStoreClient().PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(cfg.GetBucket()),
Key: aws.String(location.String()),
Body: bytes.NewReader(testContent),
})
require.NoError(t, err)
filename := "invoice.pdf"
uniqueHash := "create_result_hash_" + uuid.New().String()[:8]
// Test 1: New document returns IsDuplicate=false with correct fields
result, err := svc.Create(ctx, &Create{
Bucket: cfg.GetBucket(),
Key: location,
Hash: uniqueHash,
Filename: &filename,
})
require.NoError(t, err)
require.NotNil(t, result)
assert.NotEqual(t, uuid.Nil, result.ID, "new document should have a valid ID")
assert.False(t, result.IsDuplicate, "new document should not be a duplicate")
assert.NotNil(t, result.BatchID, "batch document should have BatchID set")
assert.Equal(t, batchID, *result.BatchID, "BatchID should match the batch")
assert.NotNil(t, result.Filename, "Filename should be set")
assert.Equal(t, filename, *result.Filename, "Filename should match")
// Test 2: Duplicate document (same hash) returns IsDuplicate=true
location2 := objectstore.BucketKey{
Location: objectstore.Import,
ClientID: clientID,
CreatedAt: time.Now().UTC(),
EntityID: uuid.New(),
BatchID: &batchID,
}
_, err = cfg.GetStoreClient().PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(cfg.GetBucket()),
Key: aws.String(location2.String()),
Body: bytes.NewReader(testContent),
})
require.NoError(t, err)
dupResult, err := svc.Create(ctx, &Create{
Bucket: cfg.GetBucket(),
Key: location2,
Hash: uniqueHash, // Same hash triggers duplicate
Filename: &filename,
})
require.NoError(t, err)
require.NotNil(t, dupResult)
assert.Equal(t, result.ID, dupResult.ID, "duplicate should reference the original document ID")
assert.True(t, dupResult.IsDuplicate, "second document with same hash should be duplicate")
assert.NotNil(t, dupResult.BatchID, "duplicate should still have BatchID")
assert.Equal(t, batchID, *dupResult.BatchID, "duplicate BatchID should match")
// Test 3: Non-batch document has nil BatchID
location3 := objectstore.BucketKey{
Location: objectstore.Import,
ClientID: clientID,
CreatedAt: time.Now().UTC(),
EntityID: uuid.New(),
// No BatchID
}
_, err = cfg.GetStoreClient().PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(cfg.GetBucket()),
Key: aws.String(location3.String()),
Body: bytes.NewReader(testContent),
})
require.NoError(t, err)
nonBatchResult, err := svc.Create(ctx, &Create{
Bucket: cfg.GetBucket(),
Key: location3,
Hash: "nonbatch_hash_" + uuid.New().String()[:8],
})
require.NoError(t, err)
require.NotNil(t, nonBatchResult)
assert.False(t, nonBatchResult.IsDuplicate, "non-batch new document should not be duplicate")
assert.Nil(t, nonBatchResult.BatchID, "non-batch document should have nil BatchID")
assert.Nil(t, nonBatchResult.Filename, "non-batch document should have nil Filename")
}
+112
View File
@@ -2,20 +2,132 @@ package documentsync_test
import (
"testing"
"time"
"queryorchestration/internal/client"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/document"
documentsync "queryorchestration/internal/document/sync"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/serviceconfig/queue"
"queryorchestration/internal/serviceconfig/queue/documentclean"
"queryorchestration/internal/test"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type DocSyncConfig struct {
serviceconfig.BaseConfig
documentclean.DocCleanConfig
queue.QueueConfig
objectstore.ObjectStoreConfig
}
func TestNewDocumentSyncService(t *testing.T) {
svc := documentsync.New(&DocSyncConfig{}, &documentsync.Services{})
assert.NotNil(t, svc)
}
// TestDocumentSyncSync_ReturnsSyncResult verifies that Sync() returns a
// *SyncResult with correct Synced and Skipped fields based on the client's
// can_sync configuration.
func TestDocumentSyncSync_ReturnsSyncResult(t *testing.T) {
cfg := &DocSyncConfig{}
test.CreateDB(t, cfg)
acfg := test.CreateAWSContainer(t, cfg)
test.SetQueueClient(t, t.Context(), cfg, acfg.ExternalEndpoint)
cfg.DocumentCleanURL = test.CreateQueue(t, cfg, test.DocCleanRunnerName)
ctx := t.Context()
// Test 1: can_sync=true -> Synced=true, Skipped=false
syncClientID := "sync_result_true_" + uuid.New().String()[:8]
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: syncClientID,
Name: "Test Sync True",
})
require.NoError(t, err)
err = cfg.GetDBQueries().AddClientCanSync(ctx, &repository.AddClientCanSyncParams{
Clientid: syncClientID,
Cansync: true,
})
require.NoError(t, err)
docID, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: syncClientID,
Hash: "hash_sync_true_" + uuid.New().String()[:8],
})
require.NoError(t, err)
part := uint16(1)
filetype := "pdf"
key := objectstore.BucketKey{
CreatedAt: time.Now().UTC(),
ClientID: syncClientID,
EntityID: uuid.New(),
Location: objectstore.Import,
Part: &part,
FileType: &filetype,
}
err = cfg.GetDBQueries().AddDocumentEntry(ctx, &repository.AddDocumentEntryParams{
Documentid: docID,
Bucket: "bucket",
Key: key.String(),
})
require.NoError(t, err)
svc := documentsync.New(cfg, &documentsync.Services{
Document: document.New(cfg),
Client: client.New(cfg),
})
result, err := svc.Sync(ctx, docID)
require.NoError(t, err)
require.NotNil(t, result)
assert.True(t, result.Synced, "should be synced when can_sync=true")
assert.False(t, result.Skipped, "should not be skipped when can_sync=true")
// Test 2: can_sync=false -> Synced=false, Skipped=true
skipClientID := "sync_result_false_" + uuid.New().String()[:8]
err = cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: skipClientID,
Name: "Test Sync False",
})
require.NoError(t, err)
err = cfg.GetDBQueries().AddClientCanSync(ctx, &repository.AddClientCanSyncParams{
Clientid: skipClientID,
Cansync: false,
})
require.NoError(t, err)
skipDocID, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: skipClientID,
Hash: "hash_sync_false_" + uuid.New().String()[:8],
})
require.NoError(t, err)
key2 := objectstore.BucketKey{
CreatedAt: time.Now().UTC(),
ClientID: skipClientID,
EntityID: uuid.New(),
Location: objectstore.Import,
Part: &part,
FileType: &filetype,
}
err = cfg.GetDBQueries().AddDocumentEntry(ctx, &repository.AddDocumentEntryParams{
Documentid: skipDocID,
Bucket: "bucket",
Key: key2.String(),
})
require.NoError(t, err)
skipResult, err := svc.Sync(ctx, skipDocID)
require.NoError(t, err)
require.NotNil(t, skipResult)
assert.False(t, skipResult.Synced, "should not be synced when can_sync=false")
assert.True(t, skipResult.Skipped, "should be skipped when can_sync=false")
}
+26 -6
View File
@@ -10,20 +10,40 @@ import (
"github.com/google/uuid"
)
func (s *Service) Sync(ctx context.Context, id uuid.UUID) error {
// SyncResult contains the result of the sync check.
//
// Fields:
// - Synced: true if the document was forwarded to the clean queue
// - Skipped: true if the client's can_sync flag is false
type SyncResult struct {
Synced bool
Skipped bool
}
// Sync checks if a document is eligible for syncing based on the client's
// can_sync configuration, and forwards eligible documents to the clean queue.
//
// Parameters:
// - ctx: request context
// - id: the document UUID to sync
//
// Returns:
// - *SyncResult: result indicating whether sync was performed or skipped
// - error: if any infrastructure operation (DB, queue) fails
func (s *Service) Sync(ctx context.Context, id uuid.UUID) (*SyncResult, error) {
doc, err := s.svc.Document.GetSummary(ctx, id)
if err != nil {
return err
return nil, err
}
j, err := s.svc.Client.Get(ctx, doc.ClientID)
if err != nil {
return err
return nil, err
}
if !j.CanSync {
slog.Debug("not syncing document", "id", id.String())
return nil
return &SyncResult{Skipped: true}, nil
}
slog.Debug("syncing document", "id", id.String())
@@ -35,8 +55,8 @@ func (s *Service) Sync(ctx context.Context, id uuid.UUID) error {
},
})
if err != nil {
return err
return nil, err
}
return nil
return &SyncResult{Synced: true}, nil
}