Merged in feature/batch-status (pull request #215)

Implement and test the batch status feature * working
2026-03-12 18:53:42 +00:00
parent 8a4029058b
commit aafe7d5b5f
39 changed files with 4252 additions and 516 deletions
@@ -0,0 +1,212 @@
+// Package doccleanrunner_test contains integration tests for the docCleanRunner.
+// Tests use real database and S3 via testcontainers (no mocks).
+package doccleanrunner_test
+
+import (
+	"bytes"
+	"testing"
+	"time"
+
+	doccleanrunner "queryorchestration/api/docCleanRunner"
+	"queryorchestration/internal/database/repository"
+	"queryorchestration/internal/document/batch/outcome"
+	documentclean "queryorchestration/internal/document/clean"
+	"queryorchestration/internal/server/runner"
+	"queryorchestration/internal/serviceconfig/objectstore"
+	"queryorchestration/internal/serviceconfig/queue"
+	"queryorchestration/internal/test"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type DocCleanConfig struct {
+	runner.BaseConfig[doccleanrunner.Body]
+	queue.QueueConfig
+	objectstore.ObjectStoreConfig
+}
+
+// TestDocCleanRunner_ReportsCleanPassed tests that for a document that has
+// already been cleaned (HasDocumentCleanEntry returns true), the runner
+// reports clean_passed.
+func TestDocCleanRunner_ReportsCleanPassed(t *testing.T) {
+	cfg := &DocCleanConfig{}
+	test.CreateDB(t, cfg)
+	ctx := t.Context()
+
+	clientID := "clean_passed_" + uuid.New().String()[:8]
+	err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
+		Clientid: clientID,
+		Name:     "Test Clean Passed",
+	})
+	require.NoError(t, err)
+
+	// Create batch
+	batchID, err := cfg.GetDBQueries().CreateBatchUpload(ctx, &repository.CreateBatchUploadParams{
+		ClientID:         clientID,
+		OriginalFilename: "test.zip",
+		TotalDocuments:   1,
+	})
+	require.NoError(t, err)
+
+	// Create document
+	docID, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
+		Clientid: clientID,
+		Hash:     "hash_clean_pass_" + uuid.New().String()[:8],
+		BatchID:  &batchID,
+	})
+	require.NoError(t, err)
+
+	// Create a successful clean entry so HasDocumentCleanEntry returns true.
+	// This takes the shortcut path in Clean() -> returns {Passed: true}
+	bucket := "bucket"
+	key := "key"
+	hash := "hash"
+	cleanID, err := cfg.GetDBQueries().AddDocumentClean(ctx, &repository.AddDocumentCleanParams{
+		Documentid: docID,
+		Bucket:     &bucket,
+		Key:        &key,
+		Hash:       &hash,
+		Mimetype: repository.NullCleanmimetype{
+			Cleanmimetype: repository.CleanmimetypeApplicationPdf,
+			Valid:         true,
+		},
+	})
+	require.NoError(t, err)
+	err = cfg.GetDBQueries().AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
+		Cleanid: cleanID,
+		Version: 1,
+	})
+	require.NoError(t, err)
+
+	// Insert sync_complete outcome (the state before clean)
+	err = cfg.GetDBQueries().InsertBatchDocumentOutcome(ctx, &repository.InsertBatchDocumentOutcomeParams{
+		BatchID:    batchID,
+		Filename:   "invoice.pdf",
+		Column3:    repository.BatchOutcomeStatusSyncComplete,
+		DocumentID: &docID,
+	})
+	require.NoError(t, err)
+
+	cleanRunner := doccleanrunner.New(&doccleanrunner.Services{
+		Clean:   documentclean.New(cfg),
+		Outcome: outcome.New(cfg.GetDBQueries()),
+	})
+
+	assert.True(t, cleanRunner.Process(ctx, doccleanrunner.Body{DocumentID: docID}))
+
+	// Verify outcome updated to clean_passed
+	rows, err := cfg.GetDBQueries().ListBatchDocumentOutcomes(ctx, batchID)
+	require.NoError(t, err)
+	require.Len(t, rows, 1)
+	assert.Equal(t, repository.BatchOutcomeStatusCleanPassed, rows[0].Outcome)
+}
+
+// TestDocCleanRunner_ReportsCleanFailed tests that a document failing clean
+// validation gets the clean_failed outcome.
+func TestDocCleanRunner_ReportsCleanFailed(t *testing.T) {
+	cfg := &DocCleanConfig{}
+	test.CreateDB(t, cfg)
+	test.CreateAWSResources(t, cfg)
+	ctx := t.Context()
+
+	clientID := "clean_failed_" + uuid.New().String()[:8]
+	err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
+		Clientid: clientID,
+		Name:     "Test Clean Failed",
+	})
+	require.NoError(t, err)
+
+	// Create batch
+	batchID, err := cfg.GetDBQueries().CreateBatchUpload(ctx, &repository.CreateBatchUploadParams{
+		ClientID:         clientID,
+		OriginalFilename: "test.zip",
+		TotalDocuments:   1,
+	})
+	require.NoError(t, err)
+
+	// Upload a non-PDF file to S3 (plain text content that will fail mime type check)
+	fileContent := []byte("This is NOT a PDF file. It has no PDF signature.")
+	location := objectstore.BucketKey{
+		Location:  objectstore.Import,
+		ClientID:  clientID,
+		CreatedAt: time.Now().UTC(),
+		EntityID:  uuid.New(),
+	}
+
+	_, err = cfg.GetStoreClient().PutObject(ctx, &s3.PutObjectInput{
+		Bucket:      aws.String(cfg.GetBucket()),
+		Key:         aws.String(location.String()),
+		Body:        bytes.NewReader(fileContent),
+		ContentType: aws.String("text/plain"),
+	})
+	require.NoError(t, err)
+
+	// Get the ETag so we can use it as the hash for IfMatch
+	headResp, err := cfg.GetStoreClient().HeadObject(ctx, &s3.HeadObjectInput{
+		Bucket: aws.String(cfg.GetBucket()),
+		Key:    aws.String(location.String()),
+	})
+	require.NoError(t, err)
+	etag := *headResp.ETag
+
+	// Create document with hash matching the ETag
+	docID, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
+		Clientid: clientID,
+		Hash:     etag,
+		BatchID:  &batchID,
+	})
+	require.NoError(t, err)
+
+	// Add document entry pointing to the S3 file
+	err = cfg.GetDBQueries().AddDocumentEntry(ctx, &repository.AddDocumentEntryParams{
+		Documentid: docID,
+		Bucket:     cfg.GetBucket(),
+		Key:        location.String(),
+	})
+	require.NoError(t, err)
+
+	// Insert sync_complete outcome
+	err = cfg.GetDBQueries().InsertBatchDocumentOutcome(ctx, &repository.InsertBatchDocumentOutcomeParams{
+		BatchID:    batchID,
+		Filename:   "bad_file.pdf",
+		Column3:    repository.BatchOutcomeStatusSyncComplete,
+		DocumentID: &docID,
+	})
+	require.NoError(t, err)
+
+	cleanRunner := doccleanrunner.New(&doccleanrunner.Services{
+		Clean:   documentclean.New(cfg),
+		Outcome: outcome.New(cfg.GetDBQueries()),
+	})
+
+	// The runner should return true (acknowledge), even for validation failures
+	result := cleanRunner.Process(ctx, doccleanrunner.Body{DocumentID: docID})
+	assert.True(t, result, "validation failures should be acknowledged (not requeued)")
+
+	// Verify outcome updated to clean_failed
+	rows, err := cfg.GetDBQueries().ListBatchDocumentOutcomes(ctx, batchID)
+	require.NoError(t, err)
+	require.Len(t, rows, 1)
+	assert.Equal(t, repository.BatchOutcomeStatusCleanFailed, rows[0].Outcome)
+}
+
+// TestDocCleanRunner_RequeuesOnInfraError tests that an infrastructure error
+// (e.g., document not found in DB) causes the runner to return false (requeue).
+func TestDocCleanRunner_RequeuesOnInfraError(t *testing.T) {
+	cfg := &DocCleanConfig{}
+	test.CreateDB(t, cfg)
+	ctx := t.Context()
+
+	cleanRunner := doccleanrunner.New(&doccleanrunner.Services{
+		Clean:   documentclean.New(cfg),
+		Outcome: outcome.New(cfg.GetDBQueries()),
+	})
+
+	// Process a non-existent document -- should fail with DB error
+	result := cleanRunner.Process(ctx, doccleanrunner.Body{DocumentID: uuid.New()})
+	assert.False(t, result, "infrastructure error should return false for requeue")
+}