Merged in jmathison/v2-upload-batch (pull request #224)

Implement v2 upload batch cleanup

* Implement v2 upload batch cleanup

* Merge remote-tracking branch 'origin/main' into jmathison/v2-upload-batch

* Address upload batch review feedback

* Raise batch worker coverage

* Fix batch cleanup review issues


Approved-by: Jay Brown

api/docCleanRunner/runner.go

@@ -5,6 +5,7 @@ import (
 	"log/slog"
 
 	"queryorchestration/internal/database/repository"
+	"queryorchestration/internal/document/batch/foldercleanup"
 	"queryorchestration/internal/document/batch/outcome"
 	documentclean "queryorchestration/internal/document/clean"
 
@@ -21,6 +22,7 @@ const Name = "docCleanRunner"
 type Services struct {
 	Clean   *documentclean.Service
 	Outcome *outcome.Reporter
+	Cleanup *foldercleanup.Service
 }
 
 // Runner processes document clean messages from the SQS queue.
@@ -63,10 +65,18 @@ func (s Runner) Process(ctx context.Context, body Body) bool {
 		outcomeName = repository.BatchOutcomeStatusCleanFailed
 		errorDetail = result.FailReason
 	}
-	if updateErr := s.svc.Outcome.Update(ctx, body.DocumentID, outcomeName, errorDetail); updateErr != nil {
+	batchIDs, updateErr := s.svc.Outcome.Update(ctx, body.DocumentID, outcomeName, errorDetail)
+	if updateErr != nil {
 		slog.Error("failed to update batch outcome",
 			"document_id", body.DocumentID, "outcome", outcomeName, "error", updateErr)
 	}
+	if s.svc.Cleanup != nil {
+		for _, batchID := range batchIDs {
+			if _, err := s.svc.Cleanup.TryFinalizeBatch(ctx, batchID); err != nil {
+				slog.Error("failed to finalize batch folder cleanup", "batch_id", batchID, "error", err)
+			}
+		}
+	}
 
 	return true // validation pass or fail -> acknowledge (don't requeue bad PDFs)
 }

api/docInitRunner/runner.go

@@ -7,6 +7,7 @@ import (
 	"log/slog"
 
 	"queryorchestration/internal/database/repository"
+	"queryorchestration/internal/document/batch/foldercleanup"
 	"queryorchestration/internal/document/batch/outcome"
 	documentinit "queryorchestration/internal/document/init"
 	"queryorchestration/internal/serviceconfig/objectstore"
@@ -26,6 +27,7 @@ type Services struct {
 	Document *documentinit.Service
 	Queries  *repository.Queries
 	Outcome  *outcome.Reporter
+	Cleanup  *foldercleanup.Service
 }
 
 // Runner processes document init messages from the SQS queue.
@@ -108,6 +110,13 @@ func (s Runner) Process(ctx context.Context, body Body) bool {
 			"document_id", result.ID, "outcome", outcomeName, "error", resolveErr)
 		// Non-fatal: document was created successfully
 	}
+	if outcomeName == repository.BatchOutcomeStatusInitDuplicate {
+		if s.svc.Cleanup != nil && result.BatchID != nil {
+			if _, err := s.svc.Cleanup.TryFinalizeBatch(ctx, *result.BatchID); err != nil {
+				slog.Error("failed to finalize batch folder cleanup", "batch_id", *result.BatchID, "error", err)
+			}
+		}
+	}
 
 	return true
 }

api/docSyncRunner/runner.go

@@ -5,6 +5,7 @@ import (
 	"log/slog"
 
 	"queryorchestration/internal/database/repository"
+	"queryorchestration/internal/document/batch/foldercleanup"
 	"queryorchestration/internal/document/batch/outcome"
 	documentsync "queryorchestration/internal/document/sync"
 
@@ -21,6 +22,7 @@ const Name = "docSyncRunner"
 type Services struct {
 	Document *documentsync.Service
 	Outcome  *outcome.Reporter
+	Cleanup  *foldercleanup.Service
 }
 
 // Runner processes document sync messages from the SQS queue.
@@ -59,10 +61,18 @@ func (s Runner) Process(ctx context.Context, body Body) bool {
 	if result.Skipped {
 		outcomeName = repository.BatchOutcomeStatusSyncSkipped
 	}
-	if updateErr := s.svc.Outcome.Update(ctx, body.DocumentID, outcomeName, nil); updateErr != nil {
+	batchIDs, updateErr := s.svc.Outcome.Update(ctx, body.DocumentID, outcomeName, nil)
+	if updateErr != nil {
 		slog.Error("failed to update batch outcome",
 			"document_id", body.DocumentID, "outcome", outcomeName, "error", updateErr)
 	}
+	if outcomeName == repository.BatchOutcomeStatusSyncSkipped && s.svc.Cleanup != nil {
+		for _, batchID := range batchIDs {
+			if _, err := s.svc.Cleanup.TryFinalizeBatch(ctx, batchID); err != nil {
+				slog.Error("failed to finalize batch folder cleanup", "batch_id", batchID, "error", err)
+			}
+		}
+	}
 
 	return true
 }