Merged in feature/batch-status (pull request #215)

Implement and test the batch status feature

* working

internal/server/api/batch_worker.go

@@ -4,12 +4,15 @@ import (
 	"archive/zip"
 	"bytes"
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
 	"io"
 	"log/slog"
 	"path"
 	"strings"
 
+	"queryorchestration/internal/database/repository"
 	"queryorchestration/internal/document/batch"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
@@ -292,6 +295,8 @@ func processZipFile(
 	if err != nil {
 		logger.Error("Failed to open file in zip", "filename", filename, "error", err)
 		_ = batchService.AddFailedFilename(ctx, batchInfo.ID, filename)
+		errMsg := err.Error()
+		_ = batchService.RecordOutcome(ctx, batchInfo.ID, filename, repository.BatchOutcomeStatusFailedOpen, &errMsg, nil)
 		return 0, 1, 0
 	}
 
@@ -300,6 +305,8 @@ func processZipFile(
 	if err != nil {
 		logger.Error("Failed to read file from zip", "filename", filename, "error", err)
 		_ = batchService.AddFailedFilename(ctx, batchInfo.ID, filename)
+		errMsg := err.Error()
+		_ = batchService.RecordOutcome(ctx, batchInfo.ID, filename, repository.BatchOutcomeStatusFailedRead, &errMsg, nil)
 		return 0, 1, 0
 	}
 
@@ -308,6 +315,8 @@ func processZipFile(
 	if err := uploadToS3(ctx, s3Client, bucket, extractedKey, fileData); err != nil {
 		logger.Error("Failed to upload extracted file to S3", "filename", filename, "error", err)
 		_ = batchService.AddFailedFilename(ctx, batchInfo.ID, filename)
+		errMsg := err.Error()
+		_ = batchService.RecordOutcome(ctx, batchInfo.ID, filename, repository.BatchOutcomeStatusFailedS3Upload, &errMsg, nil)
 		return 0, 1, 0
 	}
 
@@ -315,6 +324,7 @@ func processZipFile(
 	if !strings.HasSuffix(strings.ToLower(filename), ".pdf") {
 		logger.Warn("Invalid file type", "filename", filename)
 		_ = batchService.AddFailedFilename(ctx, batchInfo.ID, filename)
+		_ = batchService.RecordOutcome(ctx, batchInfo.ID, filename, repository.BatchOutcomeStatusInvalidType, nil, nil)
 		return 0, 0, 1
 	}
 
@@ -322,9 +332,21 @@ func processZipFile(
 	if err := uploadHandler(ctx, batchInfo.ClientID, bytes.NewReader(fileData), filename, &batchInfo.ID); err != nil {
 		logger.Error("Failed to upload document", "filename", filename, "error", err)
 		_ = batchService.AddFailedFilename(ctx, batchInfo.ID, filename)
+		errMsg := err.Error()
+		_ = batchService.RecordOutcome(ctx, batchInfo.ID, filename, repository.BatchOutcomeStatusFailedUpload, &errMsg, nil)
 		return 0, 1, 0
 	}
 
+	// Check for duplicate by computing file hash and looking up existing document
+	hash := sha256.Sum256(fileData)
+	hashStr := hex.EncodeToString(hash[:])
+	existingID, _ := batchService.CheckDuplicate(ctx, batchInfo.ClientID, hashStr)
+	if existingID != nil {
+		_ = batchService.RecordOutcome(ctx, batchInfo.ID, filename, repository.BatchOutcomeStatusDuplicate, nil, existingID)
+	} else {
+		_ = batchService.RecordOutcome(ctx, batchInfo.ID, filename, repository.BatchOutcomeStatusSubmitted, nil, nil)
+	}
+
 	logger.Info("Successfully processed file", "filename", filename, "batch_id", batchInfo.ID)
 	return 1, 0, 0
 }