Merged in feature/support-more-types (pull request #219)

support new types for import

* tests pass

* missing file

* bug fixes

internal/server/api/batch_worker.go

@@ -28,6 +28,30 @@ const (
 	ConfigKeyUploadHandler = "uploadHandler"
 )
 
+// acceptedExtensions defines the file extensions allowed in batch uploads.
+// Macro-enabled extensions (.docm, .xlsm, .pptm) are intentionally excluded --
+// documents with macros are rejected, so users must save as standard formats first.
+var acceptedExtensions = map[string]bool{
+	".pdf":  true,
+	".tiff": true,
+	".tif":  true,
+	".jpeg": true,
+	".jpg":  true,
+	".png":  true,
+	".bmp":  true,
+	".docx": true,
+	".xlsx": true,
+	".pptx": true,
+	".txt":  true,
+	".eml":  true,
+}
+
+// isAcceptedExtension checks if a filename has a supported file extension.
+func isAcceptedExtension(filename string) bool {
+	ext := strings.ToLower(path.Ext(filename))
+	return acceptedExtensions[ext]
+}
+
 // sanitizeZipPath cleans a path from a zip file to prevent path traversal attacks
 // while preserving the internal folder structure.
 // Returns empty string if the path is invalid or attempts traversal.
@@ -194,6 +218,12 @@ func processSingleBatch(
 		invalidInt32 = maxInt32 // Max int32 value
 	}
 
+	// Set total document count now that all files have been extracted
+	totalInt32 := processedInt32 + failedInt32 + invalidInt32
+	if err := batchService.SetTotalDocuments(ctx, batchInfo.ID, totalInt32); err != nil {
+		logger.Error("Failed to set batch total documents", "batch_id", batchInfo.ID, "error", err)
+	}
+
 	if err := batchService.UpdateProgress(ctx, batchInfo.ClientID, batchInfo.ID,
 		processedInt32, failedInt32, invalidInt32); err != nil {
 		logger.Error("Failed to update batch progress", "batch_id", batchInfo.ID, "error", err)
@@ -320,8 +350,8 @@ func processZipFile(
 		return 0, 1, 0
 	}
 
-	// Check if file type is valid (PDF only for now)
-	if !strings.HasSuffix(strings.ToLower(filename), ".pdf") {
+	// Check if file type is supported
+	if !isAcceptedExtension(filename) {
 		logger.Warn("Invalid file type", "filename", filename)
 		_ = batchService.AddFailedFilename(ctx, batchInfo.ID, filename)
 		_ = batchService.RecordOutcome(ctx, batchInfo.ID, filename, repository.BatchOutcomeStatusInvalidType, nil, nil)

internal/server/api/batch_worker_extension_test.go

@@ -0,0 +1,95 @@
+package api
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestIsAcceptedExtension_AllSupported(t *testing.T) {
+	supported := []string{
+		".pdf", ".tiff", ".tif", ".jpeg", ".jpg",
+		".png", ".bmp", ".docx", ".xlsx",
+		".pptx", ".txt", ".eml",
+	}
+
+	for _, ext := range supported {
+		t.Run(ext, func(t *testing.T) {
+			assert.True(t, isAcceptedExtension("document"+ext),
+				"expected %s to be accepted", ext)
+		})
+	}
+}
+
+func TestIsAcceptedExtension_CaseInsensitive(t *testing.T) {
+	cases := []struct {
+		filename string
+	}{
+		{"report.PDF"},
+		{"image.Png"},
+		{"file.DOCX"},
+		{"scan.TiFf"},
+		{"mail.EML"},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.filename, func(t *testing.T) {
+			assert.True(t, isAcceptedExtension(tc.filename),
+				"expected %s to be accepted (case insensitive)", tc.filename)
+		})
+	}
+}
+
+func TestIsAcceptedExtension_Unsupported(t *testing.T) {
+	unsupported := []struct {
+		filename string
+	}{
+		{"virus.exe"},
+		{"library.dll"},
+		{"outlook.msg"},
+		{"legacy.doc"},
+		{"old.xls"},
+		{"slides.ppt"},
+		{"archive.zip"},
+		{"data.csv"},
+		{"page.html"},
+		{"macros.docm"},
+		{"macros.xlsm"},
+		{"macros.pptm"},
+	}
+
+	for _, tc := range unsupported {
+		t.Run(tc.filename, func(t *testing.T) {
+			assert.False(t, isAcceptedExtension(tc.filename),
+				"expected %s to be rejected", tc.filename)
+		})
+	}
+}
+
+func TestIsAcceptedExtension_NoExtension(t *testing.T) {
+	assert.False(t, isAcceptedExtension("README"),
+		"file with no extension should be rejected")
+	assert.False(t, isAcceptedExtension("Makefile"),
+		"file with no extension should be rejected")
+}
+
+func TestIsAcceptedExtension_WithPath(t *testing.T) {
+	cases := []struct {
+		name     string
+		filename string
+		want     bool
+	}{
+		{"nested pdf", "folder/document.pdf", true},
+		{"deep path docx", "a/b/c/report.docx", true},
+		{"nested unsupported", "folder/file.exe", false},
+		{"path with spaces", "my folder/scan.png", true},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := isAcceptedExtension(tc.filename)
+			assert.Equal(t, tc.want, got,
+				"isAcceptedExtension(%q) = %v, want %v", tc.filename, got, tc.want)
+		})
+	}
+}

internal/server/api/batch_worker_test.go

@@ -59,8 +59,9 @@ func TestProcessBatchWork(t *testing.T) {
 	})
 	require.NoError(t, err)
 
-	// Create a batch record in the database
-	batchID, err := batchService.CreateWithStorage(ctx, clientID, "batch.zip", 2, bucket, archiveKey, int64(len(zipContent)))
+	// Create a batch record with totalDocs=0 to match production (documents.go:236).
+	// The worker must call SetTotalDocuments after extraction for progress to work.
+	batchID, err := batchService.CreateWithStorage(ctx, clientID, "batch.zip", 0, bucket, archiveKey, int64(len(zipContent)))
 	require.NoError(t, err)
 
 	// Track uploaded documents
@@ -96,8 +97,17 @@ func TestProcessBatchWork(t *testing.T) {
 	batchInfo, err := batchService.Get(ctx, clientID, batchID)
 	require.NoError(t, err)
 	assert.Equal(t, "completed", batchInfo.Status)
-	assert.Equal(t, int32(2), batchInfo.ProcessedDocuments)
-	assert.Equal(t, int32(0), batchInfo.FailedDocuments)
+	assert.Equal(t, int32(2), batchInfo.TotalDocuments, "TotalDocuments should be set after processing")
+
+	// Summary counts are derived from outcome states. After extraction, outcomes
+	// are in 'submitted' (non-terminal) state -- pipeline hasn't run yet.
+	// Only terminal outcomes (clean_passed, clean_failed, etc.) are counted.
+	outcomes, err := batchService.ListOutcomes(ctx, batchID)
+	require.NoError(t, err)
+	assert.Len(t, outcomes, 2, "should have 2 outcome rows")
+	for _, o := range outcomes {
+		assert.Equal(t, "submitted", o.Outcome)
+	}
 }
 
 // TestProcessSingleBatch tests processing a single batch
@@ -162,14 +172,23 @@ func TestProcessSingleBatch(t *testing.T) {
 	err = processSingleBatch(ctx, logger, batchService, s3Client, bucket, uploadHandler, batchDetails)
 	require.NoError(t, err)
 
-	// Verify only PDFs were uploaded
-	assert.Equal(t, 1, uploadCount) // Only 1 PDF file
+	// All three file types (PDF, TXT, DOCX) are now accepted
+	assert.Equal(t, 3, uploadCount)
 
 	// Verify batch status
 	batchInfo, err := batchService.Get(ctx, clientID, batchID)
 	require.NoError(t, err)
-	assert.Equal(t, int32(1), batchInfo.ProcessedDocuments)
-	assert.Equal(t, int32(2), batchInfo.InvalidTypeDocuments) // 1 txt + 1 docx
+	assert.Equal(t, "completed", batchInfo.Status)
+	assert.Equal(t, int32(3), batchInfo.TotalDocuments)
+
+	// After extraction, outcomes are 'submitted' (non-terminal).
+	// Derived counts only reflect terminal outcomes.
+	outcomes, err := batchService.ListOutcomes(ctx, batchID)
+	require.NoError(t, err)
+	assert.Len(t, outcomes, 3)
+	for _, o := range outcomes {
+		assert.Equal(t, "submitted", o.Outcome)
+	}
 }
 
 // TestProcessSingleBatchWithFailure tests handling of upload failures
@@ -240,10 +259,12 @@ func TestProcessSingleBatchWithFailure(t *testing.T) {
 	// Verify results
 	batchInfo, err := batchService.Get(ctx, clientID, batchID)
 	require.NoError(t, err)
-	assert.Equal(t, int32(2), batchInfo.ProcessedDocuments) // 1st and 3rd succeeded
-	assert.Equal(t, int32(1), batchInfo.FailedDocuments)    // 2nd failed
-	assert.Equal(t, "completed", batchInfo.Status)          // Status is still completed since some succeeded
+	assert.Equal(t, "completed", batchInfo.Status) // Status is still completed since some succeeded
 	assert.Contains(t, batchInfo.FailedFilenames, "document_2.pdf")
+
+	// Derived counts: 2 submitted (non-terminal), 1 failed_upload (terminal)
+	assert.Equal(t, int32(0), batchInfo.ProcessedDocuments, "submitted is non-terminal")
+	assert.Equal(t, int32(1), batchInfo.FailedDocuments, "2nd file failed upload")
 }
 
 // TestDownloadZipFromS3 tests downloading a ZIP file from S3
@@ -498,7 +519,7 @@ func TestProcessBatchWithNestedFolders(t *testing.T) {
 	require.NoError(t, err)
 
 	// Create batch
-	batchID, err := batchService.CreateWithStorage(ctx, clientID, "nested.zip", 7, bucket, archiveKey, int64(len(zipContent)))
+	batchID, err := batchService.CreateWithStorage(ctx, clientID, "nested.zip", 8, bucket, archiveKey, int64(len(zipContent)))
 	require.NoError(t, err)
 
 	// Use the real upload handler that creates documents in the database
@@ -524,9 +545,11 @@ func TestProcessBatchWithNestedFolders(t *testing.T) {
 	// Verify batch status
 	batchInfo, err := batchService.Get(ctx, clientID, batchID)
 	require.NoError(t, err)
-	assert.Equal(t, int32(7), batchInfo.ProcessedDocuments)   // 7 PDFs
-	assert.Equal(t, int32(1), batchInfo.InvalidTypeDocuments) // 1 txt file
 	assert.Equal(t, "completed", batchInfo.Status)
+	assert.Equal(t, int32(8), batchInfo.TotalDocuments, "7 PDFs + 1 TXT")
+	// After extraction, all outcomes are 'submitted' (non-terminal).
+	// Derived ProcessedDocuments only counts terminal outcomes.
+	assert.Equal(t, int32(0), batchInfo.InvalidTypeDocuments, "all types accepted")
 
 	// Manually trigger document initialization for uploaded files since S3 notifications
 	// don't work in test environment. Get all document uploads for this batch.
@@ -562,8 +585,8 @@ func TestProcessBatchWithNestedFolders(t *testing.T) {
 	batchDocs, err := cfg.GetDBQueries().ListDocumentsByBatch(ctx, &batchID)
 	require.NoError(t, err)
 
-	// Verify we have the expected number of documents
-	assert.Equal(t, 7, len(batchDocs), "Should have exactly 7 documents in database")
+	// Verify we have the expected number of documents (7 PDFs + 1 TXT)
+	assert.Equal(t, 8, len(batchDocs), "Should have exactly 8 documents in database")
 
 	// Verify filenames are preserved with their paths
 	foundFilenames := make(map[string]bool)
@@ -582,6 +605,7 @@ func TestProcessBatchWithNestedFolders(t *testing.T) {
 		"folderb/folderc/file4.pdf",
 		"folderb/file5.pdf",
 		"deep/nested/structure/test/file6.pdf",
+		"foldera/readme.txt",
 	}
 
 	for _, expectedFilename := range expectedFilenames {
@@ -874,9 +898,10 @@ func TestProcessZipFile_MixedOutcomes(t *testing.T) {
 		outcomeMap[o.Filename] = o.Outcome
 	}
 
+	// All three types are now accepted
 	assert.Equal(t, "submitted", outcomeMap["document.pdf"])
-	assert.Equal(t, "invalid_type", outcomeMap["notes.txt"])
-	assert.Equal(t, "invalid_type", outcomeMap["report.docx"])
+	assert.Equal(t, "submitted", outcomeMap["notes.txt"])
+	assert.Equal(t, "submitted", outcomeMap["report.docx"])
 }
 
 // sha256Hash computes the hex-encoded SHA-256 hash of data.
@@ -986,3 +1011,269 @@ func TestProcessBatchWithDuplicateFilenames(t *testing.T) {
 	assert.Contains(t, uploadedDocs["folder2/document.pdf"], "Folder2 document")
 	assert.Contains(t, uploadedDocs["folder1/subfolder/document.pdf"], "Subfolder document")
 }
+
+// TestProcessBatchWithAllSupportedFileTypes tests the full multi-format batch upload pipeline.
+// A batch ZIP containing one file of each supported type is uploaded and every document
+// ends up in the system as a first-class document with an ID. This is the critical
+// integration test for multi-format support.
+func TestProcessBatchWithAllSupportedFileTypes(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+	}
+
+	ctx := t.Context()
+	cfg := &BaseConfig{}
+	_ = serviceconfig.InitializeConfig(cfg)
+	test.CreateDB(t, cfg)
+	test.CreateAWSResources(t, cfg)
+
+	clientID := "test_multiformat_batch"
+	test.CreateTestClient(t, cfg, clientID, "Test Multi-Format Batch Client")
+
+	batchService := batch.New(cfg)
+	s3ClientInterface := cfg.GetStoreClient()
+	s3Client, ok := s3ClientInterface.(*s3.Client)
+	require.True(t, ok)
+	bucket := cfg.GetBucket()
+
+	// Build a ZIP containing one file of each supported type in folder structure
+	zipContent := createMultiFormatZIP(t)
+	archiveKey := fmt.Sprintf("test/%s/multiformat.zip", clientID)
+
+	_, err := s3Client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(archiveKey),
+		Body:   bytes.NewReader(zipContent),
+	})
+	require.NoError(t, err)
+
+	// 10 files total: PDF, TIFF, JPEG, PNG, BMP, DOCX, XLSX, PPTX, TXT, EML
+	batchID, err := batchService.CreateWithStorage(ctx, clientID, "multiformat.zip", 10, bucket, archiveKey, int64(len(zipContent)))
+	require.NoError(t, err)
+
+	// Use the real upload handler that persists to the database
+	uploadHandler := createDocumentUploadHandler(cfg)
+
+	batchDetails := &batch.BatchUploadDetails{
+		BatchUploadSummary: batch.BatchUploadSummary{
+			ID:               batchID,
+			ClientID:         clientID,
+			OriginalFilename: "multiformat.zip",
+			Status:           "processing",
+		},
+		ArchiveKey:    archiveKey,
+		ArchiveBucket: bucket,
+	}
+
+	logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
+	err = processSingleBatch(ctx, logger, batchService, s3Client, bucket, uploadHandler, batchDetails)
+	require.NoError(t, err)
+
+	// Verify batch completed successfully
+	batchInfo, err := batchService.Get(ctx, clientID, batchID)
+	require.NoError(t, err)
+	assert.Equal(t, "completed", batchInfo.Status, "batch should complete")
+	assert.Equal(t, int32(10), batchInfo.TotalDocuments, "all 10 files should be extracted")
+	// After extraction, outcomes are 'submitted' (non-terminal). Derived counts
+	// only reflect terminal outcomes so ProcessedDocuments is 0 at this stage.
+	assert.Equal(t, int32(0), batchInfo.FailedDocuments, "no files should fail")
+	assert.Equal(t, int32(0), batchInfo.InvalidTypeDocuments, "no files should be invalid type")
+
+	// Verify outcomes: every file should be submitted
+	outcomes, err := batchService.ListOutcomes(ctx, batchID)
+	require.NoError(t, err)
+	assert.Len(t, outcomes, 10, "should have 10 outcomes")
+
+	for _, o := range outcomes {
+		assert.Equal(t, "submitted", o.Outcome,
+			"file %s should be submitted, got %s", o.Filename, o.Outcome)
+	}
+
+	// Verify folder structure is preserved in outcomes
+	outcomeFilenames := make(map[string]bool)
+	for _, o := range outcomes {
+		outcomeFilenames[o.Filename] = true
+	}
+
+	expectedFiles := []string{
+		"documents/report.pdf",
+		"images/scan.tiff",
+		"images/photo.jpeg",
+		"images/diagram.png",
+		"images/legacy.bmp",
+		"office/proposal.docx",
+		"office/data.xlsx",
+		"office/slides.pptx",
+		"text/notes.txt",
+		"email/message.eml",
+	}
+	for _, f := range expectedFiles {
+		assert.True(t, outcomeFilenames[f], "should have outcome for %s", f)
+	}
+}
+
+// TestProcessBatchWithMixedValidInvalidFileTypes tests a batch with valid new-format
+// files and unsupported files, verifying that valid files succeed and unsupported
+// files get appropriate rejection without aborting the batch.
+func TestProcessBatchWithMixedValidInvalidFileTypes(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+	}
+
+	ctx := t.Context()
+	cfg := &BaseConfig{}
+	_ = serviceconfig.InitializeConfig(cfg)
+	test.CreateDB(t, cfg)
+	test.CreateAWSResources(t, cfg)
+
+	clientID := "test_mixed_valid_invalid"
+	test.CreateTestClient(t, cfg, clientID, "Test Mixed Valid Invalid Client")
+
+	batchService := batch.New(cfg)
+	s3ClientInterface := cfg.GetStoreClient()
+	s3Client, ok := s3ClientInterface.(*s3.Client)
+	require.True(t, ok)
+	bucket := cfg.GetBucket()
+
+	// Create ZIP with mix of valid and unsupported files
+	zipContent := createMixedValidInvalidZIP(t)
+	archiveKey := fmt.Sprintf("test/%s/mixed_valid_invalid.zip", clientID)
+
+	_, err := s3Client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(archiveKey),
+		Body:   bytes.NewReader(zipContent),
+	})
+	require.NoError(t, err)
+
+	batchID, err := batchService.CreateWithStorage(ctx, clientID, "mixed_valid_invalid.zip", 5, bucket, archiveKey, int64(len(zipContent)))
+	require.NoError(t, err)
+
+	uploadHandler := func(_ context.Context, _ string, docData io.Reader, _ string, _ *uuid.UUID) error {
+		_, _ = io.ReadAll(docData)
+		return nil
+	}
+
+	batchDetails := &batch.BatchUploadDetails{
+		BatchUploadSummary: batch.BatchUploadSummary{
+			ID:               batchID,
+			ClientID:         clientID,
+			OriginalFilename: "mixed_valid_invalid.zip",
+			Status:           "processing",
+		},
+		ArchiveKey:    archiveKey,
+		ArchiveBucket: bucket,
+	}
+
+	logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
+	err = processSingleBatch(ctx, logger, batchService, s3Client, bucket, uploadHandler, batchDetails)
+	require.NoError(t, err)
+
+	// Batch should still complete (not fail) because some files succeeded
+	batchInfo, err := batchService.Get(ctx, clientID, batchID)
+	require.NoError(t, err)
+	assert.Equal(t, "completed", batchInfo.Status)
+	// Derived counts: 3 submitted (non-terminal), 2 invalid_type (terminal)
+	assert.Equal(t, int32(0), batchInfo.ProcessedDocuments, "submitted is non-terminal")
+	assert.Equal(t, int32(2), batchInfo.InvalidTypeDocuments, "2 unsupported files")
+
+	outcomes, err := batchService.ListOutcomes(ctx, batchID)
+	require.NoError(t, err)
+
+	outcomeMap := map[string]string{}
+	for _, o := range outcomes {
+		outcomeMap[o.Filename] = o.Outcome
+	}
+
+	// Valid files submitted
+	assert.Equal(t, "submitted", outcomeMap["report.pdf"])
+	assert.Equal(t, "submitted", outcomeMap["notes.txt"])
+	assert.Equal(t, "submitted", outcomeMap["photo.png"])
+
+	// Unsupported files rejected
+	assert.Equal(t, "invalid_type", outcomeMap["malware.exe"])
+	assert.Equal(t, "invalid_type", outcomeMap["archive.zip"])
+}
+
+// createMultiFormatZIP creates a ZIP with one valid file of each supported type
+// organized in folders.
+func createMultiFormatZIP(t *testing.T) []byte {
+	t.Helper()
+
+	// Read fixture files from the types testdata directory
+	fixtureDir := "../../document/types/testdata/multiformat"
+
+	files := []struct {
+		zipPath     string
+		fixtureName string
+	}{
+		{"documents/report.pdf", ""}, // PDF needs to be inline (no fixture)
+		{"images/scan.tiff", "valid.tiff"},
+		{"images/photo.jpeg", "valid.jpeg"},
+		{"images/diagram.png", "valid.png"},
+		{"images/legacy.bmp", "valid.bmp"},
+		{"office/proposal.docx", "valid.docx"},
+		{"office/data.xlsx", "valid.xlsx"},
+		{"office/slides.pptx", "valid.pptx"},
+		{"text/notes.txt", "valid.txt"},
+		{"email/message.eml", "valid.eml"},
+	}
+
+	var buf bytes.Buffer
+	zipWriter := zip.NewWriter(&buf)
+
+	for _, f := range files {
+		var data []byte
+		var err error
+
+		if f.fixtureName == "" {
+			// Inline PDF content
+			data = []byte("%%PDF-1.4\n%%Multi-format test PDF\n%%%%EOF")
+		} else {
+			data, err = os.ReadFile(fmt.Sprintf("%s/%s", fixtureDir, f.fixtureName))
+			require.NoError(t, err, "failed to read fixture %s", f.fixtureName)
+		}
+
+		w, err := zipWriter.Create(f.zipPath)
+		require.NoError(t, err)
+		_, err = w.Write(data)
+		require.NoError(t, err)
+	}
+
+	require.NoError(t, zipWriter.Close())
+	return buf.Bytes()
+}
+
+// createMixedValidInvalidZIP creates a ZIP with valid supported files and unsupported files.
+func createMixedValidInvalidZIP(t *testing.T) []byte {
+	t.Helper()
+
+	files := []struct {
+		name    string
+		content []byte
+	}{
+		{"report.pdf", []byte("%%PDF-1.4\n%%Test\n%%%%EOF")},
+		{"notes.txt", []byte("This is a valid text file")},
+		{"photo.png", nil}, // will be loaded from fixture
+		{"malware.exe", []byte("MZ\x90\x00\x03\x00")}, // PE signature
+		{"archive.zip", []byte("Not a real zip but has .zip extension")},
+	}
+
+	// Load PNG fixture
+	pngData, err := os.ReadFile("../../document/types/testdata/multiformat/valid.png")
+	require.NoError(t, err)
+	files[2].content = pngData
+
+	var buf bytes.Buffer
+	zipWriter := zip.NewWriter(&buf)
+
+	for _, f := range files {
+		w, err := zipWriter.Create(f.name)
+		require.NoError(t, err)
+		_, err = w.Write(f.content)
+		require.NoError(t, err)
+	}
+
+	require.NoError(t, zipWriter.Close())
+	return buf.Bytes()
+}