Merged in feature/support-more-types (pull request #219)

support new types for import * tests pass * missing file * bug fixes
2026-03-31 17:40:42 +00:00
parent 8e9889accc
commit b71a28d3c0
144 changed files with 28352 additions and 91 deletions
@@ -59,8 +59,9 @@ func TestProcessBatchWork(t *testing.T) {
 	})
 	require.NoError(t, err)

-	// Create a batch record in the database
-	batchID, err := batchService.CreateWithStorage(ctx, clientID, "batch.zip", 2, bucket, archiveKey, int64(len(zipContent)))
+	// Create a batch record with totalDocs=0 to match production (documents.go:236).
+	// The worker must call SetTotalDocuments after extraction for progress to work.
+	batchID, err := batchService.CreateWithStorage(ctx, clientID, "batch.zip", 0, bucket, archiveKey, int64(len(zipContent)))
 	require.NoError(t, err)

 	// Track uploaded documents
@@ -96,8 +97,17 @@ func TestProcessBatchWork(t *testing.T) {
 	batchInfo, err := batchService.Get(ctx, clientID, batchID)
 	require.NoError(t, err)
 	assert.Equal(t, "completed", batchInfo.Status)
-	assert.Equal(t, int32(2), batchInfo.ProcessedDocuments)
-	assert.Equal(t, int32(0), batchInfo.FailedDocuments)
+	assert.Equal(t, int32(2), batchInfo.TotalDocuments, "TotalDocuments should be set after processing")
+
+	// Summary counts are derived from outcome states. After extraction, outcomes
+	// are in 'submitted' (non-terminal) state -- pipeline hasn't run yet.
+	// Only terminal outcomes (clean_passed, clean_failed, etc.) are counted.
+	outcomes, err := batchService.ListOutcomes(ctx, batchID)
+	require.NoError(t, err)
+	assert.Len(t, outcomes, 2, "should have 2 outcome rows")
+	for _, o := range outcomes {
+		assert.Equal(t, "submitted", o.Outcome)
+	}
 }

 // TestProcessSingleBatch tests processing a single batch
@@ -162,14 +172,23 @@ func TestProcessSingleBatch(t *testing.T) {
 	err = processSingleBatch(ctx, logger, batchService, s3Client, bucket, uploadHandler, batchDetails)
 	require.NoError(t, err)

-	// Verify only PDFs were uploaded
-	assert.Equal(t, 1, uploadCount) // Only 1 PDF file
+	// All three file types (PDF, TXT, DOCX) are now accepted
+	assert.Equal(t, 3, uploadCount)

 	// Verify batch status
 	batchInfo, err := batchService.Get(ctx, clientID, batchID)
 	require.NoError(t, err)
-	assert.Equal(t, int32(1), batchInfo.ProcessedDocuments)
-	assert.Equal(t, int32(2), batchInfo.InvalidTypeDocuments) // 1 txt + 1 docx
+	assert.Equal(t, "completed", batchInfo.Status)
+	assert.Equal(t, int32(3), batchInfo.TotalDocuments)
+
+	// After extraction, outcomes are 'submitted' (non-terminal).
+	// Derived counts only reflect terminal outcomes.
+	outcomes, err := batchService.ListOutcomes(ctx, batchID)
+	require.NoError(t, err)
+	assert.Len(t, outcomes, 3)
+	for _, o := range outcomes {
+		assert.Equal(t, "submitted", o.Outcome)
+	}
 }

 // TestProcessSingleBatchWithFailure tests handling of upload failures
@@ -240,10 +259,12 @@ func TestProcessSingleBatchWithFailure(t *testing.T) {
 	// Verify results
 	batchInfo, err := batchService.Get(ctx, clientID, batchID)
 	require.NoError(t, err)
-	assert.Equal(t, int32(2), batchInfo.ProcessedDocuments) // 1st and 3rd succeeded
-	assert.Equal(t, int32(1), batchInfo.FailedDocuments)    // 2nd failed
-	assert.Equal(t, "completed", batchInfo.Status)          // Status is still completed since some succeeded
+	assert.Equal(t, "completed", batchInfo.Status) // Status is still completed since some succeeded
 	assert.Contains(t, batchInfo.FailedFilenames, "document_2.pdf")
+
+	// Derived counts: 2 submitted (non-terminal), 1 failed_upload (terminal)
+	assert.Equal(t, int32(0), batchInfo.ProcessedDocuments, "submitted is non-terminal")
+	assert.Equal(t, int32(1), batchInfo.FailedDocuments, "2nd file failed upload")
 }

 // TestDownloadZipFromS3 tests downloading a ZIP file from S3
@@ -498,7 +519,7 @@ func TestProcessBatchWithNestedFolders(t *testing.T) {
 	require.NoError(t, err)

 	// Create batch
-	batchID, err := batchService.CreateWithStorage(ctx, clientID, "nested.zip", 7, bucket, archiveKey, int64(len(zipContent)))
+	batchID, err := batchService.CreateWithStorage(ctx, clientID, "nested.zip", 8, bucket, archiveKey, int64(len(zipContent)))
 	require.NoError(t, err)

 	// Use the real upload handler that creates documents in the database
@@ -524,9 +545,11 @@ func TestProcessBatchWithNestedFolders(t *testing.T) {
 	// Verify batch status
 	batchInfo, err := batchService.Get(ctx, clientID, batchID)
 	require.NoError(t, err)
-	assert.Equal(t, int32(7), batchInfo.ProcessedDocuments)   // 7 PDFs
-	assert.Equal(t, int32(1), batchInfo.InvalidTypeDocuments) // 1 txt file
 	assert.Equal(t, "completed", batchInfo.Status)
+	assert.Equal(t, int32(8), batchInfo.TotalDocuments, "7 PDFs + 1 TXT")
+	// After extraction, all outcomes are 'submitted' (non-terminal).
+	// Derived ProcessedDocuments only counts terminal outcomes.
+	assert.Equal(t, int32(0), batchInfo.InvalidTypeDocuments, "all types accepted")

 	// Manually trigger document initialization for uploaded files since S3 notifications
 	// don't work in test environment. Get all document uploads for this batch.
@@ -562,8 +585,8 @@ func TestProcessBatchWithNestedFolders(t *testing.T) {
 	batchDocs, err := cfg.GetDBQueries().ListDocumentsByBatch(ctx, &batchID)
 	require.NoError(t, err)

-	// Verify we have the expected number of documents
-	assert.Equal(t, 7, len(batchDocs), "Should have exactly 7 documents in database")
+	// Verify we have the expected number of documents (7 PDFs + 1 TXT)
+	assert.Equal(t, 8, len(batchDocs), "Should have exactly 8 documents in database")

 	// Verify filenames are preserved with their paths
 	foundFilenames := make(map[string]bool)
@@ -582,6 +605,7 @@ func TestProcessBatchWithNestedFolders(t *testing.T) {
 		"folderb/folderc/file4.pdf",
 		"folderb/file5.pdf",
 		"deep/nested/structure/test/file6.pdf",
+		"foldera/readme.txt",
 	}

 	for _, expectedFilename := range expectedFilenames {
@@ -874,9 +898,10 @@ func TestProcessZipFile_MixedOutcomes(t *testing.T) {
 		outcomeMap[o.Filename] = o.Outcome
 	}

+	// All three types are now accepted
 	assert.Equal(t, "submitted", outcomeMap["document.pdf"])
-	assert.Equal(t, "invalid_type", outcomeMap["notes.txt"])
-	assert.Equal(t, "invalid_type", outcomeMap["report.docx"])
+	assert.Equal(t, "submitted", outcomeMap["notes.txt"])
+	assert.Equal(t, "submitted", outcomeMap["report.docx"])
 }

 // sha256Hash computes the hex-encoded SHA-256 hash of data.
@@ -986,3 +1011,269 @@ func TestProcessBatchWithDuplicateFilenames(t *testing.T) {
 	assert.Contains(t, uploadedDocs["folder2/document.pdf"], "Folder2 document")
 	assert.Contains(t, uploadedDocs["folder1/subfolder/document.pdf"], "Subfolder document")
 }
+
+// TestProcessBatchWithAllSupportedFileTypes tests the full multi-format batch upload pipeline.
+// A batch ZIP containing one file of each supported type is uploaded and every document
+// ends up in the system as a first-class document with an ID. This is the critical
+// integration test for multi-format support.
+func TestProcessBatchWithAllSupportedFileTypes(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+	}
+
+	ctx := t.Context()
+	cfg := &BaseConfig{}
+	_ = serviceconfig.InitializeConfig(cfg)
+	test.CreateDB(t, cfg)
+	test.CreateAWSResources(t, cfg)
+
+	clientID := "test_multiformat_batch"
+	test.CreateTestClient(t, cfg, clientID, "Test Multi-Format Batch Client")
+
+	batchService := batch.New(cfg)
+	s3ClientInterface := cfg.GetStoreClient()
+	s3Client, ok := s3ClientInterface.(*s3.Client)
+	require.True(t, ok)
+	bucket := cfg.GetBucket()
+
+	// Build a ZIP containing one file of each supported type in folder structure
+	zipContent := createMultiFormatZIP(t)
+	archiveKey := fmt.Sprintf("test/%s/multiformat.zip", clientID)
+
+	_, err := s3Client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(archiveKey),
+		Body:   bytes.NewReader(zipContent),
+	})
+	require.NoError(t, err)
+
+	// 10 files total: PDF, TIFF, JPEG, PNG, BMP, DOCX, XLSX, PPTX, TXT, EML
+	batchID, err := batchService.CreateWithStorage(ctx, clientID, "multiformat.zip", 10, bucket, archiveKey, int64(len(zipContent)))
+	require.NoError(t, err)
+
+	// Use the real upload handler that persists to the database
+	uploadHandler := createDocumentUploadHandler(cfg)
+
+	batchDetails := &batch.BatchUploadDetails{
+		BatchUploadSummary: batch.BatchUploadSummary{
+			ID:               batchID,
+			ClientID:         clientID,
+			OriginalFilename: "multiformat.zip",
+			Status:           "processing",
+		},
+		ArchiveKey:    archiveKey,
+		ArchiveBucket: bucket,
+	}
+
+	logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
+	err = processSingleBatch(ctx, logger, batchService, s3Client, bucket, uploadHandler, batchDetails)
+	require.NoError(t, err)
+
+	// Verify batch completed successfully
+	batchInfo, err := batchService.Get(ctx, clientID, batchID)
+	require.NoError(t, err)
+	assert.Equal(t, "completed", batchInfo.Status, "batch should complete")
+	assert.Equal(t, int32(10), batchInfo.TotalDocuments, "all 10 files should be extracted")
+	// After extraction, outcomes are 'submitted' (non-terminal). Derived counts
+	// only reflect terminal outcomes so ProcessedDocuments is 0 at this stage.
+	assert.Equal(t, int32(0), batchInfo.FailedDocuments, "no files should fail")
+	assert.Equal(t, int32(0), batchInfo.InvalidTypeDocuments, "no files should be invalid type")
+
+	// Verify outcomes: every file should be submitted
+	outcomes, err := batchService.ListOutcomes(ctx, batchID)
+	require.NoError(t, err)
+	assert.Len(t, outcomes, 10, "should have 10 outcomes")
+
+	for _, o := range outcomes {
+		assert.Equal(t, "submitted", o.Outcome,
+			"file %s should be submitted, got %s", o.Filename, o.Outcome)
+	}
+
+	// Verify folder structure is preserved in outcomes
+	outcomeFilenames := make(map[string]bool)
+	for _, o := range outcomes {
+		outcomeFilenames[o.Filename] = true
+	}
+
+	expectedFiles := []string{
+		"documents/report.pdf",
+		"images/scan.tiff",
+		"images/photo.jpeg",
+		"images/diagram.png",
+		"images/legacy.bmp",
+		"office/proposal.docx",
+		"office/data.xlsx",
+		"office/slides.pptx",
+		"text/notes.txt",
+		"email/message.eml",
+	}
+	for _, f := range expectedFiles {
+		assert.True(t, outcomeFilenames[f], "should have outcome for %s", f)
+	}
+}
+
+// TestProcessBatchWithMixedValidInvalidFileTypes tests a batch with valid new-format
+// files and unsupported files, verifying that valid files succeed and unsupported
+// files get appropriate rejection without aborting the batch.
+func TestProcessBatchWithMixedValidInvalidFileTypes(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+	}
+
+	ctx := t.Context()
+	cfg := &BaseConfig{}
+	_ = serviceconfig.InitializeConfig(cfg)
+	test.CreateDB(t, cfg)
+	test.CreateAWSResources(t, cfg)
+
+	clientID := "test_mixed_valid_invalid"
+	test.CreateTestClient(t, cfg, clientID, "Test Mixed Valid Invalid Client")
+
+	batchService := batch.New(cfg)
+	s3ClientInterface := cfg.GetStoreClient()
+	s3Client, ok := s3ClientInterface.(*s3.Client)
+	require.True(t, ok)
+	bucket := cfg.GetBucket()
+
+	// Create ZIP with mix of valid and unsupported files
+	zipContent := createMixedValidInvalidZIP(t)
+	archiveKey := fmt.Sprintf("test/%s/mixed_valid_invalid.zip", clientID)
+
+	_, err := s3Client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(archiveKey),
+		Body:   bytes.NewReader(zipContent),
+	})
+	require.NoError(t, err)
+
+	batchID, err := batchService.CreateWithStorage(ctx, clientID, "mixed_valid_invalid.zip", 5, bucket, archiveKey, int64(len(zipContent)))
+	require.NoError(t, err)
+
+	uploadHandler := func(_ context.Context, _ string, docData io.Reader, _ string, _ *uuid.UUID) error {
+		_, _ = io.ReadAll(docData)
+		return nil
+	}
+
+	batchDetails := &batch.BatchUploadDetails{
+		BatchUploadSummary: batch.BatchUploadSummary{
+			ID:               batchID,
+			ClientID:         clientID,
+			OriginalFilename: "mixed_valid_invalid.zip",
+			Status:           "processing",
+		},
+		ArchiveKey:    archiveKey,
+		ArchiveBucket: bucket,
+	}
+
+	logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
+	err = processSingleBatch(ctx, logger, batchService, s3Client, bucket, uploadHandler, batchDetails)
+	require.NoError(t, err)
+
+	// Batch should still complete (not fail) because some files succeeded
+	batchInfo, err := batchService.Get(ctx, clientID, batchID)
+	require.NoError(t, err)
+	assert.Equal(t, "completed", batchInfo.Status)
+	// Derived counts: 3 submitted (non-terminal), 2 invalid_type (terminal)
+	assert.Equal(t, int32(0), batchInfo.ProcessedDocuments, "submitted is non-terminal")
+	assert.Equal(t, int32(2), batchInfo.InvalidTypeDocuments, "2 unsupported files")
+
+	outcomes, err := batchService.ListOutcomes(ctx, batchID)
+	require.NoError(t, err)
+
+	outcomeMap := map[string]string{}
+	for _, o := range outcomes {
+		outcomeMap[o.Filename] = o.Outcome
+	}
+
+	// Valid files submitted
+	assert.Equal(t, "submitted", outcomeMap["report.pdf"])
+	assert.Equal(t, "submitted", outcomeMap["notes.txt"])
+	assert.Equal(t, "submitted", outcomeMap["photo.png"])
+
+	// Unsupported files rejected
+	assert.Equal(t, "invalid_type", outcomeMap["malware.exe"])
+	assert.Equal(t, "invalid_type", outcomeMap["archive.zip"])
+}
+
+// createMultiFormatZIP creates a ZIP with one valid file of each supported type
+// organized in folders.
+func createMultiFormatZIP(t *testing.T) []byte {
+	t.Helper()
+
+	// Read fixture files from the types testdata directory
+	fixtureDir := "../../document/types/testdata/multiformat"
+
+	files := []struct {
+		zipPath     string
+		fixtureName string
+	}{
+		{"documents/report.pdf", ""}, // PDF needs to be inline (no fixture)
+		{"images/scan.tiff", "valid.tiff"},
+		{"images/photo.jpeg", "valid.jpeg"},
+		{"images/diagram.png", "valid.png"},
+		{"images/legacy.bmp", "valid.bmp"},
+		{"office/proposal.docx", "valid.docx"},
+		{"office/data.xlsx", "valid.xlsx"},
+		{"office/slides.pptx", "valid.pptx"},
+		{"text/notes.txt", "valid.txt"},
+		{"email/message.eml", "valid.eml"},
+	}
+
+	var buf bytes.Buffer
+	zipWriter := zip.NewWriter(&buf)
+
+	for _, f := range files {
+		var data []byte
+		var err error
+
+		if f.fixtureName == "" {
+			// Inline PDF content
+			data = []byte("%%PDF-1.4\n%%Multi-format test PDF\n%%%%EOF")
+		} else {
+			data, err = os.ReadFile(fmt.Sprintf("%s/%s", fixtureDir, f.fixtureName))
+			require.NoError(t, err, "failed to read fixture %s", f.fixtureName)
+		}
+
+		w, err := zipWriter.Create(f.zipPath)
+		require.NoError(t, err)
+		_, err = w.Write(data)
+		require.NoError(t, err)
+	}
+
+	require.NoError(t, zipWriter.Close())
+	return buf.Bytes()
+}
+
+// createMixedValidInvalidZIP creates a ZIP with valid supported files and unsupported files.
+func createMixedValidInvalidZIP(t *testing.T) []byte {
+	t.Helper()
+
+	files := []struct {
+		name    string
+		content []byte
+	}{
+		{"report.pdf", []byte("%%PDF-1.4\n%%Test\n%%%%EOF")},
+		{"notes.txt", []byte("This is a valid text file")},
+		{"photo.png", nil}, // will be loaded from fixture
+		{"malware.exe", []byte("MZ\x90\x00\x03\x00")}, // PE signature
+		{"archive.zip", []byte("Not a real zip but has .zip extension")},
+	}
+
+	// Load PNG fixture
+	pngData, err := os.ReadFile("../../document/types/testdata/multiformat/valid.png")
+	require.NoError(t, err)
+	files[2].content = pngData
+
+	var buf bytes.Buffer
+	zipWriter := zip.NewWriter(&buf)
+
+	for _, f := range files {
+		w, err := zipWriter.Create(f.name)
+		require.NoError(t, err)
+		_, err = w.Write(f.content)
+		require.NoError(t, err)
+	}
+
+	require.NoError(t, zipWriter.Close())
+	return buf.Bytes()
+}