Merged in feature/support-more-types (pull request #219)

support new types for import

* tests pass

* missing file

* bug fixes

internal/database/migrations/00000000000125_multiformat_support.down.sql

@@ -0,0 +1,5 @@
+-- PostgreSQL does not support removing enum values directly.
+-- To reverse this migration, the enum types would need to be recreated.
+-- This is intentionally left as a no-op since removing enum values
+-- requires recreating the type and all dependent columns.
+-- In practice, these enum values are safe to leave in place.

internal/database/migrations/00000000000125_multiformat_support.up.sql

@@ -0,0 +1,18 @@
+-- Add new MIME types for multi-format document support
+ALTER TYPE cleanMimeType ADD VALUE IF NOT EXISTS 'image/tiff';
+ALTER TYPE cleanMimeType ADD VALUE IF NOT EXISTS 'image/jpeg';
+ALTER TYPE cleanMimeType ADD VALUE IF NOT EXISTS 'image/png';
+ALTER TYPE cleanMimeType ADD VALUE IF NOT EXISTS 'image/bmp';
+ALTER TYPE cleanMimeType ADD VALUE IF NOT EXISTS 'application/docx';
+ALTER TYPE cleanMimeType ADD VALUE IF NOT EXISTS 'application/xlsx';
+ALTER TYPE cleanMimeType ADD VALUE IF NOT EXISTS 'application/pptx';
+ALTER TYPE cleanMimeType ADD VALUE IF NOT EXISTS 'text/plain';
+ALTER TYPE cleanMimeType ADD VALUE IF NOT EXISTS 'message/rfc822';
+
+-- Add new failure reasons for multi-format validation
+ALTER TYPE cleanFailType ADD VALUE IF NOT EXISTS 'password_protected';
+ALTER TYPE cleanFailType ADD VALUE IF NOT EXISTS 'contains_macros';
+ALTER TYPE cleanFailType ADD VALUE IF NOT EXISTS 'empty_content';
+ALTER TYPE cleanFailType ADD VALUE IF NOT EXISTS 'binary_content';
+ALTER TYPE cleanFailType ADD VALUE IF NOT EXISTS 'unsupported_encoding';
+ALTER TYPE cleanFailType ADD VALUE IF NOT EXISTS 'contains_attachments';

internal/database/queries/batch.sql

@@ -38,10 +38,13 @@ WHERE client_id = $1
 ORDER BY created_at DESC
 LIMIT $2 OFFSET $3;
 
+-- name: SetBatchTotalDocuments :exec
+UPDATE batch_uploads SET total_documents = $2 WHERE id = $1;
+
 -- name: UpdateBatchProgress :exec
-UPDATE batch_uploads 
-SET processed_documents = $2, 
-    failed_documents = $3, 
+UPDATE batch_uploads
+SET processed_documents = $2,
+    failed_documents = $3,
     invalid_type_documents = $4,
     progress_percent = $5
 WHERE id = $1;

internal/database/repository/batch.sql.go

@@ -568,6 +568,23 @@ func (q *Queries) ResolveBatchDocumentOutcome(ctx context.Context, arg *ResolveB
 	return err
 }
 
+const setBatchTotalDocuments = `-- name: SetBatchTotalDocuments :exec
+UPDATE batch_uploads SET total_documents = $2 WHERE id = $1
+`
+
+type SetBatchTotalDocumentsParams struct {
+	ID             uuid.UUID `db:"id"`
+	TotalDocuments int32     `db:"total_documents"`
+}
+
+// SetBatchTotalDocuments
+//
+//	UPDATE batch_uploads SET total_documents = $2 WHERE id = $1
+func (q *Queries) SetBatchTotalDocuments(ctx context.Context, arg *SetBatchTotalDocumentsParams) error {
+	_, err := q.db.Exec(ctx, setBatchTotalDocuments, arg.ID, arg.TotalDocuments)
+	return err
+}
+
 const updateBatchDocumentOutcomeByDocumentID = `-- name: UpdateBatchDocumentOutcomeByDocumentID :exec
 UPDATE batch_document_outcomes
 SET outcome = $2::batch_outcome_status,
@@ -600,9 +617,9 @@ func (q *Queries) UpdateBatchDocumentOutcomeByDocumentID(ctx context.Context, ar
 }
 
 const updateBatchProgress = `-- name: UpdateBatchProgress :exec
-UPDATE batch_uploads 
-SET processed_documents = $2, 
-    failed_documents = $3, 
+UPDATE batch_uploads
+SET processed_documents = $2,
+    failed_documents = $3,
     invalid_type_documents = $4,
     progress_percent = $5
 WHERE id = $1

internal/database/repository/models.go

@@ -143,15 +143,21 @@ func (e BatchStatus) Valid() bool {
 type Cleanfailtype string
 
 const (
-	CleanfailtypeInvalidMimetype  Cleanfailtype = "invalid_mimetype"
-	CleanfailtypeInvalidRead      Cleanfailtype = "invalid_read"
-	CleanfailtypeInvalidReadPages Cleanfailtype = "invalid_read_pages"
-	CleanfailtypeZeroPageCount    Cleanfailtype = "zero_page_count"
-	CleanfailtypeLargeFile        Cleanfailtype = "large_file"
-	CleanfailtypeSmallDimensions  Cleanfailtype = "small_dimensions"
-	CleanfailtypeLargeDimensions  Cleanfailtype = "large_dimensions"
-	CleanfailtypeSmallDpi         Cleanfailtype = "small_dpi"
-	CleanfailtypeLargeDpi         Cleanfailtype = "large_dpi"
+	CleanfailtypeInvalidMimetype     Cleanfailtype = "invalid_mimetype"
+	CleanfailtypeInvalidRead         Cleanfailtype = "invalid_read"
+	CleanfailtypeInvalidReadPages    Cleanfailtype = "invalid_read_pages"
+	CleanfailtypeZeroPageCount       Cleanfailtype = "zero_page_count"
+	CleanfailtypeLargeFile           Cleanfailtype = "large_file"
+	CleanfailtypeSmallDimensions     Cleanfailtype = "small_dimensions"
+	CleanfailtypeLargeDimensions     Cleanfailtype = "large_dimensions"
+	CleanfailtypeSmallDpi            Cleanfailtype = "small_dpi"
+	CleanfailtypeLargeDpi            Cleanfailtype = "large_dpi"
+	CleanfailtypePasswordProtected   Cleanfailtype = "password_protected"
+	CleanfailtypeContainsMacros      Cleanfailtype = "contains_macros"
+	CleanfailtypeEmptyContent        Cleanfailtype = "empty_content"
+	CleanfailtypeBinaryContent       Cleanfailtype = "binary_content"
+	CleanfailtypeUnsupportedEncoding Cleanfailtype = "unsupported_encoding"
+	CleanfailtypeContainsAttachments Cleanfailtype = "contains_attachments"
 )
 
 func (e *Cleanfailtype) Scan(src interface{}) error {
@@ -199,7 +205,13 @@ func (e Cleanfailtype) Valid() bool {
 		CleanfailtypeSmallDimensions,
 		CleanfailtypeLargeDimensions,
 		CleanfailtypeSmallDpi,
-		CleanfailtypeLargeDpi:
+		CleanfailtypeLargeDpi,
+		CleanfailtypePasswordProtected,
+		CleanfailtypeContainsMacros,
+		CleanfailtypeEmptyContent,
+		CleanfailtypeBinaryContent,
+		CleanfailtypeUnsupportedEncoding,
+		CleanfailtypeContainsAttachments:
 		return true
 	}
 	return false
@@ -208,7 +220,16 @@ func (e Cleanfailtype) Valid() bool {
 type Cleanmimetype string
 
 const (
-	CleanmimetypeApplicationPdf Cleanmimetype = "application/pdf"
+	CleanmimetypeApplicationPdf  Cleanmimetype = "application/pdf"
+	CleanmimetypeImageTiff       Cleanmimetype = "image/tiff"
+	CleanmimetypeImageJpeg       Cleanmimetype = "image/jpeg"
+	CleanmimetypeImagePng        Cleanmimetype = "image/png"
+	CleanmimetypeImageBmp        Cleanmimetype = "image/bmp"
+	CleanmimetypeApplicationDocx Cleanmimetype = "application/docx"
+	CleanmimetypeApplicationXlsx Cleanmimetype = "application/xlsx"
+	CleanmimetypeApplicationPptx Cleanmimetype = "application/pptx"
+	CleanmimetypeTextPlain       Cleanmimetype = "text/plain"
+	CleanmimetypeMessageRfc822   Cleanmimetype = "message/rfc822"
 )
 
 func (e *Cleanmimetype) Scan(src interface{}) error {
@@ -248,7 +269,16 @@ func (ns NullCleanmimetype) Value() (driver.Value, error) {
 
 func (e Cleanmimetype) Valid() bool {
 	switch e {
-	case CleanmimetypeApplicationPdf:
+	case CleanmimetypeApplicationPdf,
+		CleanmimetypeImageTiff,
+		CleanmimetypeImageJpeg,
+		CleanmimetypeImagePng,
+		CleanmimetypeImageBmp,
+		CleanmimetypeApplicationDocx,
+		CleanmimetypeApplicationXlsx,
+		CleanmimetypeApplicationPptx,
+		CleanmimetypeTextPlain,
+		CleanmimetypeMessageRfc822:
 		return true
 	}
 	return false