Merged in feature/batch-status (pull request #215)

Implement and test the batch status feature * working
2026-03-12 18:53:42 +00:00
parent 8a4029058b
commit aafe7d5b5f
39 changed files with 4252 additions and 516 deletions
@@ -0,0 +1,4 @@
+DROP INDEX IF EXISTS idx_documents_batch_id_filename;
+DROP INDEX IF EXISTS idx_batch_doc_outcomes_document_id;
+DROP TABLE IF EXISTS batch_document_outcomes;
+DROP TYPE IF EXISTS batch_outcome_status;
@@ -0,0 +1,35 @@
+CREATE TYPE batch_outcome_status AS ENUM (
+    -- Extraction-time outcomes (set by batch worker)
+    'submitted',
+    'duplicate',
+    'failed_open',
+    'failed_read',
+    'failed_s3_upload',
+    'failed_upload',
+    'invalid_type',
+    -- Pipeline outcomes (set by runners, update the row in place)
+    'init_complete',
+    'init_duplicate',
+    'sync_complete',
+    'sync_skipped',
+    'clean_passed',
+    'clean_failed'
+);
+
+CREATE TABLE batch_document_outcomes (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    batch_id uuid NOT NULL,
+    filename text NOT NULL,
+    outcome batch_outcome_status NOT NULL,
+    error_detail text,
+    document_id uuid,
+    updated_at timestamp NOT NULL DEFAULT NOW(),
+    created_at timestamp NOT NULL DEFAULT NOW(),
+    FOREIGN KEY (batch_id) REFERENCES batch_uploads(id),
+    FOREIGN KEY (document_id) REFERENCES documents(id),
+    UNIQUE (batch_id, filename)
+);
+
+CREATE INDEX idx_batch_doc_outcomes_batch_id ON batch_document_outcomes(batch_id);
+CREATE INDEX idx_batch_doc_outcomes_document_id ON batch_document_outcomes(document_id);
+CREATE INDEX idx_documents_batch_id_filename ON documents(batch_id, filename);
@@ -68,9 +68,61 @@ FROM documents
 WHERE batch_id = $1;

 -- name: GetUnprocessedBatches :many
-SELECT id, client_id, original_filename, total_documents, 
+SELECT id, client_id, original_filename, total_documents,
       processed_documents, failed_documents, invalid_type_documents,
       status, progress_percent, created_at, completed_at
-FROM batch_uploads 
+FROM batch_uploads
 WHERE status = 'processing'
-ORDER BY created_at ASC;
+ORDER BY created_at ASC;
+
+-- name: InsertBatchDocumentOutcome :exec
+-- Upsert: if a row for this (batch_id, filename) already exists (e.g. batch
+-- retry after partial failure), overwrite it with the latest outcome.
+INSERT INTO batch_document_outcomes (batch_id, filename, outcome, error_detail, document_id)
+VALUES ($1, $2, $3::batch_outcome_status, $4, $5)
+ON CONFLICT (batch_id, filename) DO UPDATE
+SET outcome = EXCLUDED.outcome,
+    error_detail = EXCLUDED.error_detail,
+    document_id = EXCLUDED.document_id,
+    updated_at = NOW();
+
+-- name: UpdateBatchDocumentOutcomeByDocumentID :exec
+-- Updates the outcome for a document progressing through the pipeline.
+-- Only updates rows in non-terminal pipeline states to prevent overwriting
+-- terminal outcomes (e.g. a 'duplicate' row from a different batch that
+-- shares the same document_id).
+UPDATE batch_document_outcomes
+SET outcome = $2::batch_outcome_status,
+    error_detail = $3,
+    updated_at = NOW()
+WHERE document_id = $1
+  AND outcome IN ('submitted', 'init_complete', 'sync_complete');
+
+-- name: ResolveBatchDocumentOutcome :exec
+-- Called by docInitRunner to set the document_id on a previously-submitted
+-- outcome row and update the outcome to an init-stage result.
+-- Finds the row by batch_id + filename since document_id is NULL at this point.
+UPDATE batch_document_outcomes
+SET document_id = $3,
+    outcome = $4::batch_outcome_status,
+    error_detail = $5,
+    updated_at = NOW()
+WHERE batch_id = $1 AND filename = $2 AND document_id IS NULL;
+
+-- name: ListBatchDocumentOutcomes :many
+-- Returns all outcome rows for a batch, with clean failure detail
+-- joined from currentCleanEntries when applicable.
+SELECT
+    bdo.id,
+    bdo.batch_id,
+    bdo.filename,
+    bdo.outcome,
+    bdo.error_detail,
+    bdo.document_id,
+    bdo.created_at,
+    bdo.updated_at,
+    cce.fail as clean_fail
+FROM batch_document_outcomes bdo
+LEFT JOIN currentCleanEntries cce ON cce.documentId = bdo.document_id
+WHERE bdo.batch_id = $1
+ORDER BY bdo.created_at ASC;
@@ -282,10 +282,10 @@ func (q *Queries) GetDocumentsByBatchId(ctx context.Context, batchID *uuid.UUID)
 }

 const getUnprocessedBatches = `-- name: GetUnprocessedBatches :many
-SELECT id, client_id, original_filename, total_documents, 
+SELECT id, client_id, original_filename, total_documents,
       processed_documents, failed_documents, invalid_type_documents,
       status, progress_percent, created_at, completed_at
-FROM batch_uploads 
+FROM batch_uploads
 WHERE status = 'processing'
 ORDER BY created_at ASC
 `
@@ -344,6 +344,121 @@ func (q *Queries) GetUnprocessedBatches(ctx context.Context) ([]*GetUnprocessedB
 	return items, nil
 }

+const insertBatchDocumentOutcome = `-- name: InsertBatchDocumentOutcome :exec
+INSERT INTO batch_document_outcomes (batch_id, filename, outcome, error_detail, document_id)
+VALUES ($1, $2, $3::batch_outcome_status, $4, $5)
+ON CONFLICT (batch_id, filename) DO UPDATE
+SET outcome = EXCLUDED.outcome,
+    error_detail = EXCLUDED.error_detail,
+    document_id = EXCLUDED.document_id,
+    updated_at = NOW()
+`
+
+type InsertBatchDocumentOutcomeParams struct {
+	BatchID     uuid.UUID          `db:"batch_id"`
+	Filename    string             `db:"filename"`
+	Column3     BatchOutcomeStatus `db:"column_3"`
+	ErrorDetail *string            `db:"error_detail"`
+	DocumentID  *uuid.UUID         `db:"document_id"`
+}
+
+// Upsert: if a row for this (batch_id, filename) already exists (e.g. batch
+// retry after partial failure), overwrite it with the latest outcome.
+//
+//	INSERT INTO batch_document_outcomes (batch_id, filename, outcome, error_detail, document_id)
+//	VALUES ($1, $2, $3::batch_outcome_status, $4, $5)
+//	ON CONFLICT (batch_id, filename) DO UPDATE
+//	SET outcome = EXCLUDED.outcome,
+//	    error_detail = EXCLUDED.error_detail,
+//	    document_id = EXCLUDED.document_id,
+//	    updated_at = NOW()
+func (q *Queries) InsertBatchDocumentOutcome(ctx context.Context, arg *InsertBatchDocumentOutcomeParams) error {
+	_, err := q.db.Exec(ctx, insertBatchDocumentOutcome,
+		arg.BatchID,
+		arg.Filename,
+		arg.Column3,
+		arg.ErrorDetail,
+		arg.DocumentID,
+	)
+	return err
+}
+
+const listBatchDocumentOutcomes = `-- name: ListBatchDocumentOutcomes :many
+SELECT
+    bdo.id,
+    bdo.batch_id,
+    bdo.filename,
+    bdo.outcome,
+    bdo.error_detail,
+    bdo.document_id,
+    bdo.created_at,
+    bdo.updated_at,
+    cce.fail as clean_fail
+FROM batch_document_outcomes bdo
+LEFT JOIN currentCleanEntries cce ON cce.documentId = bdo.document_id
+WHERE bdo.batch_id = $1
+ORDER BY bdo.created_at ASC
+`
+
+type ListBatchDocumentOutcomesRow struct {
+	ID          uuid.UUID          `db:"id"`
+	BatchID     uuid.UUID          `db:"batch_id"`
+	Filename    string             `db:"filename"`
+	Outcome     BatchOutcomeStatus `db:"outcome"`
+	ErrorDetail *string            `db:"error_detail"`
+	DocumentID  *uuid.UUID         `db:"document_id"`
+	CreatedAt   pgtype.Timestamp   `db:"created_at"`
+	UpdatedAt   pgtype.Timestamp   `db:"updated_at"`
+	CleanFail   NullCleanfailtype  `db:"clean_fail"`
+}
+
+// Returns all outcome rows for a batch, with clean failure detail
+// joined from currentCleanEntries when applicable.
+//
+//	SELECT
+//	    bdo.id,
+//	    bdo.batch_id,
+//	    bdo.filename,
+//	    bdo.outcome,
+//	    bdo.error_detail,
+//	    bdo.document_id,
+//	    bdo.created_at,
+//	    bdo.updated_at,
+//	    cce.fail as clean_fail
+//	FROM batch_document_outcomes bdo
+//	LEFT JOIN currentCleanEntries cce ON cce.documentId = bdo.document_id
+//	WHERE bdo.batch_id = $1
+//	ORDER BY bdo.created_at ASC
+func (q *Queries) ListBatchDocumentOutcomes(ctx context.Context, batchID uuid.UUID) ([]*ListBatchDocumentOutcomesRow, error) {
+	rows, err := q.db.Query(ctx, listBatchDocumentOutcomes, batchID)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	items := []*ListBatchDocumentOutcomesRow{}
+	for rows.Next() {
+		var i ListBatchDocumentOutcomesRow
+		if err := rows.Scan(
+			&i.ID,
+			&i.BatchID,
+			&i.Filename,
+			&i.Outcome,
+			&i.ErrorDetail,
+			&i.DocumentID,
+			&i.CreatedAt,
+			&i.UpdatedAt,
+			&i.CleanFail,
+		); err != nil {
+			return nil, err
+		}
+		items = append(items, &i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
 const listBatchUploads = `-- name: ListBatchUploads :many
 SELECT id, client_id, original_filename, total_documents, processed_documents, 
       failed_documents, invalid_type_documents, status, progress_percent,
@@ -415,6 +530,75 @@ func (q *Queries) ListBatchUploads(ctx context.Context, arg *ListBatchUploadsPar
 	return items, nil
 }

+const resolveBatchDocumentOutcome = `-- name: ResolveBatchDocumentOutcome :exec
+UPDATE batch_document_outcomes
+SET document_id = $3,
+    outcome = $4::batch_outcome_status,
+    error_detail = $5,
+    updated_at = NOW()
+WHERE batch_id = $1 AND filename = $2 AND document_id IS NULL
+`
+
+type ResolveBatchDocumentOutcomeParams struct {
+	BatchID     uuid.UUID          `db:"batch_id"`
+	Filename    string             `db:"filename"`
+	DocumentID  *uuid.UUID         `db:"document_id"`
+	Column4     BatchOutcomeStatus `db:"column_4"`
+	ErrorDetail *string            `db:"error_detail"`
+}
+
+// Called by docInitRunner to set the document_id on a previously-submitted
+// outcome row and update the outcome to an init-stage result.
+// Finds the row by batch_id + filename since document_id is NULL at this point.
+//
+//	UPDATE batch_document_outcomes
+//	SET document_id = $3,
+//	    outcome = $4::batch_outcome_status,
+//	    error_detail = $5,
+//	    updated_at = NOW()
+//	WHERE batch_id = $1 AND filename = $2 AND document_id IS NULL
+func (q *Queries) ResolveBatchDocumentOutcome(ctx context.Context, arg *ResolveBatchDocumentOutcomeParams) error {
+	_, err := q.db.Exec(ctx, resolveBatchDocumentOutcome,
+		arg.BatchID,
+		arg.Filename,
+		arg.DocumentID,
+		arg.Column4,
+		arg.ErrorDetail,
+	)
+	return err
+}
+
+const updateBatchDocumentOutcomeByDocumentID = `-- name: UpdateBatchDocumentOutcomeByDocumentID :exec
+UPDATE batch_document_outcomes
+SET outcome = $2::batch_outcome_status,
+    error_detail = $3,
+    updated_at = NOW()
+WHERE document_id = $1
+  AND outcome IN ('submitted', 'init_complete', 'sync_complete')
+`
+
+type UpdateBatchDocumentOutcomeByDocumentIDParams struct {
+	DocumentID  *uuid.UUID         `db:"document_id"`
+	Column2     BatchOutcomeStatus `db:"column_2"`
+	ErrorDetail *string            `db:"error_detail"`
+}
+
+// Updates the outcome for a document progressing through the pipeline.
+// Only updates rows in non-terminal pipeline states to prevent overwriting
+// terminal outcomes (e.g. a 'duplicate' row from a different batch that
+// shares the same document_id).
+//
+//	UPDATE batch_document_outcomes
+//	SET outcome = $2::batch_outcome_status,
+//	    error_detail = $3,
+//	    updated_at = NOW()
+//	WHERE document_id = $1
+//	  AND outcome IN ('submitted', 'init_complete', 'sync_complete')
+func (q *Queries) UpdateBatchDocumentOutcomeByDocumentID(ctx context.Context, arg *UpdateBatchDocumentOutcomeByDocumentIDParams) error {
+	_, err := q.db.Exec(ctx, updateBatchDocumentOutcomeByDocumentID, arg.DocumentID, arg.Column2, arg.ErrorDetail)
+	return err
+}
+
 const updateBatchProgress = `-- name: UpdateBatchProgress :exec
 UPDATE batch_uploads 
 SET processed_documents = $2, 
@@ -1,6 +1,6 @@
 // Code generated by sqlc. DO NOT EDIT.
 // versions:
-//   sqlc v1.30.0
+//   sqlc v1.27.0

 package repository

@@ -12,6 +12,79 @@ import (
 	"github.com/jackc/pgx/v5/pgtype"
 )

+type BatchOutcomeStatus string
+
+const (
+	BatchOutcomeStatusSubmitted      BatchOutcomeStatus = "submitted"
+	BatchOutcomeStatusDuplicate      BatchOutcomeStatus = "duplicate"
+	BatchOutcomeStatusFailedOpen     BatchOutcomeStatus = "failed_open"
+	BatchOutcomeStatusFailedRead     BatchOutcomeStatus = "failed_read"
+	BatchOutcomeStatusFailedS3Upload BatchOutcomeStatus = "failed_s3_upload"
+	BatchOutcomeStatusFailedUpload   BatchOutcomeStatus = "failed_upload"
+	BatchOutcomeStatusInvalidType    BatchOutcomeStatus = "invalid_type"
+	BatchOutcomeStatusInitComplete   BatchOutcomeStatus = "init_complete"
+	BatchOutcomeStatusInitDuplicate  BatchOutcomeStatus = "init_duplicate"
+	BatchOutcomeStatusSyncComplete   BatchOutcomeStatus = "sync_complete"
+	BatchOutcomeStatusSyncSkipped    BatchOutcomeStatus = "sync_skipped"
+	BatchOutcomeStatusCleanPassed    BatchOutcomeStatus = "clean_passed"
+	BatchOutcomeStatusCleanFailed    BatchOutcomeStatus = "clean_failed"
+)
+
+func (e *BatchOutcomeStatus) Scan(src interface{}) error {
+	switch s := src.(type) {
+	case []byte:
+		*e = BatchOutcomeStatus(s)
+	case string:
+		*e = BatchOutcomeStatus(s)
+	default:
+		return fmt.Errorf("unsupported scan type for BatchOutcomeStatus: %T", src)
+	}
+	return nil
+}
+
+type NullBatchOutcomeStatus struct {
+	BatchOutcomeStatus BatchOutcomeStatus
+	Valid              bool // Valid is true if BatchOutcomeStatus is not NULL
+}
+
+// Scan implements the Scanner interface.
+func (ns *NullBatchOutcomeStatus) Scan(value interface{}) error {
+	if value == nil {
+		ns.BatchOutcomeStatus, ns.Valid = "", false
+		return nil
+	}
+	ns.Valid = true
+	return ns.BatchOutcomeStatus.Scan(value)
+}
+
+// Value implements the driver Valuer interface.
+func (ns NullBatchOutcomeStatus) Value() (driver.Value, error) {
+	if !ns.Valid {
+		return nil, nil
+	}
+	return string(ns.BatchOutcomeStatus), nil
+}
+
+func (e BatchOutcomeStatus) Valid() bool {
+	switch e {
+	case BatchOutcomeStatusSubmitted,
+		BatchOutcomeStatusDuplicate,
+		BatchOutcomeStatusFailedOpen,
+		BatchOutcomeStatusFailedRead,
+		BatchOutcomeStatusFailedS3Upload,
+		BatchOutcomeStatusFailedUpload,
+		BatchOutcomeStatusInvalidType,
+		BatchOutcomeStatusInitComplete,
+		BatchOutcomeStatusInitDuplicate,
+		BatchOutcomeStatusSyncComplete,
+		BatchOutcomeStatusSyncSkipped,
+		BatchOutcomeStatusCleanPassed,
+		BatchOutcomeStatusCleanFailed:
+		return true
+	}
+	return false
+}
+
 type BatchStatus string

 const (
@@ -181,6 +254,17 @@ func (e Cleanmimetype) Valid() bool {
 	return false
 }

+type BatchDocumentOutcome struct {
+	ID          uuid.UUID          `db:"id"`
+	BatchID     uuid.UUID          `db:"batch_id"`
+	Filename    string             `db:"filename"`
+	Outcome     BatchOutcomeStatus `db:"outcome"`
+	ErrorDetail *string            `db:"error_detail"`
+	DocumentID  *uuid.UUID         `db:"document_id"`
+	UpdatedAt   pgtype.Timestamp   `db:"updated_at"`
+	CreatedAt   pgtype.Timestamp   `db:"created_at"`
+}
+
 type BatchUpload struct {
 	ID                   uuid.UUID        `db:"id"`
 	ClientID             string           `db:"client_id"`
@@ -1,6 +1,6 @@
 // Code generated by sqlc. DO NOT EDIT.
 // versions:
-//   sqlc v1.30.0
+//   sqlc v1.27.0
 // source: uisettings.sql

 package repository