jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in jmathison/v2-upload-batch (pull request #224)

Browse Source 
Implement v2 upload batch cleanup

* Implement v2 upload batch cleanup

* Merge remote-tracking branch 'origin/main' into jmathison/v2-upload-batch

* Address upload batch review feedback

* Raise batch worker coverage

* Fix batch cleanup review issues


Approved-by: Jay Brown
This commit is contained in:
Jacob Mathison
2026-05-08 21:42:46 +00:00
parent 7b820b18c2
commit a080ca59d8
35 changed files with 2587 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/database/queries/batch.sql
+150 -3
View File
@@ -55,6 +55,11 @@ SET status = $2::batch_status,
completed_at = CASE WHEN $2::batch_status IN ('completed', 'failed', 'cancelled') THEN NOW() ELSE NULL END
WHERE id = $1;
-- name: SetBatchCompletedAt :exec
UPDATE batch_uploads
SET completed_at = @completed_at
WHERE id = @id;
-- name: AddFailedFilename :exec
UPDATE batch_uploads
SET failed_filenames = failed_filenames || jsonb_build_array($2::text)
@@ -89,7 +94,7 @@ SET outcome = EXCLUDED.outcome,
document_id = EXCLUDED.document_id,
updated_at = NOW();
-- name: UpdateBatchDocumentOutcomeByDocumentID :exec
-- name: UpdateBatchDocumentOutcomeByDocumentID :many
-- Updates the outcome for a document progressing through the pipeline.
-- Only updates rows in non-terminal pipeline states to prevent overwriting
-- terminal outcomes (e.g. a 'duplicate' row from a different batch that
@@ -99,7 +104,8 @@ SET outcome = $2::batch_outcome_status,
error_detail = $3,
updated_at = NOW()
WHERE document_id = $1
AND outcome IN ('submitted', 'init_complete', 'sync_complete');
AND outcome IN ('submitted', 'init_complete', 'sync_complete')
RETURNING batch_id;
-- name: ResolveBatchDocumentOutcome :exec
-- Called by docInitRunner to set the document_id on a previously-submitted
@@ -128,4 +134,145 @@ SELECT
FROM batch_document_outcomes bdo
LEFT JOIN currentCleanEntries cce ON cce.documentId = bdo.document_id
WHERE bdo.batch_id = $1
ORDER BY bdo.created_at ASC;
ORDER BY bdo.created_at ASC;
-- name: RecordBatchFolderCandidate :exec
-- Records a non-root folder touched by a batch upload. Once a batch observes a
-- folder as newly created, retries must preserve that fact.
INSERT INTO batch_folder_candidates (batch_id, folder_id, path, existed_before)
VALUES (@batch_id, @folder_id, @path, @existed_before)
ON CONFLICT (batch_id, folder_id) DO UPDATE
SET path = EXCLUDED.path,
existed_before = batch_folder_candidates.existed_before AND EXCLUDED.existed_before;
-- name: ListBatchFolderCandidates :many
SELECT batch_id, folder_id, path, existed_before, created_at
FROM batch_folder_candidates
WHERE batch_id = @batch_id
ORDER BY path;
-- name: LockBatchUploadForFolderCleanup :one
SELECT id, status, total_documents, created_at, completed_at, folder_cleanup_completed_at
FROM batch_uploads
WHERE id = @batch_id
FOR UPDATE;
-- name: CountBatchOutcomes :one
SELECT COUNT(*)::int
FROM batch_document_outcomes
WHERE batch_id = @batch_id;
-- name: CountNonTerminalBatchOutcomes :one
SELECT COUNT(*)::int
FROM batch_document_outcomes
WHERE batch_id = @batch_id
AND outcome IN ('submitted', 'init_complete', 'sync_complete');
-- name: CountAcceptedBatchOutcomes :one
SELECT COUNT(*)::int
FROM batch_document_outcomes
WHERE batch_id = @batch_id
AND outcome IN ('clean_passed', 'sync_skipped');
-- name: ListFolderCleanupReadyBatchIDs :many
-- @sqlc-vet-disable
SELECT bu.id
FROM batch_uploads bu
WHERE bu.folder_cleanup_completed_at IS NULL
AND bu.status IN ('completed', 'failed', 'cancelled')
AND EXISTS (
SELECT 1
FROM batch_folder_candidates bfc
WHERE bfc.batch_id = bu.id
)
AND (
(
bu.total_documents > 0
AND
(SELECT COUNT(*) FROM batch_document_outcomes bdo WHERE bdo.batch_id = bu.id) = bu.total_documents
AND NOT EXISTS (
SELECT 1
FROM batch_document_outcomes bdo
WHERE bdo.batch_id = bu.id
AND bdo.outcome IN ('submitted', 'init_complete', 'sync_complete')
)
)
OR (
bu.status IN ('failed', 'cancelled')
AND (SELECT COUNT(*) FROM batch_document_outcomes bdo WHERE bdo.batch_id = bu.id) > 0
AND bu.completed_at IS NOT NULL
AND bu.completed_at <= NOW() - (sqlc.arg(stale_after_seconds)::int * INTERVAL '1 second')
)
)
ORDER BY bu.created_at ASC
LIMIT sqlc.arg(limit_count);
-- name: DetachRejectedBatchDocumentsFromCandidateFolders :execrows
-- @sqlc-vet-disable
UPDATE documents d
SET folderId = NULL
WHERE d.batch_id = @batch_id
AND d.folderId IS NOT NULL
AND EXISTS (
SELECT 1
FROM batch_folder_candidates bfc
WHERE bfc.batch_id = @batch_id
AND bfc.folder_id = d.folderId
)
AND EXISTS (
SELECT 1
FROM batch_document_outcomes bdo
WHERE bdo.batch_id = @batch_id
AND bdo.document_id = d.id
AND (
bdo.outcome IN (
'duplicate',
'init_duplicate',
'invalid_type',
'failed_open',
'failed_read',
'failed_s3_upload',
'failed_upload',
'clean_failed'
)
OR (@include_nonterminal::boolean AND bdo.outcome IN ('submitted', 'init_complete', 'sync_complete'))
)
);
-- name: DetachRejectedBatchDocumentUploadsFromCandidateFolders :execrows
-- @sqlc-vet-disable
UPDATE documentUploads du
SET folder_id = NULL
WHERE du.batch_id = @batch_id
AND du.folder_id IS NOT NULL
AND EXISTS (
SELECT 1
FROM batch_folder_candidates bfc
WHERE bfc.batch_id = @batch_id
AND bfc.folder_id = du.folder_id
)
AND EXISTS (
SELECT 1
FROM batch_document_outcomes bdo
WHERE bdo.batch_id = @batch_id
AND bdo.filename = du.filename
AND (
bdo.outcome IN (
'duplicate',
'init_duplicate',
'invalid_type',
'failed_open',
'failed_read',
'failed_s3_upload',
'failed_upload',
'clean_failed'
)
OR (@include_nonterminal::boolean AND bdo.outcome IN ('submitted', 'init_complete', 'sync_complete'))
)
);
-- name: MarkBatchFolderCleanupCompleted :exec
UPDATE batch_uploads
SET folder_cleanup_completed_at = NOW()
WHERE id = @batch_id
AND folder_cleanup_completed_at IS NULL;