Merged in jmathison/v2-upload-batch (pull request #224)

Implement v2 upload batch cleanup

* Implement v2 upload batch cleanup

* Merge remote-tracking branch 'origin/main' into jmathison/v2-upload-batch

* Address upload batch review feedback

* Raise batch worker coverage

* Fix batch cleanup review issues


Approved-by: Jay Brown

internal/database/migrations/00000000000132_batch_folder_candidates.down.sql

@@ -0,0 +1,8 @@
+DROP INDEX IF EXISTS idx_batch_uploads_folder_cleanup_pending;
+
+ALTER TABLE batch_uploads
+DROP COLUMN IF EXISTS folder_cleanup_completed_at;
+
+DROP INDEX IF EXISTS idx_batch_folder_candidates_folder_id;
+DROP INDEX IF EXISTS idx_batch_folder_candidates_batch_id;
+DROP TABLE IF EXISTS batch_folder_candidates;

internal/database/migrations/00000000000132_batch_folder_candidates.up.sql

@@ -0,0 +1,18 @@
+CREATE TABLE batch_folder_candidates (
+    batch_id uuid NOT NULL REFERENCES batch_uploads(id) ON DELETE CASCADE,
+    folder_id uuid NOT NULL REFERENCES folders(id) ON DELETE CASCADE,
+    path text NOT NULL,
+    existed_before boolean NOT NULL,
+    created_at timestamp NOT NULL DEFAULT NOW(),
+    PRIMARY KEY (batch_id, folder_id)
+);
+
+CREATE INDEX idx_batch_folder_candidates_batch_id ON batch_folder_candidates(batch_id);
+CREATE INDEX idx_batch_folder_candidates_folder_id ON batch_folder_candidates(folder_id);
+
+ALTER TABLE batch_uploads
+ADD COLUMN folder_cleanup_completed_at timestamp;
+
+CREATE INDEX idx_batch_uploads_folder_cleanup_pending
+ON batch_uploads(created_at)
+WHERE folder_cleanup_completed_at IS NULL;

internal/database/queries/batch.sql

@@ -55,6 +55,11 @@ SET status = $2::batch_status,
     completed_at = CASE WHEN $2::batch_status IN ('completed', 'failed', 'cancelled') THEN NOW() ELSE NULL END
 WHERE id = $1;
 
+-- name: SetBatchCompletedAt :exec
+UPDATE batch_uploads
+SET completed_at = @completed_at
+WHERE id = @id;
+
 -- name: AddFailedFilename :exec
 UPDATE batch_uploads 
 SET failed_filenames = failed_filenames || jsonb_build_array($2::text)
@@ -89,7 +94,7 @@ SET outcome = EXCLUDED.outcome,
     document_id = EXCLUDED.document_id,
     updated_at = NOW();
 
--- name: UpdateBatchDocumentOutcomeByDocumentID :exec
+-- name: UpdateBatchDocumentOutcomeByDocumentID :many
 -- Updates the outcome for a document progressing through the pipeline.
 -- Only updates rows in non-terminal pipeline states to prevent overwriting
 -- terminal outcomes (e.g. a 'duplicate' row from a different batch that
@@ -99,7 +104,8 @@ SET outcome = $2::batch_outcome_status,
     error_detail = $3,
     updated_at = NOW()
 WHERE document_id = $1
-  AND outcome IN ('submitted', 'init_complete', 'sync_complete');
+  AND outcome IN ('submitted', 'init_complete', 'sync_complete')
+RETURNING batch_id;
 
 -- name: ResolveBatchDocumentOutcome :exec
 -- Called by docInitRunner to set the document_id on a previously-submitted
@@ -128,4 +134,145 @@ SELECT
 FROM batch_document_outcomes bdo
 LEFT JOIN currentCleanEntries cce ON cce.documentId = bdo.document_id
 WHERE bdo.batch_id = $1
-ORDER BY bdo.created_at ASC;
+ORDER BY bdo.created_at ASC;
+
+-- name: RecordBatchFolderCandidate :exec
+-- Records a non-root folder touched by a batch upload. Once a batch observes a
+-- folder as newly created, retries must preserve that fact.
+INSERT INTO batch_folder_candidates (batch_id, folder_id, path, existed_before)
+VALUES (@batch_id, @folder_id, @path, @existed_before)
+ON CONFLICT (batch_id, folder_id) DO UPDATE
+SET path = EXCLUDED.path,
+    existed_before = batch_folder_candidates.existed_before AND EXCLUDED.existed_before;
+
+-- name: ListBatchFolderCandidates :many
+SELECT batch_id, folder_id, path, existed_before, created_at
+FROM batch_folder_candidates
+WHERE batch_id = @batch_id
+ORDER BY path;
+
+-- name: LockBatchUploadForFolderCleanup :one
+SELECT id, status, total_documents, created_at, completed_at, folder_cleanup_completed_at
+FROM batch_uploads
+WHERE id = @batch_id
+FOR UPDATE;
+
+-- name: CountBatchOutcomes :one
+SELECT COUNT(*)::int
+FROM batch_document_outcomes
+WHERE batch_id = @batch_id;
+
+-- name: CountNonTerminalBatchOutcomes :one
+SELECT COUNT(*)::int
+FROM batch_document_outcomes
+WHERE batch_id = @batch_id
+  AND outcome IN ('submitted', 'init_complete', 'sync_complete');
+
+-- name: CountAcceptedBatchOutcomes :one
+SELECT COUNT(*)::int
+FROM batch_document_outcomes
+WHERE batch_id = @batch_id
+  AND outcome IN ('clean_passed', 'sync_skipped');
+
+-- name: ListFolderCleanupReadyBatchIDs :many
+-- @sqlc-vet-disable
+SELECT bu.id
+FROM batch_uploads bu
+WHERE bu.folder_cleanup_completed_at IS NULL
+  AND bu.status IN ('completed', 'failed', 'cancelled')
+  AND EXISTS (
+      SELECT 1
+      FROM batch_folder_candidates bfc
+      WHERE bfc.batch_id = bu.id
+  )
+  AND (
+      (
+          bu.total_documents > 0
+          AND
+          (SELECT COUNT(*) FROM batch_document_outcomes bdo WHERE bdo.batch_id = bu.id) = bu.total_documents
+          AND NOT EXISTS (
+              SELECT 1
+              FROM batch_document_outcomes bdo
+              WHERE bdo.batch_id = bu.id
+                AND bdo.outcome IN ('submitted', 'init_complete', 'sync_complete')
+          )
+      )
+      OR (
+          bu.status IN ('failed', 'cancelled')
+          AND (SELECT COUNT(*) FROM batch_document_outcomes bdo WHERE bdo.batch_id = bu.id) > 0
+          AND bu.completed_at IS NOT NULL
+          AND bu.completed_at <= NOW() - (sqlc.arg(stale_after_seconds)::int * INTERVAL '1 second')
+      )
+  )
+ORDER BY bu.created_at ASC
+LIMIT sqlc.arg(limit_count);
+
+-- name: DetachRejectedBatchDocumentsFromCandidateFolders :execrows
+-- @sqlc-vet-disable
+UPDATE documents d
+SET folderId = NULL
+WHERE d.batch_id = @batch_id
+  AND d.folderId IS NOT NULL
+  AND EXISTS (
+      SELECT 1
+      FROM batch_folder_candidates bfc
+      WHERE bfc.batch_id = @batch_id
+        AND bfc.folder_id = d.folderId
+  )
+  AND EXISTS (
+      SELECT 1
+      FROM batch_document_outcomes bdo
+      WHERE bdo.batch_id = @batch_id
+        AND bdo.document_id = d.id
+        AND (
+            bdo.outcome IN (
+                'duplicate',
+                'init_duplicate',
+                'invalid_type',
+                'failed_open',
+                'failed_read',
+                'failed_s3_upload',
+                'failed_upload',
+                'clean_failed'
+            )
+            OR (@include_nonterminal::boolean AND bdo.outcome IN ('submitted', 'init_complete', 'sync_complete'))
+        )
+  );
+
+-- name: DetachRejectedBatchDocumentUploadsFromCandidateFolders :execrows
+-- @sqlc-vet-disable
+UPDATE documentUploads du
+SET folder_id = NULL
+WHERE du.batch_id = @batch_id
+  AND du.folder_id IS NOT NULL
+  AND EXISTS (
+      SELECT 1
+      FROM batch_folder_candidates bfc
+      WHERE bfc.batch_id = @batch_id
+        AND bfc.folder_id = du.folder_id
+  )
+  AND EXISTS (
+      SELECT 1
+      FROM batch_document_outcomes bdo
+      WHERE bdo.batch_id = @batch_id
+        AND bdo.filename = du.filename
+        AND (
+            bdo.outcome IN (
+                'duplicate',
+                'init_duplicate',
+                'invalid_type',
+                'failed_open',
+                'failed_read',
+                'failed_s3_upload',
+                'failed_upload',
+                'clean_failed'
+            )
+            OR (@include_nonterminal::boolean AND bdo.outcome IN ('submitted', 'init_complete', 'sync_complete'))
+        )
+  );
+
+-- name: MarkBatchFolderCleanupCompleted :exec
+UPDATE batch_uploads
+SET folder_cleanup_completed_at = NOW()
+WHERE id = @batch_id
+  AND folder_cleanup_completed_at IS NULL;

internal/database/queries/eula.sql

@@ -108,3 +108,30 @@ FROM eulaAgreements
 WHERE eulaVersionId = $1
 ORDER BY agreedAt DESC
 LIMIT $2 OFFSET $3;
+
+-- name: ActivateLocalDevEulaVersion :one
+-- Activates or updates the local development EULA version and makes it current.
+WITH cleared AS (
+    UPDATE eulaVersions
+    SET isCurrent = FALSE
+    WHERE isCurrent = TRUE
+),
+upserted AS (
+    INSERT INTO eulaVersions (version, title, content, effectiveDate, createdBy, isCurrent, activatedAt, activatedBy)
+    VALUES (@version, @title, @content, NOW(), 'local-dev', TRUE, NOW(), 'local-dev')
+    ON CONFLICT (version) DO UPDATE
+    SET title = EXCLUDED.title,
+        content = EXCLUDED.content,
+        effectiveDate = EXCLUDED.effectiveDate,
+        isCurrent = TRUE,
+        activatedAt = NOW(),
+        activatedBy = EXCLUDED.activatedBy
+    RETURNING id
+)
+SELECT id FROM upserted;
+
+-- name: CreateLocalDevEulaAgreement :exec
+-- Records an idempotent local development EULA agreement.
+INSERT INTO eulaAgreements (cognitoSubjectId, userEmail, eulaVersionId, agreedFromIp)
+VALUES (@cognito_subject_id, @user_email, @eula_version_id, '127.0.0.1')
+ON CONFLICT (cognitoSubjectId, eulaVersionId) DO NOTHING;

internal/database/queries/folders.sql

@@ -88,6 +88,15 @@ VALUES ($1, $2, $3, $4)
 ON CONFLICT (clientId, path) DO UPDATE SET path = EXCLUDED.path
 RETURNING *;
 
+-- name: CreateFolderIfAbsent :one
+-- Insert a folder only when it does not already exist.
+-- Batch uploads use this to distinguish newly-created folder paths from
+-- pre-existing paths without mutating existing folders.
+INSERT INTO folders (path, parentId, clientId, createdBy)
+VALUES (@path, @parent_id, @client_id, @created_by)
+ON CONFLICT (clientId, path) DO NOTHING
+RETURNING *;
+
 -- name: GetDocumentsByFolderEnriched :many
 -- Gets documents in a folder with all enriched metadata fields
 SELECT
@@ -180,3 +189,35 @@ WITH RECURSIVE folder_tree AS (
     JOIN folder_tree ft ON f.parentId = ft.id
 )
 DELETE FROM folders WHERE id IN (SELECT id FROM folder_tree);
+
+-- name: ListSafeAutoCreatedFolderCandidates :many
+-- @sqlc-vet-disable
+-- Candidate folders can be deleted only after every batch that touched that
+-- folder has finished folder cleanup. The path equality check protects renamed
+-- folders from deletion.
+SELECT DISTINCT f.id, f.path
+FROM folders f
+JOIN batch_folder_candidates bfc ON bfc.folder_id = f.id
+WHERE bfc.existed_before = false
+  AND bfc.path = f.path
+  AND f.path <> '/'
+  AND NOT EXISTS (
+      SELECT 1
+      FROM batch_folder_candidates other_bfc
+      JOIN batch_uploads bu ON bu.id = other_bfc.batch_id
+      WHERE other_bfc.folder_id = f.id
+        AND bu.folder_cleanup_completed_at IS NULL
+  )
+ORDER BY f.path DESC
+LIMIT sqlc.arg(limit_count);
+
+-- name: DeleteSafeAutoCreatedFolder :execrows
+-- @sqlc-vet-disable
+-- Delete one auto-created folder only when it is currently empty and no longer
+-- referenced by documents, uploads, or child folders.
+DELETE FROM folders f
+WHERE f.id = @folder_id
+  AND f.path <> '/'
+  AND NOT EXISTS (SELECT 1 FROM documents d WHERE d.folderId = f.id)
+  AND NOT EXISTS (SELECT 1 FROM documentUploads du WHERE du.folder_id = f.id)
+  AND NOT EXISTS (SELECT 1 FROM folders child WHERE child.parentId = f.id);

internal/database/repository/batch.sql.go

@@ -33,6 +33,44 @@ func (q *Queries) AddFailedFilename(ctx context.Context, arg *AddFailedFilenameP
 	return err
 }
 
+const countAcceptedBatchOutcomes = `-- name: CountAcceptedBatchOutcomes :one
+SELECT COUNT(*)::int
+FROM batch_document_outcomes
+WHERE batch_id = $1
+  AND outcome IN ('clean_passed', 'sync_skipped')
+`
+
+// CountAcceptedBatchOutcomes
+//
+//	SELECT COUNT(*)::int
+//	FROM batch_document_outcomes
+//	WHERE batch_id = $1
+//	  AND outcome IN ('clean_passed', 'sync_skipped')
+func (q *Queries) CountAcceptedBatchOutcomes(ctx context.Context, batchID uuid.UUID) (int32, error) {
+	row := q.db.QueryRow(ctx, countAcceptedBatchOutcomes, batchID)
+	var column_1 int32
+	err := row.Scan(&column_1)
+	return column_1, err
+}
+
+const countBatchOutcomes = `-- name: CountBatchOutcomes :one
+SELECT COUNT(*)::int
+FROM batch_document_outcomes
+WHERE batch_id = $1
+`
+
+// CountBatchOutcomes
+//
+//	SELECT COUNT(*)::int
+//	FROM batch_document_outcomes
+//	WHERE batch_id = $1
+func (q *Queries) CountBatchOutcomes(ctx context.Context, batchID uuid.UUID) (int32, error) {
+	row := q.db.QueryRow(ctx, countBatchOutcomes, batchID)
+	var column_1 int32
+	err := row.Scan(&column_1)
+	return column_1, err
+}
+
 const countDocumentsByBatchId = `-- name: CountDocumentsByBatchId :one
 SELECT COUNT(*) as count
 FROM documents 
@@ -51,6 +89,26 @@ func (q *Queries) CountDocumentsByBatchId(ctx context.Context, batchID *uuid.UUI
 	return count, err
 }
 
+const countNonTerminalBatchOutcomes = `-- name: CountNonTerminalBatchOutcomes :one
+SELECT COUNT(*)::int
+FROM batch_document_outcomes
+WHERE batch_id = $1
+  AND outcome IN ('submitted', 'init_complete', 'sync_complete')
+`
+
+// CountNonTerminalBatchOutcomes
+//
+//	SELECT COUNT(*)::int
+//	FROM batch_document_outcomes
+//	WHERE batch_id = $1
+//	  AND outcome IN ('submitted', 'init_complete', 'sync_complete')
+func (q *Queries) CountNonTerminalBatchOutcomes(ctx context.Context, batchID uuid.UUID) (int32, error) {
+	row := q.db.QueryRow(ctx, countNonTerminalBatchOutcomes, batchID)
+	var column_1 int32
+	err := row.Scan(&column_1)
+	return column_1, err
+}
+
 const createBatchUpload = `-- name: CreateBatchUpload :one
 INSERT INTO batch_uploads (client_id, original_filename, total_documents)
 VALUES ($1, $2, $3) RETURNING id
@@ -128,6 +186,158 @@ func (q *Queries) CreateBatchUploadWithStorage(ctx context.Context, arg *CreateB
 	return &i, err
 }
 
+const detachRejectedBatchDocumentUploadsFromCandidateFolders = `-- name: DetachRejectedBatchDocumentUploadsFromCandidateFolders :execrows
+UPDATE documentUploads du
+SET folder_id = NULL
+WHERE du.batch_id = $1
+  AND du.folder_id IS NOT NULL
+  AND EXISTS (
+      SELECT 1
+      FROM batch_folder_candidates bfc
+      WHERE bfc.batch_id = $1
+        AND bfc.folder_id = du.folder_id
+  )
+  AND EXISTS (
+      SELECT 1
+      FROM batch_document_outcomes bdo
+      WHERE bdo.batch_id = $1
+        AND bdo.filename = du.filename
+        AND (
+            bdo.outcome IN (
+                'duplicate',
+                'init_duplicate',
+                'invalid_type',
+                'failed_open',
+                'failed_read',
+                'failed_s3_upload',
+                'failed_upload',
+                'clean_failed'
+            )
+            OR ($2::boolean AND bdo.outcome IN ('submitted', 'init_complete', 'sync_complete'))
+        )
+  )
+`
+
+type DetachRejectedBatchDocumentUploadsFromCandidateFoldersParams struct {
+	BatchID            *uuid.UUID `db:"batch_id"`
+	IncludeNonterminal bool       `db:"include_nonterminal"`
+}
+
+// @sqlc-vet-disable
+//
+//	UPDATE documentUploads du
+//	SET folder_id = NULL
+//	WHERE du.batch_id = $1
+//	  AND du.folder_id IS NOT NULL
+//	  AND EXISTS (
+//	      SELECT 1
+//	      FROM batch_folder_candidates bfc
+//	      WHERE bfc.batch_id = $1
+//	        AND bfc.folder_id = du.folder_id
+//	  )
+//	  AND EXISTS (
+//	      SELECT 1
+//	      FROM batch_document_outcomes bdo
+//	      WHERE bdo.batch_id = $1
+//	        AND bdo.filename = du.filename
+//	        AND (
+//	            bdo.outcome IN (
+//	                'duplicate',
+//	                'init_duplicate',
+//	                'invalid_type',
+//	                'failed_open',
+//	                'failed_read',
+//	                'failed_s3_upload',
+//	                'failed_upload',
+//	                'clean_failed'
+//	            )
+//	            OR ($2::boolean AND bdo.outcome IN ('submitted', 'init_complete', 'sync_complete'))
+//	        )
+//	  )
+func (q *Queries) DetachRejectedBatchDocumentUploadsFromCandidateFolders(ctx context.Context, arg *DetachRejectedBatchDocumentUploadsFromCandidateFoldersParams) (int64, error) {
+	result, err := q.db.Exec(ctx, detachRejectedBatchDocumentUploadsFromCandidateFolders, arg.BatchID, arg.IncludeNonterminal)
+	if err != nil {
+		return 0, err
+	}
+	return result.RowsAffected(), nil
+}
+
+const detachRejectedBatchDocumentsFromCandidateFolders = `-- name: DetachRejectedBatchDocumentsFromCandidateFolders :execrows
+UPDATE documents d
+SET folderId = NULL
+WHERE d.batch_id = $1
+  AND d.folderId IS NOT NULL
+  AND EXISTS (
+      SELECT 1
+      FROM batch_folder_candidates bfc
+      WHERE bfc.batch_id = $1
+        AND bfc.folder_id = d.folderId
+  )
+  AND EXISTS (
+      SELECT 1
+      FROM batch_document_outcomes bdo
+      WHERE bdo.batch_id = $1
+        AND bdo.document_id = d.id
+        AND (
+            bdo.outcome IN (
+                'duplicate',
+                'init_duplicate',
+                'invalid_type',
+                'failed_open',
+                'failed_read',
+                'failed_s3_upload',
+                'failed_upload',
+                'clean_failed'
+            )
+            OR ($2::boolean AND bdo.outcome IN ('submitted', 'init_complete', 'sync_complete'))
+        )
+  )
+`
+
+type DetachRejectedBatchDocumentsFromCandidateFoldersParams struct {
+	BatchID            *uuid.UUID `db:"batch_id"`
+	IncludeNonterminal bool       `db:"include_nonterminal"`
+}
+
+// @sqlc-vet-disable
+//
+//	UPDATE documents d
+//	SET folderId = NULL
+//	WHERE d.batch_id = $1
+//	  AND d.folderId IS NOT NULL
+//	  AND EXISTS (
+//	      SELECT 1
+//	      FROM batch_folder_candidates bfc
+//	      WHERE bfc.batch_id = $1
+//	        AND bfc.folder_id = d.folderId
+//	  )
+//	  AND EXISTS (
+//	      SELECT 1
+//	      FROM batch_document_outcomes bdo
+//	      WHERE bdo.batch_id = $1
+//	        AND bdo.document_id = d.id
+//	        AND (
+//	            bdo.outcome IN (
+//	                'duplicate',
+//	                'init_duplicate',
+//	                'invalid_type',
+//	                'failed_open',
+//	                'failed_read',
+//	                'failed_s3_upload',
+//	                'failed_upload',
+//	                'clean_failed'
+//	            )
+//	            OR ($2::boolean AND bdo.outcome IN ('submitted', 'init_complete', 'sync_complete'))
+//	        )
+//	  )
+func (q *Queries) DetachRejectedBatchDocumentsFromCandidateFolders(ctx context.Context, arg *DetachRejectedBatchDocumentsFromCandidateFoldersParams) (int64, error) {
+	result, err := q.db.Exec(ctx, detachRejectedBatchDocumentsFromCandidateFolders, arg.BatchID, arg.IncludeNonterminal)
+	if err != nil {
+		return 0, err
+	}
+	return result.RowsAffected(), nil
+}
+
 const getBatchUpload = `-- name: GetBatchUpload :one
 SELECT id, client_id, original_filename, total_documents, processed_documents, 
        failed_documents, invalid_type_documents, status, progress_percent,
@@ -459,6 +669,45 @@ func (q *Queries) ListBatchDocumentOutcomes(ctx context.Context, batchID uuid.UU
 	return items, nil
 }
 
+const listBatchFolderCandidates = `-- name: ListBatchFolderCandidates :many
+SELECT batch_id, folder_id, path, existed_before, created_at
+FROM batch_folder_candidates
+WHERE batch_id = $1
+ORDER BY path
+`
+
+// ListBatchFolderCandidates
+//
+//	SELECT batch_id, folder_id, path, existed_before, created_at
+//	FROM batch_folder_candidates
+//	WHERE batch_id = $1
+//	ORDER BY path
+func (q *Queries) ListBatchFolderCandidates(ctx context.Context, batchID uuid.UUID) ([]*BatchFolderCandidate, error) {
+	rows, err := q.db.Query(ctx, listBatchFolderCandidates, batchID)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	items := []*BatchFolderCandidate{}
+	for rows.Next() {
+		var i BatchFolderCandidate
+		if err := rows.Scan(
+			&i.BatchID,
+			&i.FolderID,
+			&i.Path,
+			&i.ExistedBefore,
+			&i.CreatedAt,
+		); err != nil {
+			return nil, err
+		}
+		items = append(items, &i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
 const listBatchUploads = `-- name: ListBatchUploads :many
 SELECT id, client_id, original_filename, total_documents, processed_documents, 
        failed_documents, invalid_type_documents, status, progress_percent,
@@ -530,6 +779,183 @@ func (q *Queries) ListBatchUploads(ctx context.Context, arg *ListBatchUploadsPar
 	return items, nil
 }
 
+const listFolderCleanupReadyBatchIDs = `-- name: ListFolderCleanupReadyBatchIDs :many
+SELECT bu.id
+FROM batch_uploads bu
+WHERE bu.folder_cleanup_completed_at IS NULL
+  AND bu.status IN ('completed', 'failed', 'cancelled')
+  AND EXISTS (
+      SELECT 1
+      FROM batch_folder_candidates bfc
+      WHERE bfc.batch_id = bu.id
+  )
+  AND (
+      (
+          bu.total_documents > 0
+          AND
+          (SELECT COUNT(*) FROM batch_document_outcomes bdo WHERE bdo.batch_id = bu.id) = bu.total_documents
+          AND NOT EXISTS (
+              SELECT 1
+              FROM batch_document_outcomes bdo
+              WHERE bdo.batch_id = bu.id
+                AND bdo.outcome IN ('submitted', 'init_complete', 'sync_complete')
+          )
+      )
+      OR (
+          bu.status IN ('failed', 'cancelled')
+          AND (SELECT COUNT(*) FROM batch_document_outcomes bdo WHERE bdo.batch_id = bu.id) > 0
+          AND bu.completed_at IS NOT NULL
+          AND bu.completed_at <= NOW() - ($1::int * INTERVAL '1 second')
+      )
+  )
+ORDER BY bu.created_at ASC
+LIMIT $2
+`
+
+type ListFolderCleanupReadyBatchIDsParams struct {
+	StaleAfterSeconds int32 `db:"stale_after_seconds"`
+	LimitCount        int64 `db:"limit_count"`
+}
+
+// @sqlc-vet-disable
+//
+//	SELECT bu.id
+//	FROM batch_uploads bu
+//	WHERE bu.folder_cleanup_completed_at IS NULL
+//	  AND bu.status IN ('completed', 'failed', 'cancelled')
+//	  AND EXISTS (
+//	      SELECT 1
+//	      FROM batch_folder_candidates bfc
+//	      WHERE bfc.batch_id = bu.id
+//	  )
+//	  AND (
+//	      (
+//	          bu.total_documents > 0
+//	          AND
+//	          (SELECT COUNT(*) FROM batch_document_outcomes bdo WHERE bdo.batch_id = bu.id) = bu.total_documents
+//	          AND NOT EXISTS (
+//	              SELECT 1
+//	              FROM batch_document_outcomes bdo
+//	              WHERE bdo.batch_id = bu.id
+//	                AND bdo.outcome IN ('submitted', 'init_complete', 'sync_complete')
+//	          )
+//	      )
+//	      OR (
+//	          bu.status IN ('failed', 'cancelled')
+//	          AND (SELECT COUNT(*) FROM batch_document_outcomes bdo WHERE bdo.batch_id = bu.id) > 0
+//	          AND bu.completed_at IS NOT NULL
+//	          AND bu.completed_at <= NOW() - ($1::int * INTERVAL '1 second')
+//	      )
+//	  )
+//	ORDER BY bu.created_at ASC
+//	LIMIT $2
+func (q *Queries) ListFolderCleanupReadyBatchIDs(ctx context.Context, arg *ListFolderCleanupReadyBatchIDsParams) ([]uuid.UUID, error) {
+	rows, err := q.db.Query(ctx, listFolderCleanupReadyBatchIDs, arg.StaleAfterSeconds, arg.LimitCount)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	items := []uuid.UUID{}
+	for rows.Next() {
+		var id uuid.UUID
+		if err := rows.Scan(&id); err != nil {
+			return nil, err
+		}
+		items = append(items, id)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
+const lockBatchUploadForFolderCleanup = `-- name: LockBatchUploadForFolderCleanup :one
+SELECT id, status, total_documents, created_at, completed_at, folder_cleanup_completed_at
+FROM batch_uploads
+WHERE id = $1
+FOR UPDATE
+`
+
+type LockBatchUploadForFolderCleanupRow struct {
+	ID                       uuid.UUID        `db:"id"`
+	Status                   BatchStatus      `db:"status"`
+	TotalDocuments           int32            `db:"total_documents"`
+	CreatedAt                pgtype.Timestamp `db:"created_at"`
+	CompletedAt              pgtype.Timestamp `db:"completed_at"`
+	FolderCleanupCompletedAt pgtype.Timestamp `db:"folder_cleanup_completed_at"`
+}
+
+// LockBatchUploadForFolderCleanup
+//
+//	SELECT id, status, total_documents, created_at, completed_at, folder_cleanup_completed_at
+//	FROM batch_uploads
+//	WHERE id = $1
+//	FOR UPDATE
+func (q *Queries) LockBatchUploadForFolderCleanup(ctx context.Context, batchID uuid.UUID) (*LockBatchUploadForFolderCleanupRow, error) {
+	row := q.db.QueryRow(ctx, lockBatchUploadForFolderCleanup, batchID)
+	var i LockBatchUploadForFolderCleanupRow
+	err := row.Scan(
+		&i.ID,
+		&i.Status,
+		&i.TotalDocuments,
+		&i.CreatedAt,
+		&i.CompletedAt,
+		&i.FolderCleanupCompletedAt,
+	)
+	return &i, err
+}
+
+const markBatchFolderCleanupCompleted = `-- name: MarkBatchFolderCleanupCompleted :exec
+UPDATE batch_uploads
+SET folder_cleanup_completed_at = NOW()
+WHERE id = $1
+  AND folder_cleanup_completed_at IS NULL
+`
+
+// MarkBatchFolderCleanupCompleted
+//
+//	UPDATE batch_uploads
+//	SET folder_cleanup_completed_at = NOW()
+//	WHERE id = $1
+//	  AND folder_cleanup_completed_at IS NULL
+func (q *Queries) MarkBatchFolderCleanupCompleted(ctx context.Context, batchID uuid.UUID) error {
+	_, err := q.db.Exec(ctx, markBatchFolderCleanupCompleted, batchID)
+	return err
+}
+
+const recordBatchFolderCandidate = `-- name: RecordBatchFolderCandidate :exec
+INSERT INTO batch_folder_candidates (batch_id, folder_id, path, existed_before)
+VALUES ($1, $2, $3, $4)
+ON CONFLICT (batch_id, folder_id) DO UPDATE
+SET path = EXCLUDED.path,
+    existed_before = batch_folder_candidates.existed_before AND EXCLUDED.existed_before
+`
+
+type RecordBatchFolderCandidateParams struct {
+	BatchID       uuid.UUID `db:"batch_id"`
+	FolderID      uuid.UUID `db:"folder_id"`
+	Path          string    `db:"path"`
+	ExistedBefore bool      `db:"existed_before"`
+}
+
+// Records a non-root folder touched by a batch upload. Once a batch observes a
+// folder as newly created, retries must preserve that fact.
+//
+//	INSERT INTO batch_folder_candidates (batch_id, folder_id, path, existed_before)
+//	VALUES ($1, $2, $3, $4)
+//	ON CONFLICT (batch_id, folder_id) DO UPDATE
+//	SET path = EXCLUDED.path,
+//	    existed_before = batch_folder_candidates.existed_before AND EXCLUDED.existed_before
+func (q *Queries) RecordBatchFolderCandidate(ctx context.Context, arg *RecordBatchFolderCandidateParams) error {
+	_, err := q.db.Exec(ctx, recordBatchFolderCandidate,
+		arg.BatchID,
+		arg.FolderID,
+		arg.Path,
+		arg.ExistedBefore,
+	)
+	return err
+}
+
 const resolveBatchDocumentOutcome = `-- name: ResolveBatchDocumentOutcome :exec
 UPDATE batch_document_outcomes
 SET document_id = $3,
@@ -568,6 +994,27 @@ func (q *Queries) ResolveBatchDocumentOutcome(ctx context.Context, arg *ResolveB
 	return err
 }
 
+const setBatchCompletedAt = `-- name: SetBatchCompletedAt :exec
+UPDATE batch_uploads
+SET completed_at = $1
+WHERE id = $2
+`
+
+type SetBatchCompletedAtParams struct {
+	CompletedAt pgtype.Timestamp `db:"completed_at"`
+	ID          uuid.UUID        `db:"id"`
+}
+
+// SetBatchCompletedAt
+//
+//	UPDATE batch_uploads
+//	SET completed_at = $1
+//	WHERE id = $2
+func (q *Queries) SetBatchCompletedAt(ctx context.Context, arg *SetBatchCompletedAtParams) error {
+	_, err := q.db.Exec(ctx, setBatchCompletedAt, arg.CompletedAt, arg.ID)
+	return err
+}
+
 const setBatchTotalDocuments = `-- name: SetBatchTotalDocuments :exec
 UPDATE batch_uploads SET total_documents = $2 WHERE id = $1
 `
@@ -585,13 +1032,14 @@ func (q *Queries) SetBatchTotalDocuments(ctx context.Context, arg *SetBatchTotal
 	return err
 }
 
-const updateBatchDocumentOutcomeByDocumentID = `-- name: UpdateBatchDocumentOutcomeByDocumentID :exec
+const updateBatchDocumentOutcomeByDocumentID = `-- name: UpdateBatchDocumentOutcomeByDocumentID :many
 UPDATE batch_document_outcomes
 SET outcome = $2::batch_outcome_status,
     error_detail = $3,
     updated_at = NOW()
 WHERE document_id = $1
   AND outcome IN ('submitted', 'init_complete', 'sync_complete')
+RETURNING batch_id
 `
 
 type UpdateBatchDocumentOutcomeByDocumentIDParams struct {
@@ -611,9 +1059,25 @@ type UpdateBatchDocumentOutcomeByDocumentIDParams struct {
 //	    updated_at = NOW()
 //	WHERE document_id = $1
 //	  AND outcome IN ('submitted', 'init_complete', 'sync_complete')
-func (q *Queries) UpdateBatchDocumentOutcomeByDocumentID(ctx context.Context, arg *UpdateBatchDocumentOutcomeByDocumentIDParams) error {
-	_, err := q.db.Exec(ctx, updateBatchDocumentOutcomeByDocumentID, arg.DocumentID, arg.Column2, arg.ErrorDetail)
-	return err
+//	RETURNING batch_id
+func (q *Queries) UpdateBatchDocumentOutcomeByDocumentID(ctx context.Context, arg *UpdateBatchDocumentOutcomeByDocumentIDParams) ([]uuid.UUID, error) {
+	rows, err := q.db.Query(ctx, updateBatchDocumentOutcomeByDocumentID, arg.DocumentID, arg.Column2, arg.ErrorDetail)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	items := []uuid.UUID{}
+	for rows.Next() {
+		var batch_id uuid.UUID
+		if err := rows.Scan(&batch_id); err != nil {
+			return nil, err
+		}
+		items = append(items, batch_id)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
 }
 
 const updateBatchProgress = `-- name: UpdateBatchProgress :exec

internal/database/repository/eula.sql.go

@@ -12,6 +12,60 @@ import (
 	"github.com/jackc/pgx/v5/pgtype"
 )
 
+const activateLocalDevEulaVersion = `-- name: ActivateLocalDevEulaVersion :one
+WITH cleared AS (
+    UPDATE eulaVersions
+    SET isCurrent = FALSE
+    WHERE isCurrent = TRUE
+),
+upserted AS (
+    INSERT INTO eulaVersions (version, title, content, effectiveDate, createdBy, isCurrent, activatedAt, activatedBy)
+    VALUES ($1, $2, $3, NOW(), 'local-dev', TRUE, NOW(), 'local-dev')
+    ON CONFLICT (version) DO UPDATE
+    SET title = EXCLUDED.title,
+        content = EXCLUDED.content,
+        effectiveDate = EXCLUDED.effectiveDate,
+        isCurrent = TRUE,
+        activatedAt = NOW(),
+        activatedBy = EXCLUDED.activatedBy
+    RETURNING id
+)
+SELECT id FROM upserted
+`
+
+type ActivateLocalDevEulaVersionParams struct {
+	Version string `db:"version"`
+	Title   string `db:"title"`
+	Content string `db:"content"`
+}
+
+// Activates or updates the local development EULA version and makes it current.
+//
+//	WITH cleared AS (
+//	    UPDATE eulaVersions
+//	    SET isCurrent = FALSE
+//	    WHERE isCurrent = TRUE
+//	),
+//	upserted AS (
+//	    INSERT INTO eulaVersions (version, title, content, effectiveDate, createdBy, isCurrent, activatedAt, activatedBy)
+//	    VALUES ($1, $2, $3, NOW(), 'local-dev', TRUE, NOW(), 'local-dev')
+//	    ON CONFLICT (version) DO UPDATE
+//	    SET title = EXCLUDED.title,
+//	        content = EXCLUDED.content,
+//	        effectiveDate = EXCLUDED.effectiveDate,
+//	        isCurrent = TRUE,
+//	        activatedAt = NOW(),
+//	        activatedBy = EXCLUDED.activatedBy
+//	    RETURNING id
+//	)
+//	SELECT id FROM upserted
+func (q *Queries) ActivateLocalDevEulaVersion(ctx context.Context, arg *ActivateLocalDevEulaVersionParams) (uuid.UUID, error) {
+	row := q.db.QueryRow(ctx, activateLocalDevEulaVersion, arg.Version, arg.Title, arg.Content)
+	var id uuid.UUID
+	err := row.Scan(&id)
+	return id, err
+}
+
 const clearCurrentEulaVersions = `-- name: ClearCurrentEulaVersions :exec
 UPDATE eulaVersions
 SET isCurrent = FALSE
@@ -167,6 +221,28 @@ func (q *Queries) CreateEulaVersion(ctx context.Context, arg *CreateEulaVersionP
 	return &i, err
 }
 
+const createLocalDevEulaAgreement = `-- name: CreateLocalDevEulaAgreement :exec
+INSERT INTO eulaAgreements (cognitoSubjectId, userEmail, eulaVersionId, agreedFromIp)
+VALUES ($1, $2, $3, '127.0.0.1')
+ON CONFLICT (cognitoSubjectId, eulaVersionId) DO NOTHING
+`
+
+type CreateLocalDevEulaAgreementParams struct {
+	CognitoSubjectID string    `db:"cognito_subject_id"`
+	UserEmail        string    `db:"user_email"`
+	EulaVersionID    uuid.UUID `db:"eula_version_id"`
+}
+
+// Records an idempotent local development EULA agreement.
+//
+//	INSERT INTO eulaAgreements (cognitoSubjectId, userEmail, eulaVersionId, agreedFromIp)
+//	VALUES ($1, $2, $3, '127.0.0.1')
+//	ON CONFLICT (cognitoSubjectId, eulaVersionId) DO NOTHING
+func (q *Queries) CreateLocalDevEulaAgreement(ctx context.Context, arg *CreateLocalDevEulaAgreementParams) error {
+	_, err := q.db.Exec(ctx, createLocalDevEulaAgreement, arg.CognitoSubjectID, arg.UserEmail, arg.EulaVersionID)
+	return err
+}
+
 const getCurrentEulaVersion = `-- name: GetCurrentEulaVersion :one
 SELECT id, version, title, content, effectiveDate, createdAt, createdBy, isCurrent, activatedAt, activatedBy
 FROM eulaVersions

internal/database/repository/folders.sql.go

@@ -91,6 +91,47 @@ func (q *Queries) CreateFolder(ctx context.Context, arg *CreateFolderParams) (*F
 	return &i, err
 }
 
+const createFolderIfAbsent = `-- name: CreateFolderIfAbsent :one
+INSERT INTO folders (path, parentId, clientId, createdBy)
+VALUES ($1, $2, $3, $4)
+ON CONFLICT (clientId, path) DO NOTHING
+RETURNING id, path, parentid, clientid, createdat, createdby
+`
+
+type CreateFolderIfAbsentParams struct {
+	Path      string     `db:"path"`
+	ParentID  *uuid.UUID `db:"parent_id"`
+	ClientID  string     `db:"client_id"`
+	CreatedBy string     `db:"created_by"`
+}
+
+// Insert a folder only when it does not already exist.
+// Batch uploads use this to distinguish newly-created folder paths from
+// pre-existing paths without mutating existing folders.
+//
+//	INSERT INTO folders (path, parentId, clientId, createdBy)
+//	VALUES ($1, $2, $3, $4)
+//	ON CONFLICT (clientId, path) DO NOTHING
+//	RETURNING id, path, parentid, clientid, createdat, createdby
+func (q *Queries) CreateFolderIfAbsent(ctx context.Context, arg *CreateFolderIfAbsentParams) (*Folder, error) {
+	row := q.db.QueryRow(ctx, createFolderIfAbsent,
+		arg.Path,
+		arg.ParentID,
+		arg.ClientID,
+		arg.CreatedBy,
+	)
+	var i Folder
+	err := row.Scan(
+		&i.ID,
+		&i.Path,
+		&i.Parentid,
+		&i.Clientid,
+		&i.Createdat,
+		&i.Createdby,
+	)
+	return &i, err
+}
+
 const deleteDocumentUploadsInFolderTree = `-- name: DeleteDocumentUploadsInFolderTree :exec
 WITH RECURSIVE folder_tree AS (
     SELECT id FROM folders WHERE id = $1
@@ -116,6 +157,33 @@ func (q *Queries) DeleteDocumentUploadsInFolderTree(ctx context.Context, folderI
 	return err
 }
 
+const deleteSafeAutoCreatedFolder = `-- name: DeleteSafeAutoCreatedFolder :execrows
+DELETE FROM folders f
+WHERE f.id = $1
+  AND f.path <> '/'
+  AND NOT EXISTS (SELECT 1 FROM documents d WHERE d.folderId = f.id)
+  AND NOT EXISTS (SELECT 1 FROM documentUploads du WHERE du.folder_id = f.id)
+  AND NOT EXISTS (SELECT 1 FROM folders child WHERE child.parentId = f.id)
+`
+
+// @sqlc-vet-disable
+// Delete one auto-created folder only when it is currently empty and no longer
+// referenced by documents, uploads, or child folders.
+//
+//	DELETE FROM folders f
+//	WHERE f.id = $1
+//	  AND f.path <> '/'
+//	  AND NOT EXISTS (SELECT 1 FROM documents d WHERE d.folderId = f.id)
+//	  AND NOT EXISTS (SELECT 1 FROM documentUploads du WHERE du.folder_id = f.id)
+//	  AND NOT EXISTS (SELECT 1 FROM folders child WHERE child.parentId = f.id)
+func (q *Queries) DeleteSafeAutoCreatedFolder(ctx context.Context, folderID uuid.UUID) (int64, error) {
+	result, err := q.db.Exec(ctx, deleteSafeAutoCreatedFolder, folderID)
+	if err != nil {
+		return 0, err
+	}
+	return result.RowsAffected(), nil
+}
+
 const getClientRootFolder = `-- name: GetClientRootFolder :one
 SELECT id, path, parentid, clientid, createdat, createdby FROM folders
 WHERE clientId = $1 AND path = '/'
@@ -743,6 +811,69 @@ func (q *Queries) HardDeleteFolderTree(ctx context.Context, folderID *uuid.UUID)
 	return result.RowsAffected(), nil
 }
 
+const listSafeAutoCreatedFolderCandidates = `-- name: ListSafeAutoCreatedFolderCandidates :many
+SELECT DISTINCT f.id, f.path
+FROM folders f
+JOIN batch_folder_candidates bfc ON bfc.folder_id = f.id
+WHERE bfc.existed_before = false
+  AND bfc.path = f.path
+  AND f.path <> '/'
+  AND NOT EXISTS (
+      SELECT 1
+      FROM batch_folder_candidates other_bfc
+      JOIN batch_uploads bu ON bu.id = other_bfc.batch_id
+      WHERE other_bfc.folder_id = f.id
+        AND bu.folder_cleanup_completed_at IS NULL
+  )
+ORDER BY f.path DESC
+LIMIT $1
+`
+
+type ListSafeAutoCreatedFolderCandidatesRow struct {
+	ID   uuid.UUID `db:"id"`
+	Path string    `db:"path"`
+}
+
+// @sqlc-vet-disable
+// Candidate folders can be deleted only after every batch that touched that
+// folder has finished folder cleanup. The path equality check protects renamed
+// folders from deletion.
+//
+//	SELECT DISTINCT f.id, f.path
+//	FROM folders f
+//	JOIN batch_folder_candidates bfc ON bfc.folder_id = f.id
+//	WHERE bfc.existed_before = false
+//	  AND bfc.path = f.path
+//	  AND f.path <> '/'
+//	  AND NOT EXISTS (
+//	      SELECT 1
+//	      FROM batch_folder_candidates other_bfc
+//	      JOIN batch_uploads bu ON bu.id = other_bfc.batch_id
+//	      WHERE other_bfc.folder_id = f.id
+//	        AND bu.folder_cleanup_completed_at IS NULL
+//	  )
+//	ORDER BY f.path DESC
+//	LIMIT $1
+func (q *Queries) ListSafeAutoCreatedFolderCandidates(ctx context.Context, limitCount int64) ([]*ListSafeAutoCreatedFolderCandidatesRow, error) {
+	rows, err := q.db.Query(ctx, listSafeAutoCreatedFolderCandidates, limitCount)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	items := []*ListSafeAutoCreatedFolderCandidatesRow{}
+	for rows.Next() {
+		var i ListSafeAutoCreatedFolderCandidatesRow
+		if err := rows.Scan(&i.ID, &i.Path); err != nil {
+			return nil, err
+		}
+		items = append(items, &i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
 const renameFolder = `-- name: RenameFolder :exec
 UPDATE folders
 SET path = $2

internal/database/repository/models.go

@@ -348,22 +348,31 @@ type BatchDocumentOutcome struct {
 	CreatedAt   pgtype.Timestamp   `db:"created_at"`
 }
 
+type BatchFolderCandidate struct {
+	BatchID       uuid.UUID        `db:"batch_id"`
+	FolderID      uuid.UUID        `db:"folder_id"`
+	Path          string           `db:"path"`
+	ExistedBefore bool             `db:"existed_before"`
+	CreatedAt     pgtype.Timestamp `db:"created_at"`
+}
+
 type BatchUpload struct {
-	ID                   uuid.UUID        `db:"id"`
-	ClientID             string           `db:"client_id"`
-	OriginalFilename     string           `db:"original_filename"`
-	TotalDocuments       int32            `db:"total_documents"`
-	ProcessedDocuments   int32            `db:"processed_documents"`
-	FailedDocuments      int32            `db:"failed_documents"`
-	InvalidTypeDocuments int32            `db:"invalid_type_documents"`
-	Status               BatchStatus      `db:"status"`
-	ProgressPercent      int32            `db:"progress_percent"`
-	FailedFilenames      []byte           `db:"failed_filenames"`
-	CreatedAt            pgtype.Timestamp `db:"created_at"`
-	CompletedAt          pgtype.Timestamp `db:"completed_at"`
-	ArchiveBucket        *string          `db:"archive_bucket"`
-	ArchiveKey           *string          `db:"archive_key"`
-	FileSizeBytes        *int64           `db:"file_size_bytes"`
+	ID                       uuid.UUID        `db:"id"`
+	ClientID                 string           `db:"client_id"`
+	OriginalFilename         string           `db:"original_filename"`
+	TotalDocuments           int32            `db:"total_documents"`
+	ProcessedDocuments       int32            `db:"processed_documents"`
+	FailedDocuments          int32            `db:"failed_documents"`
+	InvalidTypeDocuments     int32            `db:"invalid_type_documents"`
+	Status                   BatchStatus      `db:"status"`
+	ProgressPercent          int32            `db:"progress_percent"`
+	FailedFilenames          []byte           `db:"failed_filenames"`
+	CreatedAt                pgtype.Timestamp `db:"created_at"`
+	CompletedAt              pgtype.Timestamp `db:"completed_at"`
+	ArchiveBucket            *string          `db:"archive_bucket"`
+	ArchiveKey               *string          `db:"archive_key"`
+	FileSizeBytes            *int64           `db:"file_size_bytes"`
+	FolderCleanupCompletedAt pgtype.Timestamp `db:"folder_cleanup_completed_at"`
 }
 
 type BotSession struct {