Merged in jmathison/v2-upload-batch (pull request #224)

Implement v2 upload batch cleanup

* Implement v2 upload batch cleanup

* Merge remote-tracking branch 'origin/main' into jmathison/v2-upload-batch

* Address upload batch review feedback

* Raise batch worker coverage

* Fix batch cleanup review issues


Approved-by: Jay Brown

internal/database/repository/folders.sql.go

@@ -91,6 +91,47 @@ func (q *Queries) CreateFolder(ctx context.Context, arg *CreateFolderParams) (*F
 	return &i, err
 }
 
+const createFolderIfAbsent = `-- name: CreateFolderIfAbsent :one
+INSERT INTO folders (path, parentId, clientId, createdBy)
+VALUES ($1, $2, $3, $4)
+ON CONFLICT (clientId, path) DO NOTHING
+RETURNING id, path, parentid, clientid, createdat, createdby
+`
+
+type CreateFolderIfAbsentParams struct {
+	Path      string     `db:"path"`
+	ParentID  *uuid.UUID `db:"parent_id"`
+	ClientID  string     `db:"client_id"`
+	CreatedBy string     `db:"created_by"`
+}
+
+// Insert a folder only when it does not already exist.
+// Batch uploads use this to distinguish newly-created folder paths from
+// pre-existing paths without mutating existing folders.
+//
+//	INSERT INTO folders (path, parentId, clientId, createdBy)
+//	VALUES ($1, $2, $3, $4)
+//	ON CONFLICT (clientId, path) DO NOTHING
+//	RETURNING id, path, parentid, clientid, createdat, createdby
+func (q *Queries) CreateFolderIfAbsent(ctx context.Context, arg *CreateFolderIfAbsentParams) (*Folder, error) {
+	row := q.db.QueryRow(ctx, createFolderIfAbsent,
+		arg.Path,
+		arg.ParentID,
+		arg.ClientID,
+		arg.CreatedBy,
+	)
+	var i Folder
+	err := row.Scan(
+		&i.ID,
+		&i.Path,
+		&i.Parentid,
+		&i.Clientid,
+		&i.Createdat,
+		&i.Createdby,
+	)
+	return &i, err
+}
+
 const deleteDocumentUploadsInFolderTree = `-- name: DeleteDocumentUploadsInFolderTree :exec
 WITH RECURSIVE folder_tree AS (
     SELECT id FROM folders WHERE id = $1
@@ -116,6 +157,33 @@ func (q *Queries) DeleteDocumentUploadsInFolderTree(ctx context.Context, folderI
 	return err
 }
 
+const deleteSafeAutoCreatedFolder = `-- name: DeleteSafeAutoCreatedFolder :execrows
+DELETE FROM folders f
+WHERE f.id = $1
+  AND f.path <> '/'
+  AND NOT EXISTS (SELECT 1 FROM documents d WHERE d.folderId = f.id)
+  AND NOT EXISTS (SELECT 1 FROM documentUploads du WHERE du.folder_id = f.id)
+  AND NOT EXISTS (SELECT 1 FROM folders child WHERE child.parentId = f.id)
+`
+
+// @sqlc-vet-disable
+// Delete one auto-created folder only when it is currently empty and no longer
+// referenced by documents, uploads, or child folders.
+//
+//	DELETE FROM folders f
+//	WHERE f.id = $1
+//	  AND f.path <> '/'
+//	  AND NOT EXISTS (SELECT 1 FROM documents d WHERE d.folderId = f.id)
+//	  AND NOT EXISTS (SELECT 1 FROM documentUploads du WHERE du.folder_id = f.id)
+//	  AND NOT EXISTS (SELECT 1 FROM folders child WHERE child.parentId = f.id)
+func (q *Queries) DeleteSafeAutoCreatedFolder(ctx context.Context, folderID uuid.UUID) (int64, error) {
+	result, err := q.db.Exec(ctx, deleteSafeAutoCreatedFolder, folderID)
+	if err != nil {
+		return 0, err
+	}
+	return result.RowsAffected(), nil
+}
+
 const getClientRootFolder = `-- name: GetClientRootFolder :one
 SELECT id, path, parentid, clientid, createdat, createdby FROM folders
 WHERE clientId = $1 AND path = '/'
@@ -743,6 +811,69 @@ func (q *Queries) HardDeleteFolderTree(ctx context.Context, folderID *uuid.UUID)
 	return result.RowsAffected(), nil
 }
 
+const listSafeAutoCreatedFolderCandidates = `-- name: ListSafeAutoCreatedFolderCandidates :many
+SELECT DISTINCT f.id, f.path
+FROM folders f
+JOIN batch_folder_candidates bfc ON bfc.folder_id = f.id
+WHERE bfc.existed_before = false
+  AND bfc.path = f.path
+  AND f.path <> '/'
+  AND NOT EXISTS (
+      SELECT 1
+      FROM batch_folder_candidates other_bfc
+      JOIN batch_uploads bu ON bu.id = other_bfc.batch_id
+      WHERE other_bfc.folder_id = f.id
+        AND bu.folder_cleanup_completed_at IS NULL
+  )
+ORDER BY f.path DESC
+LIMIT $1
+`
+
+type ListSafeAutoCreatedFolderCandidatesRow struct {
+	ID   uuid.UUID `db:"id"`
+	Path string    `db:"path"`
+}
+
+// @sqlc-vet-disable
+// Candidate folders can be deleted only after every batch that touched that
+// folder has finished folder cleanup. The path equality check protects renamed
+// folders from deletion.
+//
+//	SELECT DISTINCT f.id, f.path
+//	FROM folders f
+//	JOIN batch_folder_candidates bfc ON bfc.folder_id = f.id
+//	WHERE bfc.existed_before = false
+//	  AND bfc.path = f.path
+//	  AND f.path <> '/'
+//	  AND NOT EXISTS (
+//	      SELECT 1
+//	      FROM batch_folder_candidates other_bfc
+//	      JOIN batch_uploads bu ON bu.id = other_bfc.batch_id
+//	      WHERE other_bfc.folder_id = f.id
+//	        AND bu.folder_cleanup_completed_at IS NULL
+//	  )
+//	ORDER BY f.path DESC
+//	LIMIT $1
+func (q *Queries) ListSafeAutoCreatedFolderCandidates(ctx context.Context, limitCount int64) ([]*ListSafeAutoCreatedFolderCandidatesRow, error) {
+	rows, err := q.db.Query(ctx, listSafeAutoCreatedFolderCandidates, limitCount)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	items := []*ListSafeAutoCreatedFolderCandidatesRow{}
+	for rows.Next() {
+		var i ListSafeAutoCreatedFolderCandidatesRow
+		if err := rows.Scan(&i.ID, &i.Path); err != nil {
+			return nil, err
+		}
+		items = append(items, &i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
 const renameFolder = `-- name: RenameFolder :exec
 UPDATE folders
 SET path = $2