jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/doc_import (pull request #177)

Browse Source 
integration of background processor

* integration part 1

* feature working

* fix mimetype issue
This commit is contained in:
Jay Brown
2025-08-20 19:01:13 +00:00
parent 7a693d42f5
commit 720a84be92
34 changed files with 2682 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/database/queries/batch.sql
+9 -1
View File
@@ -71,4 +71,12 @@ WHERE batch_id = $1;
-- name: CountDocumentsByBatchId :one
SELECT COUNT(*) as count
FROM documents
WHERE batch_id = $1;
WHERE batch_id = $1;
-- name: GetUnprocessedBatches :many
SELECT id, client_id, original_filename, total_documents,
processed_documents, failed_documents, invalid_type_documents,
status, progress_percent, created_at, completed_at
FROM batch_uploads
WHERE status = 'processing'
ORDER BY created_at ASC;
internal/database/repository/batch.sql.go
+63
View File
@@ -304,6 +304,69 @@ func (q *Queries) GetDocumentsByBatchId(ctx context.Context, batchID *uuid.UUID)
return items, nil
}
const getUnprocessedBatches = `-- name: GetUnprocessedBatches :many
SELECT id, client_id, original_filename, total_documents,
processed_documents, failed_documents, invalid_type_documents,
status, progress_percent, created_at, completed_at
FROM batch_uploads
WHERE status = 'processing'
ORDER BY created_at ASC
`
type GetUnprocessedBatchesRow struct {
ID uuid.UUID `db:"id"`
ClientID string `db:"client_id"`
OriginalFilename string `db:"original_filename"`
TotalDocuments int32 `db:"total_documents"`
ProcessedDocuments int32 `db:"processed_documents"`
FailedDocuments int32 `db:"failed_documents"`
InvalidTypeDocuments int32 `db:"invalid_type_documents"`
Status BatchStatus `db:"status"`
ProgressPercent int32 `db:"progress_percent"`
CreatedAt pgtype.Timestamp `db:"created_at"`
CompletedAt pgtype.Timestamp `db:"completed_at"`
}
// GetUnprocessedBatches
//
// SELECT id, client_id, original_filename, total_documents,
// processed_documents, failed_documents, invalid_type_documents,
// status, progress_percent, created_at, completed_at
// FROM batch_uploads
// WHERE status = 'processing'
// ORDER BY created_at ASC
func (q *Queries) GetUnprocessedBatches(ctx context.Context) ([]*GetUnprocessedBatchesRow, error) {
rows, err := q.db.Query(ctx, getUnprocessedBatches)
if err != nil {
return nil, err
}
defer rows.Close()
items := []*GetUnprocessedBatchesRow{}
for rows.Next() {
var i GetUnprocessedBatchesRow
if err := rows.Scan(
&i.ID,
&i.ClientID,
&i.OriginalFilename,
&i.TotalDocuments,
&i.ProcessedDocuments,
&i.FailedDocuments,
&i.InvalidTypeDocuments,
&i.Status,
&i.ProgressPercent,
&i.CreatedAt,
&i.CompletedAt,
); err != nil {
return nil, err
}
items = append(items, &i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listBatchUploads = `-- name: ListBatchUploads :many
SELECT id, client_id, original_filename, total_documents, processed_documents,
failed_documents, invalid_type_documents, status, progress_percent,
internal/database/repository/batch_test.go
+103
View File
@@ -392,6 +392,109 @@ func TestBatchUpload(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, int64(4), count)
})
t.Run("GetUnprocessedBatches", func(t *testing.T) {
t.Parallel()
ctx := t.Context()
cfg := &serviceconfig.BaseConfig{}
test.CreateDB(t, cfg)
queries := cfg.GetDBQueries()
// Create multiple clients
clientID1 := fmt.Sprintf("TEST_CLIENT_UNPROCESSED1_%s", uuid.New().String()[:8])
clientID2 := fmt.Sprintf("TEST_CLIENT_UNPROCESSED2_%s", uuid.New().String()[:8])
for _, clientID := range []string{clientID1, clientID2} {
err := queries.CreateClient(ctx, &repository.CreateClientParams{
Name: fmt.Sprintf("Test Client %s", clientID),
Clientid: clientID,
})
require.NoError(t, err)
}
// Create batches with different statuses
var processingBatchIDs []uuid.UUID
// Client 1: 2 processing batches
for i := 0; i < 2; i++ {
batchID, err := queries.CreateBatchUpload(ctx, &repository.CreateBatchUploadParams{
ClientID: clientID1,
OriginalFilename: fmt.Sprintf("processing%d.zip", i),
TotalDocuments: 10,
})
require.NoError(t, err)
processingBatchIDs = append(processingBatchIDs, batchID)
}
// Client 2: 1 processing batch
batchID, err := queries.CreateBatchUpload(ctx, &repository.CreateBatchUploadParams{
ClientID: clientID2,
OriginalFilename: "processing_client2.zip",
TotalDocuments: 5,
})
require.NoError(t, err)
processingBatchIDs = append(processingBatchIDs, batchID)
// Create completed batch (should not be returned)
completedBatchID, err := queries.CreateBatchUpload(ctx, &repository.CreateBatchUploadParams{
ClientID: clientID1,
OriginalFilename: "completed.zip",
TotalDocuments: 8,
})
require.NoError(t, err)
err = queries.UpdateBatchStatus(ctx, &repository.UpdateBatchStatusParams{
ID: completedBatchID,
Column2: repository.BatchStatusCompleted,
})
require.NoError(t, err)
// Create failed batch (should not be returned)
failedBatchID, err := queries.CreateBatchUpload(ctx, &repository.CreateBatchUploadParams{
ClientID: clientID2,
OriginalFilename: "failed.zip",
TotalDocuments: 3,
})
require.NoError(t, err)
err = queries.UpdateBatchStatus(ctx, &repository.UpdateBatchStatusParams{
ID: failedBatchID,
Column2: repository.BatchStatusFailed,
})
require.NoError(t, err)
// Get unprocessed batches
unprocessedBatches, err := queries.GetUnprocessedBatches(ctx)
require.NoError(t, err)
// Should return exactly 3 processing batches
assert.Len(t, unprocessedBatches, 3)
// Verify all returned batches have processing status
returnedBatchIDs := make([]uuid.UUID, len(unprocessedBatches))
for i, batch := range unprocessedBatches {
assert.Equal(t, repository.BatchStatusProcessing, batch.Status)
returnedBatchIDs[i] = batch.ID
}
// Verify all processing batches are returned
for _, expectedID := range processingBatchIDs {
assert.Contains(t, returnedBatchIDs, expectedID)
}
// Verify completed and failed batches are NOT returned
assert.NotContains(t, returnedBatchIDs, completedBatchID)
assert.NotContains(t, returnedBatchIDs, failedBatchID)
// Verify batches are ordered by created_at ASC (oldest first)
if len(unprocessedBatches) > 1 {
for i := 0; i < len(unprocessedBatches)-1; i++ {
assert.True(t,
unprocessedBatches[i].CreatedAt.Time.Before(unprocessedBatches[i+1].CreatedAt.Time) ||
unprocessedBatches[i].CreatedAt.Time.Equal(unprocessedBatches[i+1].CreatedAt.Time),
"batches should be ordered by created_at ASC")
}
}
})
}
func TestBatchStatusEnum(t *testing.T) {
internal/database/repository/document_test.go
+13 -9
View File
@@ -3,6 +3,7 @@ package repository_test
import (
"fmt"
"testing"
"time"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/serviceconfig"
@@ -28,7 +29,7 @@ func TestDocument(t *testing.T) {
clientId := fmt.Sprintf("EXAMPLE_%s", uuid.New().String()[:8])
err := queries.CreateClient(ctx, &repository.CreateClientParams{
Name: "example_client",
Name: fmt.Sprintf("example_client_%s", clientId),
Clientid: clientId,
})
require.NoError(t, err)
@@ -48,7 +49,7 @@ func TestDocument(t *testing.T) {
clientId := fmt.Sprintf("EXAMPLE_%s", uuid.New().String()[:8])
err := queries.CreateClient(ctx, &repository.CreateClientParams{
Name: "example_client",
Name: fmt.Sprintf("example_client_%s", clientId),
Clientid: clientId,
})
require.NoError(t, err)
@@ -79,7 +80,7 @@ func TestDocument(t *testing.T) {
clientId := fmt.Sprintf("EXAMPLE_%s", uuid.New().String()[:8])
err := queries.CreateClient(ctx, &repository.CreateClientParams{
Name: "example_client",
Name: fmt.Sprintf("example_client_%s", clientId),
Clientid: clientId,
})
require.NoError(t, err)
@@ -116,7 +117,7 @@ func TestDocument(t *testing.T) {
clientId := fmt.Sprintf("EXAMPLE_%s", uuid.New().String()[:8])
err := queries.CreateClient(ctx, &repository.CreateClientParams{
Name: "example_client",
Name: fmt.Sprintf("example_client_%s", clientId),
Clientid: clientId,
})
require.NoError(t, err)
@@ -164,7 +165,7 @@ func TestDocument(t *testing.T) {
clientId := fmt.Sprintf("EXAMPLE_%s", uuid.New().String()[:8])
err := queries.CreateClient(ctx, &repository.CreateClientParams{
Name: "example_client",
Name: fmt.Sprintf("example_client_%s", clientId),
Clientid: clientId,
})
require.NoError(t, err)
@@ -197,7 +198,7 @@ func TestDocument(t *testing.T) {
clientId := fmt.Sprintf("EXAMPLE_%s", uuid.New().String()[:8])
err := queries.CreateClient(ctx, &repository.CreateClientParams{
Name: "example_client",
Name: fmt.Sprintf("example_client_%s", clientId),
Clientid: clientId,
})
require.NoError(t, err)
@@ -231,7 +232,7 @@ func TestDocument(t *testing.T) {
clientId := fmt.Sprintf("EXAMPLE_%s", uuid.New().String()[:8])
err := queries.CreateClient(ctx, &repository.CreateClientParams{
Name: "example_client",
Name: fmt.Sprintf("example_client_%s", clientId),
Clientid: clientId,
})
require.NoError(t, err)
@@ -263,7 +264,7 @@ func TestDocument(t *testing.T) {
clientId := fmt.Sprintf("EXAMPLE_%s", uuid.New().String()[:8])
err := queries.CreateClient(ctx, &repository.CreateClientParams{
Name: "example_client",
Name: fmt.Sprintf("example_client_%s", clientId),
Clientid: clientId,
})
require.NoError(t, err)
@@ -305,7 +306,7 @@ func TestDocument(t *testing.T) {
clientId := fmt.Sprintf("EXAMPLE_%s", uuid.New().String()[:8])
err := queries.CreateClient(ctx, &repository.CreateClientParams{
Name: "example_client",
Name: fmt.Sprintf("example_client_%s", clientId),
Clientid: clientId,
})
require.NoError(t, err)
@@ -328,6 +329,9 @@ func TestDocument(t *testing.T) {
})
require.NoError(t, err)
// Add a small delay to ensure UUID v7 timestamps are different
time.Sleep(2 * time.Millisecond)
buckettwo := "buckettwo"
keytwo := "keytwo"
err = queries.AddDocumentEntry(ctx, &repository.AddDocumentEntryParams{