2025-02-03 17:30:50 +00:00
|
|
|
package docinitrunner
|
2024-12-18 18:54:48 +00:00
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"context"
|
2025-09-05 18:25:31 +00:00
|
|
|
"database/sql"
|
|
|
|
|
"errors"
|
2025-02-06 15:00:26 +00:00
|
|
|
"log/slog"
|
2025-03-05 12:05:46 +00:00
|
|
|
|
2025-09-05 18:25:31 +00:00
|
|
|
"queryorchestration/internal/database/repository"
|
2026-03-12 18:53:42 +00:00
|
|
|
"queryorchestration/internal/document/batch/outcome"
|
2025-02-03 17:30:50 +00:00
|
|
|
documentinit "queryorchestration/internal/document/init"
|
2025-04-03 19:17:24 +00:00
|
|
|
"queryorchestration/internal/serviceconfig/objectstore"
|
2025-11-26 19:23:42 +00:00
|
|
|
|
|
|
|
|
"github.com/google/uuid"
|
2024-12-18 18:54:48 +00:00
|
|
|
)
|
|
|
|
|
|
2025-02-03 17:30:50 +00:00
|
|
|
const Name = "docInitRunner"
|
|
|
|
|
|
2026-03-12 18:53:42 +00:00
|
|
|
// Services holds the dependencies for the docInitRunner.
|
|
|
|
|
//
|
|
|
|
|
// Fields:
|
|
|
|
|
// - Document: service for creating/deduplicating documents
|
|
|
|
|
// - Queries: SQLC-generated database queries
|
|
|
|
|
// - Outcome: reporter for writing batch pipeline outcomes
|
2025-01-24 16:12:25 +00:00
|
|
|
type Services struct {
|
2025-02-03 17:30:50 +00:00
|
|
|
Document *documentinit.Service
|
2025-09-05 18:25:31 +00:00
|
|
|
Queries *repository.Queries
|
2026-03-12 18:53:42 +00:00
|
|
|
Outcome *outcome.Reporter
|
2025-01-24 16:12:25 +00:00
|
|
|
}
|
|
|
|
|
|
2026-03-12 18:53:42 +00:00
|
|
|
// Runner processes document init messages from the SQS queue.
|
2025-02-03 17:30:50 +00:00
|
|
|
type Runner struct {
|
2025-04-22 14:40:16 +00:00
|
|
|
svc *Services
|
2024-12-18 18:54:48 +00:00
|
|
|
}
|
|
|
|
|
|
2026-03-12 18:53:42 +00:00
|
|
|
// New creates a new docInitRunner.
|
|
|
|
|
//
|
|
|
|
|
// Parameters:
|
|
|
|
|
// - svc: the runner's service dependencies
|
|
|
|
|
//
|
|
|
|
|
// Returns:
|
|
|
|
|
// - Runner: the runner instance
|
2025-04-22 14:40:16 +00:00
|
|
|
func New(svc *Services) Runner {
|
2025-02-03 17:30:50 +00:00
|
|
|
return Runner{
|
2025-04-22 14:40:16 +00:00
|
|
|
svc: svc,
|
2024-12-18 18:54:48 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-03-12 18:53:42 +00:00
|
|
|
// Body is the SQS message body for docInitRunner.
|
2025-04-02 18:50:03 +00:00
|
|
|
type Body struct {
|
2025-04-03 19:17:24 +00:00
|
|
|
Bucket string `json:"bucket" validate:"required"`
|
|
|
|
|
Key string `json:"key" validate:"required"`
|
|
|
|
|
Hash string `json:"hash" validate:"required"`
|
2025-03-05 19:49:03 +00:00
|
|
|
}
|
|
|
|
|
|
2026-03-12 18:53:42 +00:00
|
|
|
// Process handles a single document init message. Returns true to acknowledge
|
|
|
|
|
// the message (success), or false to requeue (infrastructure error).
|
2025-04-02 18:50:03 +00:00
|
|
|
func (s Runner) Process(ctx context.Context, body Body) bool {
|
2025-04-03 19:17:24 +00:00
|
|
|
key, err := objectstore.ParseBucketKey(body.Key)
|
|
|
|
|
if err != nil {
|
2025-10-08 23:12:51 +00:00
|
|
|
slog.Error("unable to parse key", "key", body.Key, "error", err)
|
2025-04-03 19:17:24 +00:00
|
|
|
return false
|
|
|
|
|
}
|
|
|
|
|
|
2025-11-26 19:23:42 +00:00
|
|
|
// Get filename and folder_id from documentUploads table
|
2025-09-05 18:25:31 +00:00
|
|
|
var filename *string
|
2025-11-26 19:23:42 +00:00
|
|
|
var folderID *uuid.UUID
|
2025-09-05 18:25:31 +00:00
|
|
|
upload, err := s.svc.Queries.GetDocumentUploadByKey(ctx, &repository.GetDocumentUploadByKeyParams{
|
2025-04-03 19:17:24 +00:00
|
|
|
Bucket: body.Bucket,
|
2025-09-05 18:25:31 +00:00
|
|
|
Key: body.Key,
|
|
|
|
|
})
|
|
|
|
|
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
|
|
|
|
slog.Error("unable to get document upload", "bucket", body.Bucket, "key", body.Key, "error", err)
|
|
|
|
|
return false
|
|
|
|
|
}
|
2025-11-26 19:23:42 +00:00
|
|
|
if err == nil {
|
|
|
|
|
if upload.Filename != nil {
|
|
|
|
|
filename = upload.Filename
|
|
|
|
|
}
|
|
|
|
|
folderID = upload.FolderID
|
2025-09-05 18:25:31 +00:00
|
|
|
}
|
|
|
|
|
|
2026-03-12 18:53:42 +00:00
|
|
|
result, err := s.svc.Document.Create(ctx, &documentinit.Create{
|
2025-11-26 19:23:42 +00:00
|
|
|
Key: key,
|
|
|
|
|
Bucket: body.Bucket,
|
|
|
|
|
Hash: body.Hash,
|
|
|
|
|
Filename: filename,
|
|
|
|
|
OriginalPath: filename, // For batch uploads, filename contains the full original path
|
|
|
|
|
FolderID: folderID,
|
2025-03-05 19:49:03 +00:00
|
|
|
})
|
|
|
|
|
if err != nil {
|
2025-04-02 18:50:03 +00:00
|
|
|
return false
|
2024-12-18 18:54:48 +00:00
|
|
|
}
|
2026-03-12 18:53:42 +00:00
|
|
|
slog.Debug("created document", "id", result.ID, "filename", filename, "folder_id", folderID)
|
|
|
|
|
|
|
|
|
|
// Report outcome for batch documents
|
|
|
|
|
outcomeName := repository.BatchOutcomeStatusInitComplete
|
|
|
|
|
if result.IsDuplicate {
|
|
|
|
|
outcomeName = repository.BatchOutcomeStatusInitDuplicate
|
|
|
|
|
}
|
|
|
|
|
var resolveFilename string
|
|
|
|
|
if result.Filename != nil {
|
|
|
|
|
resolveFilename = *result.Filename
|
|
|
|
|
}
|
|
|
|
|
if resolveErr := s.svc.Outcome.Resolve(ctx, result.BatchID, resolveFilename,
|
|
|
|
|
result.ID, outcomeName, nil); resolveErr != nil {
|
|
|
|
|
slog.Error("failed to resolve batch outcome",
|
|
|
|
|
"document_id", result.ID, "outcome", outcomeName, "error", resolveErr)
|
|
|
|
|
// Non-fatal: document was created successfully
|
|
|
|
|
}
|
2024-12-18 18:54:48 +00:00
|
|
|
|
2025-04-02 18:50:03 +00:00
|
|
|
return true
|
2025-03-05 19:49:03 +00:00
|
|
|
}
|