jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/textextract (pull request #108)

Browse Source 
Start adding Textract + UUID changes

* base

* startclient

* ts

* short

* tests
This commit is contained in:
Michael McGuinness
2025-03-20 11:06:41 +00:00
parent 1587da9d11
commit 53ea7d34e6
196 changed files with 25972 additions and 1456 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/document/text/create.go
+3 -4
View File
@@ -5,7 +5,6 @@ import (
"log/slog"
querysyncrunner "queryorchestration/api/querySyncRunner"
"queryorchestration/internal/database"
"queryorchestration/internal/serviceconfig/queue"
"github.com/google/uuid"
@@ -14,13 +13,13 @@ import (
func (s *Service) Extract(ctx context.Context, id uuid.UUID) error {
slog.Debug("extracting document text", "id", id.String())
isextracted, err := s.cfg.GetDBQueries().IsDocumentTextExtracted(ctx, database.MustToDBUUID(id))
isextracted, err := s.cfg.GetDBQueries().IsDocumentTextExtracted(ctx, id)
if err != nil {
return err
}
if !isextracted {
entry, err := s.cfg.GetDBQueries().GetCleanEntryByDocId(ctx, database.MustToDBUUID(id))
entry, err := s.cfg.GetDBQueries().GetCleanEntryByDocId(ctx, id)
if err != nil {
return err
} else if entry.Fail.Valid {
@@ -28,7 +27,7 @@ func (s *Service) Extract(ctx context.Context, id uuid.UUID) error {
return nil
}
err = s.extract(ctx, database.MustToUUID(entry.ID))
err = s.extract(ctx, entry.ID)
if err != nil {
return err
}
internal/document/text/create_test.go
+20 -11
View File
@@ -5,11 +5,11 @@ import (
"fmt"
"testing"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/document"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/queue/querysync"
"queryorchestration/internal/serviceconfig/textract"
queuemock "queryorchestration/mocks/queue"
"github.com/stretchr/testify/require"
@@ -23,6 +23,7 @@ import (
type DocTextConfig struct {
serviceconfig.BaseConfig
querysync.QuerySyncConfig
textract.TextractConfig
}
func TestCreate(t *testing.T) {
@@ -49,29 +50,37 @@ func TestCreate(t *testing.T) {
Key: "/i/am/here",
}
pool.ExpectQuery("name: IsDocumentTextExtracted :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows(
pool.ExpectQuery("name: IsDocumentTextExtracted :one").WithArgs(id).WillReturnRows(
pgxmock.NewRows([]string{"isextracted"}).
AddRow(false),
)
cleanId := database.MustToDBUUID(uuid.New())
cleanId := uuid.New()
mimeType := "application/pdf"
dbmimetype := repository.NullCleanmimetype{
Valid: true,
Cleanmimetype: repository.Cleanmimetype(mimeType),
}
pool.ExpectQuery("name: GetCleanEntryByDocId :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows(
pool.ExpectQuery("name: GetCleanEntryByDocId :one").WithArgs(id).WillReturnRows(
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "mimetype", "fail"}).
AddRow(cleanId, database.MustToDBUUID(id), &inloc.Bucket, &inloc.Key, int32(1), dbmimetype, repository.NullCleanfailtype{}),
AddRow(cleanId, id, &inloc.Bucket, &inloc.Key, int32(1), dbmimetype, repository.NullCleanfailtype{}),
)
pool.ExpectQuery("name: GetCleanEntry :one").WithArgs(cleanId).WillReturnRows(
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "mimetype", "fail"}).
AddRow(cleanId, database.MustToDBUUID(id), &inloc.Bucket, &inloc.Key, int32(1), dbmimetype, repository.NullCleanfailtype{}),
AddRow(cleanId, id, &inloc.Bucket, &inloc.Key, int32(1), dbmimetype, repository.NullCleanfailtype{}),
)
pool.ExpectQuery("name: AddDocumentTextEntry :one").WithArgs(pgxmock.AnyArg(), inloc.Bucket, inloc.Key, cleanId).
textId := uuid.New()
pool.ExpectBegin()
pool.ExpectQuery("name: GetDocumentTextExtractionByHash :one").WithArgs(cleanId, "example").WillReturnRows(
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "hash", "cleanEntryId"}),
)
pool.ExpectQuery("name: AddDocumentText :one").WithArgs(inloc.Bucket, inloc.Key, cleanId, "example").
WillReturnRows(
pgxmock.NewRows([]string{"id"}).
AddRow(database.MustToDBUUID(uuid.New())),
AddRow(textId),
)
pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(textId, pgxmock.AnyArg()).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
mockSQS.EXPECT().
SendMessage(
@@ -93,12 +102,12 @@ func TestCreate(t *testing.T) {
Cleanfailtype: repository.CleanfailtypeInvalidMimetype,
}
pool.ExpectQuery("name: IsDocumentTextExtracted :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows(
pool.ExpectQuery("name: IsDocumentTextExtracted :one").WithArgs(id).WillReturnRows(
pgxmock.NewRows([]string{"isextracted"}).AddRow(false),
)
pool.ExpectQuery("name: GetCleanEntryByDocId :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows(
pool.ExpectQuery("name: GetCleanEntryByDocId :one").WithArgs(id).WillReturnRows(
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "mimetype", "fail"}).
AddRow(database.MustToDBUUID(id), database.MustToDBUUID(uuid.New()), nil, nil, int64(1), nil, fail),
AddRow(id, uuid.New(), nil, nil, int64(1), nil, fail),
)
err = svc.Extract(ctx, id)
internal/document/text/extract.go
+65 -15
View File
@@ -2,9 +2,10 @@ package documenttext
import (
"context"
"database/sql"
"errors"
"fmt"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/document"
"queryorchestration/internal/serviceconfig/build"
@@ -12,37 +13,86 @@ import (
"github.com/google/uuid"
)
func (s *Service) executeExtraction(ctx context.Context, cleanId uuid.UUID) (*document.Location, error) {
entry, err := s.cfg.GetDBQueries().GetCleanEntry(ctx, database.MustToDBUUID(cleanId))
type extraction struct {
TextLocation document.Location
Hash string
}
func (s *Service) executeExtraction(ctx context.Context, cleanId uuid.UUID) (*extraction, error) {
entry, err := s.cfg.GetDBQueries().GetCleanEntry(ctx, cleanId)
if err != nil {
return nil, err
} else if entry.Fail.Valid {
return nil, fmt.Errorf("no valid cleaning")
}
return &document.Location{
Bucket: *entry.Bucket,
Key: *entry.Key,
// analysis, detection - diff?
// split page by page?
// output format
// index
// pages
// block types
return &extraction{
Hash: "example",
TextLocation: document.Location{
Bucket: *entry.Bucket,
Key: *entry.Key,
},
}, nil
}
func (s *Service) extract(ctx context.Context, cleanId uuid.UUID) error {
outLocation, err := s.executeExtraction(ctx, cleanId)
details, err := s.executeExtraction(ctx, cleanId)
if err != nil {
return err
}
version := build.GetVersionUnixTimestamp()
_, err = s.cfg.GetDBQueries().AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Version: version,
Bucket: outLocation.Bucket,
Key: outLocation.Key,
Cleanentryid: database.MustToDBUUID(cleanId),
})
err = s.storeExtraction(ctx, cleanId, details)
if err != nil {
return err
}
return nil
}
func (s *Service) storeExtraction(ctx context.Context, cleanId uuid.UUID, details *extraction) error {
version := build.GetVersionUnixTimestamp()
return s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, q *repository.Queries) error {
existing, err := q.GetDocumentTextExtractionByHash(ctx, &repository.GetDocumentTextExtractionByHashParams{
Hash: details.Hash,
Cleanentryid: cleanId,
})
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return err
}
var textId uuid.UUID
if existing == nil || errors.Is(err, sql.ErrNoRows) {
newTextId, err := q.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Bucket: details.TextLocation.Bucket,
Key: details.TextLocation.Key,
Cleanentryid: cleanId,
Hash: details.Hash,
})
if err != nil {
return err
}
textId = newTextId
} else {
textId = existing.ID
}
err = q.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Version: version,
Textid: textId,
})
if err != nil {
return err
}
return nil
})
}
internal/document/text/extract_test.go
+66 -11
View File
@@ -4,7 +4,6 @@ import (
"context"
"testing"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/document"
@@ -36,7 +35,7 @@ func TestExtract(t *testing.T) {
}
cleanId := uuid.New()
dbCleanId := database.MustToDBUUID(cleanId)
dbCleanId := cleanId
mimeType := "application/pdf"
dbmimetype := repository.NullCleanmimetype{
Valid: true,
@@ -44,13 +43,21 @@ func TestExtract(t *testing.T) {
}
pool.ExpectQuery("name: GetCleanEntry :one").WithArgs(dbCleanId).WillReturnRows(
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "mimetype", "fail"}).
AddRow(dbCleanId, database.MustToDBUUID(id), &inloc.Bucket, &inloc.Key, int64(1), dbmimetype, repository.NullCleanfailtype{}),
AddRow(dbCleanId, id, &inloc.Bucket, &inloc.Key, int64(1), dbmimetype, repository.NullCleanfailtype{}),
)
pool.ExpectQuery("name: AddDocumentTextEntry :one").WithArgs(pgxmock.AnyArg(), inloc.Bucket, inloc.Key, dbCleanId).
textId := uuid.New()
pool.ExpectBegin()
pool.ExpectQuery("name: GetDocumentTextExtractionByHash :one").WithArgs(dbCleanId, "example").WillReturnRows(
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "hash", "cleanEntryId"}),
)
pool.ExpectQuery("name: AddDocumentText :one").WithArgs(inloc.Bucket, inloc.Key, dbCleanId, "example").
WillReturnRows(
pgxmock.NewRows([]string{"id"}).
AddRow(database.MustToDBUUID(uuid.New())),
AddRow(textId),
)
pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(textId, pgxmock.AnyArg()).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
err = svc.extract(ctx, cleanId)
require.NoError(t, err)
@@ -58,14 +65,14 @@ func TestExtract(t *testing.T) {
t.Run("fail clean", func(t *testing.T) {
id := uuid.New()
cleanId := database.MustToDBUUID(uuid.New())
cleanId := uuid.New()
fail := repository.NullCleanfailtype{
Valid: true,
Cleanfailtype: repository.CleanfailtypeInvalidMimetype,
}
pool.ExpectQuery("name: GetCleanEntry :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows(
pool.ExpectQuery("name: GetCleanEntry :one").WithArgs(id).WillReturnRows(
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "mimetype", "fail"}).
AddRow(cleanId, database.MustToDBUUID(id), nil, nil, int64(1), nil, fail),
AddRow(cleanId, id, nil, nil, int64(1), nil, fail),
)
err = svc.extract(ctx, id)
@@ -91,17 +98,65 @@ func TestExecuteExtraction(t *testing.T) {
Bucket: "buck",
Key: "key",
}
extract := extraction{
TextLocation: location,
Hash: "example",
}
dbmimetype := repository.NullCleanmimetype{
Valid: true,
Cleanmimetype: repository.CleanmimetypeApplicationPdf,
}
pool.ExpectQuery("name: GetCleanEntry :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows(
pool.ExpectQuery("name: GetCleanEntry :one").WithArgs(id).WillReturnRows(
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "mimetype", "fail"}).
AddRow(database.MustToDBUUID(uuid.New()), database.MustToDBUUID(id), &location.Bucket, &location.Key, int32(1), dbmimetype, repository.NullCleanfailtype{}),
AddRow(uuid.New(), id, &location.Bucket, &location.Key, int32(1), dbmimetype, repository.NullCleanfailtype{}),
)
outloc, err := svc.executeExtraction(ctx, id)
require.NoError(t, err)
assert.EqualExportedValues(t, location, *outloc)
assert.EqualExportedValues(t, extract, *outloc)
}
func TestStoreExtraction(t *testing.T) {
pool, err := pgxmock.NewPool()
require.NoError(t, err)
cfg := &DocTextConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := Service{
cfg: cfg,
}
ctx := context.Background()
t.Run("not existing", func(t *testing.T) {
cleanId := uuid.New()
dbCleanId := cleanId
textId := uuid.New()
extract := extraction{
TextLocation: document.Location{
Bucket: "buck",
Key: "key",
},
Hash: "example",
}
pool.ExpectBegin()
pool.ExpectQuery("name: GetDocumentTextExtractionByHash :one").WithArgs(dbCleanId, extract.Hash).WillReturnRows(
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "hash", "cleanEntryId"}),
)
pool.ExpectQuery("name: AddDocumentText :one").WithArgs(extract.TextLocation.Bucket, extract.TextLocation.Key, dbCleanId, "example").
WillReturnRows(
pgxmock.NewRows([]string{"id"}).
AddRow(textId),
)
pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(textId, pgxmock.AnyArg()).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
err = svc.storeExtraction(ctx, cleanId, &extract)
require.NoError(t, err)
})
}
internal/document/text/service.go
+2
View File
@@ -3,11 +3,13 @@ package documenttext
import (
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/queue/querysync"
"queryorchestration/internal/serviceconfig/textract"
)
type ConfigProvider interface {
serviceconfig.ConfigProvider
querysync.ConfigProvider
textract.ConfigProvider
}
type Service struct {
internal/document/text/service_test.go
+2
View File
@@ -6,6 +6,7 @@ import (
documenttext "queryorchestration/internal/document/text"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/queue/querysync"
"queryorchestration/internal/serviceconfig/textract"
"github.com/stretchr/testify/assert"
)
@@ -13,6 +14,7 @@ import (
type DocTextConfig struct {
serviceconfig.BaseConfig
querysync.QuerySyncConfig
textract.TextractConfig
}
func TestService(t *testing.T) {