jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/part (pull request #113)

Browse Source 
Size Import Parts

* parts

* think

* workingpart

* textout
This commit is contained in:
Michael McGuinness
2025-04-08 12:44:09 +00:00
parent 60812aba91
commit edc50c7510
21 changed files with 873 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/document/text/extract_test.go
+6 -1
View File
@@ -78,7 +78,12 @@ func TestCreate(t *testing.T) {
jobId := "hello"
triggerId := uuid.New()
pool.ExpectBegin()
pool.ExpectQuery("name: AddDocumentTextTrigger :one").WithArgs(cleanId, pgxmock.AnyArg()).WillReturnRows(
pool.ExpectQuery("name: GetTextractOutputCurrentPart :one").WithArgs(&clientId, pgxmock.AnyArg()).
WillReturnRows(
pgxmock.NewRows([]string{"part", "count"}).
AddRow(0, 2),
)
pool.ExpectQuery("name: AddDocumentTextTrigger :one").WithArgs(cleanId, pgxmock.AnyArg(), pgxmock.AnyArg(), uint16(0)).WillReturnRows(
pgxmock.NewRows([]string{"id"}).AddRow(triggerId),
)
internal/document/text/processExtract.go
+21
View File
@@ -15,6 +15,7 @@ import (
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
)
type ProcessParams struct {
@@ -103,12 +104,32 @@ func (s *Service) storeProcess(ctx context.Context, text io.Reader, trigger *rep
CreatedAt: createdAt,
EntityID: trigger.ID,
}
currentPart, err := q.GetTextOutCurrentPart(ctx, &repository.GetTextOutCurrentPartParams{
Clientid: &key.ClientID,
Querydate: pgtype.Timestamptz{
Time: key.CreatedAt,
Valid: true,
},
})
if err != nil {
return err
}
part := s.cfg.GetDirectoryPart(currentPart.Part, currentPart.Count)
key.Part = &part
keyStr := key.String()
textId, err = q.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Bucket: params.Bucket,
Key: keyStr,
Hash: hash,
Part: *key.Part,
Createdat: pgtype.Timestamp{
Time: key.CreatedAt,
Valid: true,
},
})
if err != nil {
return err
internal/document/text/processExtract_test.go
+14 -2
View File
@@ -143,17 +143,24 @@ func TestStoreTrigger(t *testing.T) {
require.NoError(t, err)
})
t.Run("not exists", func(t *testing.T) {
part := uint16(0)
key := objectstore.BucketKey{
ClientID: clientId,
Location: objectstore.TextOut,
CreatedAt: time.Now().UTC(),
EntityID: triggerId,
Part: &part,
}
hash := `"09644372e99020106946045c6fd2d70b"`
pool.ExpectBegin()
pool.ExpectQuery("name: GetDocumentTextExtractionByHash :one").WithArgs(cleanId, hash).
WillReturnError(sql.ErrNoRows)
pool.ExpectQuery("name: AddDocumentText :one").WithArgs(bucket, key.String(), hash).
pool.ExpectQuery("name: GetTextOutCurrentPart :one").WithArgs(&clientId, pgxmock.AnyArg()).
WillReturnRows(
pgxmock.NewRows([]string{"part", "count"}).
AddRow(0, 2),
)
pool.ExpectQuery("name: AddDocumentText :one").WithArgs(bucket, key.String(), hash, pgxmock.AnyArg(), uint16(0)).
WillReturnRows(
pgxmock.NewRows([]string{"id"}).
AddRow(textId),
@@ -233,7 +240,12 @@ func TestProcessTrigger(t *testing.T) {
pool.ExpectBegin()
pool.ExpectQuery("name: GetDocumentTextExtractionByHash :one").WithArgs(cleanId, hash).
WillReturnError(sql.ErrNoRows)
pool.ExpectQuery("name: AddDocumentText :one").WithArgs(bucket, key.String(), hash).
pool.ExpectQuery("name: GetTextOutCurrentPart :one").WithArgs(&clientId, pgxmock.AnyArg()).
WillReturnRows(
pgxmock.NewRows([]string{"part", "count"}).
AddRow(0, 2),
)
pool.ExpectQuery("name: AddDocumentText :one").WithArgs(bucket, key.String(), hash, pgxmock.AnyArg(), uint16(0)).
WillReturnRows(
pgxmock.NewRows([]string{"id"}).
AddRow(textId),
internal/document/text/triggerExtract.go
+30 -10
View File
@@ -11,28 +11,48 @@ import (
"github.com/aws/aws-sdk-go-v2/service/textract"
"github.com/aws/aws-sdk-go-v2/service/textract/types"
"github.com/jackc/pgx/v5/pgtype"
)
func (s *Service) triggerExtract(ctx context.Context, clean *repository.Currentcleanentry) error {
version := build.GetVersionUnixTimestamp()
return s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, q *repository.Queries) error {
triggerId, err := s.cfg.GetDBQueries().AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
Cleanid: clean.ID,
Version: version,
key := objectstore.BucketKey{
CreatedAt: time.Now().UTC(),
Location: objectstore.TextTextract,
ClientID: clean.Clientid,
}
currentPart, err := q.GetTextractOutputCurrentPart(ctx, &repository.GetTextractOutputCurrentPartParams{
Clientid: &clean.Clientid,
Querydate: pgtype.Timestamptz{
Time: key.CreatedAt,
Valid: true,
},
})
if err != nil {
return err
}
key := objectstore.BucketKey{
CreatedAt: time.Now().UTC(),
Location: objectstore.TextTextract,
ClientID: clean.Clientid,
EntityID: triggerId,
}
keyStr := key.String()
part := s.cfg.GetDirectoryPart(currentPart.Part, currentPart.Count)
key.Part = &part
triggerId, err := s.cfg.GetDBQueries().AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
Cleanid: clean.ID,
Version: version,
Createdat: pgtype.Timestamp{
Time: key.CreatedAt,
Valid: true,
},
Part: *key.Part,
})
if err != nil {
return err
}
key.EntityID = triggerId
keyStr := key.String()
jobTag := triggerId.String()
res, err := s.cfg.GetTextractClient().StartDocumentTextDetection(ctx, &textract.StartDocumentTextDetectionInput{
internal/document/text/triggerExtract_test.go
+9 -2
View File
@@ -38,11 +38,18 @@ func TestTriggerExtract(t *testing.T) {
bucket := "bucket_name"
key := "/i/am/here"
hash := "hhasshhh"
clientId := "AA"
jobId := "hello"
triggerId := uuid.New()
pool.ExpectBegin()
pool.ExpectQuery("name: AddDocumentTextTrigger :one").WithArgs(cleanId, pgxmock.AnyArg()).WillReturnRows(
pool.ExpectQuery("name: GetTextractOutputCurrentPart :one").WithArgs(&clientId, pgxmock.AnyArg()).
WillReturnRows(
pgxmock.NewRows([]string{"part", "count"}).
AddRow(0, 2),
)
pool.ExpectQuery("name: AddDocumentTextTrigger :one").WithArgs(cleanId, pgxmock.AnyArg(), pgxmock.AnyArg(), uint16(0)).WillReturnRows(
pgxmock.NewRows([]string{"id"}).AddRow(triggerId),
)
@@ -64,7 +71,7 @@ func TestTriggerExtract(t *testing.T) {
err = svc.triggerExtract(ctx, &repository.Currentcleanentry{
ID: cleanId,
Clientid: "AA",
Clientid: clientId,
Key: &key,
Bucket: &bucket,
Hash: &hash,