Merged in feature/postprocessing (pull request #114)
Feature/postprocessing * tests * passtest * fixshorttests * mosttests * improvingbasedockerfile * testspeeds * testing * host * canparallel * clean * passfullsuite * singlepagemax * test * findfeatures * findstables * tbls * tablestoo * tablestoo * lateraltests * tableloc * cleanup * inlinetable * childids * cleanup * tests
This commit is contained in:
Michael McGuinness
@@ -6,7 +6,6 @@ import (
|
||||
|
||||
documenttext "queryorchestration/internal/document/text"
|
||||
|
||||
"github.com/go-playground/validator/v10"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
@@ -17,14 +16,12 @@ type Services struct {
|
||||
}
|
||||
|
||||
type Runner struct {
|
||||
validator *validator.Validate
|
||||
svc *Services
|
||||
svc *Services
|
||||
}
|
||||
|
||||
func New(validator *validator.Validate, svc *Services) Runner {
|
||||
func New(svc *Services) Runner {
|
||||
return Runner{
|
||||
validator: validator,
|
||||
svc: svc,
|
||||
svc: svc,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,11 @@ package doctextrunner_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -17,8 +22,9 @@ import (
|
||||
queuemock "queryorchestration/mocks/queue"
|
||||
textractmock "queryorchestration/mocks/textract"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/sqs"
|
||||
awstextract "github.com/aws/aws-sdk-go-v2/service/textract"
|
||||
"github.com/go-playground/validator/v10"
|
||||
"github.com/google/uuid"
|
||||
"github.com/pashagolub/pgxmock/v3"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -36,84 +42,169 @@ type DocTextConfig struct {
|
||||
|
||||
func TestDocCleanRunner(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
pool, err := pgxmock.NewPool()
|
||||
require.NoError(t, err)
|
||||
|
||||
cfg := &DocTextConfig{}
|
||||
cfg.DBPool = pool
|
||||
cfg.DBQueries = repository.New(pool)
|
||||
mockSQS := queuemock.NewMockSQSClient(t)
|
||||
cfg.QueueClient = mockSQS
|
||||
cfg.QuerySyncURL = "/i/am/here"
|
||||
mockS3 := objectstoremock.NewMockS3Client(t)
|
||||
cfg.StoreClient = mockS3
|
||||
mockStore := objectstoremock.NewMockS3Client(t)
|
||||
cfg.StoreClient = mockStore
|
||||
mockTextract := textractmock.NewMockTextractClient(t)
|
||||
cfg.TextractClient = mockTextract
|
||||
mockQueue := queuemock.NewMockSQSClient(t)
|
||||
cfg.QueueClient = mockQueue
|
||||
cfg.QuerySyncURL = "ex"
|
||||
|
||||
runner := doctextrunner.New(validator.New(), &doctextrunner.Services{
|
||||
runner := doctextrunner.New(&doctextrunner.Services{
|
||||
Text: documenttext.New(cfg),
|
||||
})
|
||||
assert.NotNil(t, runner)
|
||||
doc := doctextrunner.Body{
|
||||
DocumentID: uuid.New(),
|
||||
}
|
||||
|
||||
t.Run("valid", func(t *testing.T) {
|
||||
doc := doctextrunner.Body{
|
||||
DocumentID: uuid.New(),
|
||||
}
|
||||
cleanId := uuid.New()
|
||||
bucket := "bucket"
|
||||
hash := `"a06d5e0e795adeb6d0b3732330dbcc15"`
|
||||
textId := uuid.New()
|
||||
key := objectstore.BucketKey{
|
||||
ClientID: "aa",
|
||||
Location: objectstore.Import,
|
||||
CreatedAt: time.Now().UTC(),
|
||||
EntityID: cleanId,
|
||||
}
|
||||
keyStr := key.String()
|
||||
filename := "../../assets/sampleGeneration/generated/helloWorld.gen"
|
||||
file, err := os.Open(filename)
|
||||
require.NoError(t, err)
|
||||
defer file.Close()
|
||||
|
||||
cleanId := uuid.New()
|
||||
bucket := "hello"
|
||||
key := objectstore.BucketKey{
|
||||
CreatedAt: time.Now().UTC(),
|
||||
ClientID: "hi",
|
||||
EntityID: uuid.New(),
|
||||
Location: objectstore.Import,
|
||||
}
|
||||
keyStr := key.String()
|
||||
hash := "example"
|
||||
decoder := json.NewDecoder(file)
|
||||
var textractBaseOut map[string][]awstextract.AnalyzeDocumentOutput
|
||||
err = decoder.Decode(&textractBaseOut)
|
||||
require.NoError(t, err)
|
||||
|
||||
pool.ExpectQuery("name: IsDocumentTextExtracted :one").WithArgs(doc.DocumentID).WillReturnRows(
|
||||
pgxmock.NewRows([]string{"isextracted"}).
|
||||
AddRow(false),
|
||||
mimetype := repository.NullCleanmimetype{
|
||||
Valid: true,
|
||||
Cleanmimetype: repository.CleanmimetypeApplicationPdf,
|
||||
}
|
||||
pool.ExpectQuery("name: IsDocumentTextExtracted :one").WithArgs(doc.DocumentID).WillReturnRows(
|
||||
pgxmock.NewRows([]string{"isextracted"}).
|
||||
AddRow(false),
|
||||
)
|
||||
pool.ExpectQuery("name: GetCleanEntryByDocId :one").WithArgs(doc.DocumentID).
|
||||
WillReturnRows(
|
||||
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "hash", "mimetype", "fail", "clientId"}).
|
||||
AddRow(cleanId, doc.DocumentID, &bucket, &keyStr, int64(123), &hash, mimetype, nil, key.ClientID),
|
||||
)
|
||||
mimeType := "application/pdf"
|
||||
dbmimetype := repository.NullCleanmimetype{
|
||||
Valid: true,
|
||||
Cleanmimetype: repository.Cleanmimetype(mimeType),
|
||||
}
|
||||
pool.ExpectQuery("name: GetCleanEntryByDocId :one").WithArgs(doc.DocumentID).WillReturnRows(
|
||||
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "hash", "mimetype", "fail", "externalClientId"}).
|
||||
AddRow(cleanId, doc.DocumentID, &bucket, &keyStr, int64(1), &hash, dbmimetype, repository.NullCleanfailtype{}, key.ClientID),
|
||||
)
|
||||
jobId := "hello"
|
||||
triggerId := uuid.New()
|
||||
pool.ExpectBegin()
|
||||
pool.ExpectQuery("name: GetTextractOutputCurrentPart :one").WithArgs(&key.ClientID, pgxmock.AnyArg()).
|
||||
WillReturnRows(
|
||||
pgxmock.NewRows([]string{"part", "count"}).
|
||||
AddRow(0, 2),
|
||||
)
|
||||
|
||||
pool.ExpectQuery("name: AddDocumentTextTrigger :one").WithArgs(cleanId, pgxmock.AnyArg(), pgxmock.AnyArg(), uint16(0)).WillReturnRows(
|
||||
pgxmock.NewRows([]string{"id"}).AddRow(triggerId),
|
||||
)
|
||||
|
||||
mockTextract.EXPECT().
|
||||
StartDocumentTextDetection(
|
||||
mock.Anything,
|
||||
mock.MatchedBy(func(in *awstextract.StartDocumentTextDetectionInput) bool {
|
||||
bodyMsg := io.NopCloser(strings.NewReader(pdfHelloWorld))
|
||||
mockStore.EXPECT().
|
||||
GetObject(
|
||||
mock.Anything,
|
||||
mock.MatchedBy(func(in *s3.GetObjectInput) bool {
|
||||
return *in.Bucket == bucket && *in.Key == keyStr
|
||||
}),
|
||||
mock.Anything,
|
||||
).
|
||||
Return(&s3.GetObjectOutput{
|
||||
Body: bodyMsg,
|
||||
}, nil)
|
||||
sent := 0
|
||||
mockTextract.EXPECT().
|
||||
AnalyzeDocument(
|
||||
mock.Anything,
|
||||
mock.MatchedBy(func(in *awstextract.AnalyzeDocumentInput) bool {
|
||||
if sent == 0 {
|
||||
sent++
|
||||
return true
|
||||
}),
|
||||
mock.Anything,
|
||||
).
|
||||
Return(&awstextract.StartDocumentTextDetectionOutput{
|
||||
JobId: &jobId,
|
||||
}, nil)
|
||||
}
|
||||
return false
|
||||
}),
|
||||
mock.Anything,
|
||||
).
|
||||
Return(&textractBaseOut["base"][0], nil).Once()
|
||||
|
||||
pool.ExpectExec("name: AddDocumentTextTriggerJobId :exec").WithArgs(&jobId, triggerId).
|
||||
WillReturnResult(pgxmock.NewResult("", 1))
|
||||
pool.ExpectCommit()
|
||||
pool.ExpectBegin()
|
||||
pool.ExpectQuery("name: GetDocumentTextExtractionByHash :one").WithArgs(cleanId, hash).WillReturnRows(
|
||||
pgxmock.NewRows([]string{"id"}).
|
||||
AddRow(textId),
|
||||
)
|
||||
pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(textId, pgxmock.AnyArg()).
|
||||
WillReturnResult(pgxmock.NewResult("", 1))
|
||||
pool.ExpectCommit()
|
||||
|
||||
assert.True(t, runner.Process(ctx, doc))
|
||||
})
|
||||
mockQueue.EXPECT().
|
||||
SendMessage(
|
||||
mock.Anything,
|
||||
mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
|
||||
return *in.QueueUrl == cfg.QuerySyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", doc.DocumentID.String())
|
||||
}),
|
||||
mock.Anything,
|
||||
).
|
||||
Return(&sqs.SendMessageOutput{}, nil)
|
||||
|
||||
assert.True(t, runner.Process(ctx, doc))
|
||||
}
|
||||
|
||||
const pdfHelloWorld = `%PDF-1.4
|
||||
%����
|
||||
1 0 obj
|
||||
<<
|
||||
/Type /Catalog
|
||||
/Pages 2 0 R
|
||||
/Version /1.4
|
||||
>>
|
||||
endobj
|
||||
2 0 obj
|
||||
<<
|
||||
/Type /Pages
|
||||
/Kids [3 0 R]
|
||||
/Count 1
|
||||
>>
|
||||
endobj
|
||||
3 0 obj
|
||||
<<
|
||||
/Type /Page
|
||||
/Parent 2 0 R
|
||||
/MediaBox [0 0 612 792]
|
||||
/Resources <<
|
||||
/Font <<
|
||||
/F1 <<
|
||||
/Type /Font
|
||||
/Subtype /Type1
|
||||
/BaseFont /Helvetica
|
||||
>>
|
||||
>>
|
||||
>>
|
||||
/Contents 4 0 R
|
||||
>>
|
||||
endobj
|
||||
4 0 obj
|
||||
<<
|
||||
/Length
|
||||
44
|
||||
>>
|
||||
stream
|
||||
BT
|
||||
/F1 24 Tf
|
||||
100 700 Td
|
||||
(Hello World!) Tj
|
||||
ET
|
||||
endstream
|
||||
endobj
|
||||
xref
|
||||
0 5
|
||||
0000000000 65535 f
|
||||
0000000015 00000 n
|
||||
0000000086 00000 n
|
||||
0000000151 00000 n
|
||||
0000000376 00000 n
|
||||
trailer
|
||||
<<
|
||||
/Size 5
|
||||
/Root 1 0 R
|
||||
>>
|
||||
startxref
|
||||
472
|
||||
%%EOF`
|
||||
|
||||
Reference in New Issue
Block a user