Merged in feature/postprocessing (pull request #114)

Feature/postprocessing * tests * passtest * fixshorttests * mosttests * improvingbasedockerfile * testspeeds * testing * host * canparallel * clean * passfullsuite * singlepagemax * test * findfeatures * findstables * tbls * tablestoo * tablestoo * lateraltests * tableloc * cleanup * inlinetable * childids * cleanup * tests
2025-04-22 14:40:16 +00:00
parent edc50c7510
commit fee71e7740
404 changed files with 211048 additions and 2820 deletions
@@ -0,0 +1,102 @@
+package documenttext
+
+import (
+	"context"
+	"io"
+	"strings"
+	"testing"
+	"time"
+
+	"queryorchestration/internal/database/repository"
+	"queryorchestration/internal/serviceconfig/objectstore"
+	objectstoremock "queryorchestration/mocks/objectstore"
+	textractmock "queryorchestration/mocks/textract"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/textract"
+	"github.com/google/uuid"
+	"github.com/pashagolub/pgxmock/v3"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestExecuteExtract(t *testing.T) {
+	ctx := context.Background()
+	pool, err := pgxmock.NewPool()
+	require.NoError(t, err)
+
+	cfg := &DocTextConfig{}
+	cfg.DBPool = pool
+	cfg.DBQueries = repository.New(pool)
+	mockStore := objectstoremock.NewMockS3Client(t)
+	cfg.StoreClient = mockStore
+	mockTextract := textractmock.NewMockTextractClient(t)
+	cfg.TextractClient = mockTextract
+
+	svc := Service{
+		cfg: cfg,
+	}
+
+	cleanId := uuid.New()
+	docId := uuid.New()
+	bucket := "bucket"
+	hash := `"a06d5e0e795adeb6d0b3732330dbcc15"`
+	textId := uuid.New()
+	key := objectstore.BucketKey{
+		ClientID:  "aa",
+		Location:  objectstore.Import,
+		CreatedAt: time.Now().UTC(),
+		EntityID:  cleanId,
+	}
+	keyStr := key.String()
+	textractFile := "../../../assets/sampleGeneration/generated/helloWorld.gen"
+	textractBaseOut := getTextractFileResponse(t, textractFile)
+
+	mimetype := repository.NullCleanmimetype{
+		Valid:         true,
+		Cleanmimetype: repository.CleanmimetypeApplicationPdf,
+	}
+	pool.ExpectQuery("name: GetCleanEntryByDocId :one").WithArgs(docId).
+		WillReturnRows(
+			pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "hash", "mimetype", "fail", "clientId"}).
+				AddRow(cleanId, docId, &bucket, &keyStr, int64(123), &hash, mimetype, nil, key.ClientID),
+		)
+	bodyMsg := io.NopCloser(strings.NewReader(pdfHelloWorld))
+	mockStore.EXPECT().
+		GetObject(
+			mock.Anything,
+			mock.MatchedBy(func(in *s3.GetObjectInput) bool {
+				return *in.Bucket == bucket && *in.Key == keyStr
+			}),
+			mock.Anything,
+		).
+		Return(&s3.GetObjectOutput{
+			Body: bodyMsg,
+		}, nil)
+	sent := 0
+	mockTextract.EXPECT().
+		AnalyzeDocument(
+			mock.Anything,
+			mock.MatchedBy(func(in *textract.AnalyzeDocumentInput) bool {
+				if sent == 0 {
+					sent++
+					return true
+				}
+				return false
+			}),
+			mock.Anything,
+		).
+		Return(&textractBaseOut["base"][0], nil).Once()
+
+	pool.ExpectBegin()
+	pool.ExpectQuery("name: GetDocumentTextExtractionByHash :one").WithArgs(cleanId, hash).WillReturnRows(
+		pgxmock.NewRows([]string{"id"}).
+			AddRow(textId),
+	)
+	pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(textId, pgxmock.AnyArg()).
+		WillReturnResult(pgxmock.NewResult("", 1))
+	pool.ExpectCommit()
+
+	err = svc.executeExtraction(ctx, docId)
+	require.NoError(t, err)
+}