Merged in feature/checkbox (pull request #145)

Checkbox Primary Edge Cases * hascheckboxlogic * gen * preppinggen * exploringcheckbox * removeunselected * tests * failingtest * failintests * nomockqueryapi * db * name * nocontainer * deleterow * cnc * extradocsandupdatetextracts * moretables * appndedtables * baseversion
2025-05-20 12:47:22 +00:00
parent beb221937b
commit 61d12bf8df
69 changed files with 223611 additions and 78482 deletions
@@ -1,7 +1,6 @@
 package doctextrunner_test

 import (
-	"context"
 	"fmt"
 	"io"
 	"strings"
@@ -25,7 +24,6 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/sqs"
 	awstextract "github.com/aws/aws-sdk-go-v2/service/textract"
 	"github.com/google/uuid"
-	"github.com/pashagolub/pgxmock/v3"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
@@ -39,14 +37,12 @@ type DocTextConfig struct {
 	objectstore.ObjectStoreConfig
 }

-func TestDocCleanRunner(t *testing.T) {
-	ctx := context.Background()
-	pool, err := pgxmock.NewPool()
-	require.NoError(t, err)
-
+func TestDocTextRunner(t *testing.T) {
+	t.Parallel()
 	cfg := &DocTextConfig{}
-	cfg.DBPool = pool
-	cfg.DBQueries = repository.New(pool)
+	net := test.GetNetwork(t)
+	test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{})
+
 	mockStore := objectstoremock.NewMockS3Client(t)
 	cfg.StoreClient = mockStore
 	mockTextract := textractmock.NewMockTextractClient(t)
@@ -58,84 +54,99 @@ func TestDocCleanRunner(t *testing.T) {
 	runner := doctextrunner.New(&doctextrunner.Services{
 		Text: documenttext.New(cfg),
 	})
-	doc := doctextrunner.Body{
-		DocumentID: uuid.New(),
-	}

-	cleanId := uuid.New()
-	bucket := "bucket"
-	hash := `"a06d5e0e795adeb6d0b3732330dbcc15"`
-	textId := uuid.New()
+	err := cfg.GetDBQueries().CreateClient(t.Context(), &repository.CreateClientParams{
+		Clientid: "clientid",
+		Name:     "client_name",
+	})
+	require.NoError(t, err)
+	docId, err := cfg.GetDBQueries().CreateDocument(t.Context(), &repository.CreateDocumentParams{
+		Clientid: "clientid",
+		Hash:     "hash",
+	})
+	require.NoError(t, err)
+	part := uint16(1)
+	filetype := "pdf"
 	key := objectstore.BucketKey{
-		ClientID:  "aa",
-		Location:  objectstore.Import,
 		CreatedAt: time.Now().UTC(),
-		EntityID:  cleanId,
+		ClientID:  "clientid",
+		EntityID:  uuid.New(),
+		Location:  objectstore.Import,
+		Part:      &part,
+		FileType:  &filetype,
 	}
+	err = cfg.GetDBQueries().AddDocumentEntry(t.Context(), &repository.AddDocumentEntryParams{
+		Documentid: docId,
+		Bucket:     "bucket",
+		Key:        key.String(),
+	})
+	require.NoError(t, err)
+	bucket := "bucket"
+	hash := "hash"
 	keyStr := key.String()
-	filename := "../../assets/textract/helloWorld.gen"
-	textractBaseOut := test.GetTextractFileResponse(t, filename)
+	cleanId, err := cfg.GetDBQueries().AddDocumentClean(t.Context(), &repository.AddDocumentCleanParams{
+		Documentid: docId,
+		Bucket:     &bucket,
+		Hash:       &hash,
+		Key:        &keyStr,
+		Mimetype: repository.NullCleanmimetype{
+			Valid:         true,
+			Cleanmimetype: repository.CleanmimetypeApplicationPdf,
+		},
+	})
+	require.NoError(t, err)
+	err = cfg.GetDBQueries().AddDocumentCleanEntry(t.Context(), &repository.AddDocumentCleanEntryParams{
+		Cleanid: cleanId,
+		Version: 1,
+	})
+	require.NoError(t, err)

-	mimetype := repository.NullCleanmimetype{
-		Valid:         true,
-		Cleanmimetype: repository.CleanmimetypeApplicationPdf,
-	}
-	pool.ExpectQuery("name: IsDocumentTextExtracted :one").WithArgs(doc.DocumentID).WillReturnRows(
-		pgxmock.NewRows([]string{"isextracted"}).
-			AddRow(false),
-	)
-	pool.ExpectQuery("name: GetCleanEntryByDocId :one").WithArgs(doc.DocumentID).
-		WillReturnRows(
-			pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "hash", "mimetype", "fail", "clientId"}).
-				AddRow(cleanId, doc.DocumentID, &bucket, &keyStr, int64(123), &hash, mimetype, nil, key.ClientID),
-		)
 	bodyMsg := io.NopCloser(strings.NewReader(pdfHelloWorld))
 	mockStore.EXPECT().
 		GetObject(
 			mock.Anything,
 			mock.MatchedBy(func(in *s3.GetObjectInput) bool {
-				return *in.Bucket == bucket && *in.Key == keyStr
+				return *in.Bucket == "bucket" && *in.Key == key.String()
 			}),
 			mock.Anything,
 		).
 		Return(&s3.GetObjectOutput{
 			Body: bodyMsg,
 		}, nil)
-	sent := 0
+	filename := "../../assets/textract/helloWorld.gen"
+	textractBaseOut := test.GetTextractFileResponse(t, filename)
 	mockTextract.EXPECT().
 		AnalyzeDocument(
 			mock.Anything,
 			mock.MatchedBy(func(in *awstextract.AnalyzeDocumentInput) bool {
-				if sent == 0 {
-					sent++
-					return true
-				}
-				return false
+				return true
 			}),
 			mock.Anything,
 		).
 		Return(textractBaseOut[0]["base"], nil).Once()
-
-	pool.ExpectBegin()
-	pool.ExpectQuery("name: GetDocumentTextExtractionByHash :one").WithArgs(cleanId, hash).WillReturnRows(
-		pgxmock.NewRows([]string{"id"}).
-			AddRow(textId),
-	)
-	pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(textId, pgxmock.AnyArg()).
-		WillReturnResult(pgxmock.NewResult("", 1))
-	pool.ExpectCommit()
-
+	mockStore.EXPECT().
+		PutObject(
+			mock.Anything,
+			mock.MatchedBy(func(in *s3.PutObjectInput) bool {
+				return *in.Bucket == "bucket"
+			}),
+			mock.Anything,
+		).
+		Return(&s3.PutObjectOutput{}, nil)
 	mockQueue.EXPECT().
 		SendMessage(
 			mock.Anything,
 			mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
-				return *in.QueueUrl == cfg.QuerySyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", doc.DocumentID.String())
+				return *in.QueueUrl == cfg.QuerySyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", docId.String())
 			}),
 			mock.Anything,
 		).
 		Return(&sqs.SendMessageOutput{}, nil)

-	assert.True(t, runner.Process(ctx, doc))
+	doc := doctextrunner.Body{
+		DocumentID: docId,
+	}
+	assert.True(t, runner.Process(t.Context(), doc))
 }

 const pdfHelloWorld = `%PDF-1.4