Merged in feature/splitqueryrunning (pull request #57)

Split Query Running + Debugging Full Flow

* completedquerysyncrunner

* spliitinglogic

* synccomplete

* informdependents

* only push same collector

* deps

* livetesting

* foundissue

* some issues resolved

* activeupdate

* collectorupdatefixes

* fix dbquesries

* tests

* tests

* pollingdebug

internal/document/clean/clean.go

@@ -2,6 +2,7 @@ package documentclean
 
 import (
 	"context"
+	"log/slog"
 	"queryorchestration/internal/database"
 	"queryorchestration/internal/database/repository"
 	"queryorchestration/internal/document"
@@ -20,6 +21,8 @@ func (s *Service) executeCleanTasks(params *CleanParams) (*document.Location, er
 }
 
 func (s *Service) clean(ctx context.Context, id uuid.UUID) error {
+	slog.Debug("cleaning document", "id", id.String())
+
 	docId := database.MustToDBUUID(id)
 
 	entry, err := s.cfg.GetDBQueries().GetDocumentEntry(ctx, docId)

internal/document/clean/create.go

@@ -9,7 +9,7 @@ import (
 	"github.com/google/uuid"
 )
 
-func (s *Service) Create(ctx context.Context, id uuid.UUID) error {
+func (s *Service) Clean(ctx context.Context, id uuid.UUID) error {
 	isclean, err := s.cfg.GetDBQueries().IsDocumentClean(ctx, database.MustToDBUUID(id))
 	if err != nil {
 		return err
@@ -33,7 +33,7 @@ func (s *Service) Create(ctx context.Context, id uuid.UUID) error {
 func (s *Service) informClean(ctx context.Context, id uuid.UUID) error {
 	err := s.cfg.SendToQueue(ctx, &queue.SendParams{
 		QueueURL: s.cfg.GetDocumentTextURL(),
-		Body: doctextrunner.Create{
+		Body: doctextrunner.Body{
 			ID: id,
 		},
 	})

internal/document/clean/create_test.go

@@ -74,7 +74,7 @@ func TestCreate(t *testing.T) {
 		).
 		Return(&sqs.SendMessageOutput{}, nil)
 
-	err = svc.Create(ctx, id)
+	err = svc.Clean(ctx, id)
 	assert.NoError(t, err)
 }
 

internal/document/init/create.go

@@ -5,6 +5,7 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
+	"log/slog"
 	doccleanrunner "queryorchestration/api/docCleanRunner"
 	"queryorchestration/internal/database"
 	"queryorchestration/internal/database/repository"
@@ -90,9 +91,12 @@ func (s *Service) submitCreate(ctx context.Context, params *createDocumentParams
 			if err != nil {
 				return err
 			}
+			slog.Debug("document created", "id", createid.String(), "job", params.JobID.String())
 
 			dbid = createid
 		} else {
+			slog.Debug("document exists", "id", params.ID.String(), "job", params.JobID.String())
+
 			dbid = *params.ID
 		}
 
@@ -123,7 +127,7 @@ func (s *Service) informCreate(ctx context.Context, id uuid.UUID, j *job.Job) er
 
 	err := s.cfg.SendToQueue(ctx, &queue.SendParams{
 		QueueURL: s.cfg.GetDocumentCleanURL(),
-		Body: doccleanrunner.Create{
+		Body: doccleanrunner.Body{
 			ID: id,
 		},
 	})

internal/document/text/create.go

@@ -2,6 +2,7 @@ package documenttext
 
 import (
 	"context"
+	"log/slog"
 	querysyncrunner "queryorchestration/api/querySyncRunner"
 	"queryorchestration/internal/database"
 	"queryorchestration/internal/serviceconfig/queue"
@@ -9,7 +10,9 @@ import (
 	"github.com/google/uuid"
 )
 
-func (s *Service) Create(ctx context.Context, id uuid.UUID) error {
+func (s *Service) Extract(ctx context.Context, id uuid.UUID) error {
+	slog.Debug("extracting document text", "id", id.String())
+
 	isextracted, err := s.cfg.GetDBQueries().IsDocumentTextExtracted(ctx, database.MustToDBUUID(id))
 	if err != nil {
 		return err
@@ -33,7 +36,7 @@ func (s *Service) Create(ctx context.Context, id uuid.UUID) error {
 func (s *Service) informExtraction(ctx context.Context, id uuid.UUID) error {
 	err := s.cfg.SendToQueue(ctx, &queue.SendParams{
 		QueueURL: s.cfg.GetQuerySyncURL(),
-		Body: querysyncrunner.Create{
+		Body: querysyncrunner.Body{
 			ID: id,
 		},
 	})

internal/document/text/create_test.go

@@ -58,8 +58,8 @@ func TestCreate(t *testing.T) {
 			AddRow(false),
 	)
 	pool.ExpectQuery("name: GetDocumentCleanEntry :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows(
-		pgxmock.NewRows([]string{"documentId", "bucket", "key"}).
-			AddRow(database.MustToDBUUID(id), inloc.Bucket, inloc.Key),
+		pgxmock.NewRows([]string{"documentId", "bucket", "key", "version"}).
+			AddRow(database.MustToDBUUID(id), inloc.Bucket, inloc.Key, int32(1)),
 	)
 	pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(database.MustToDBUUID(id), int32(1), inloc.Bucket, inloc.Key).
 		WillReturnResult(pgxmock.NewResult("", 1))
@@ -74,7 +74,7 @@ func TestCreate(t *testing.T) {
 		).
 		Return(&sqs.SendMessageOutput{}, nil)
 
-	err = svc.Create(ctx, id)
+	err = svc.Extract(ctx, id)
 	assert.NoError(t, err)
 }
 

internal/document/text/extract_test.go

@@ -37,8 +37,8 @@ func TestExtract(t *testing.T) {
 	}
 
 	pool.ExpectQuery("name: GetDocumentCleanEntry :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows(
-		pgxmock.NewRows([]string{"documentId", "bucket", "key"}).
-			AddRow(database.MustToDBUUID(id), inloc.Bucket, inloc.Key),
+		pgxmock.NewRows([]string{"documentId", "bucket", "key", "version"}).
+			AddRow(database.MustToDBUUID(id), inloc.Bucket, inloc.Key, int32(1)),
 	)
 	pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(database.MustToDBUUID(id), int32(1), inloc.Bucket, inloc.Key).
 		WillReturnResult(pgxmock.NewResult("", 1))

internal/document/version.go

@@ -2,21 +2,8 @@ package document
 
 import (
 	"fmt"
-
-	"github.com/google/uuid"
 )
 
-type IsTextExtractedParams struct {
-	DocumentID      uuid.UUID
-	MinCleanVersion int32
-	MinTextVersion  int32
-}
-
-func (s *Service) IsTextExtracted(params *IsTextExtractedParams) error {
-	// TODO
-	return nil
-}
-
 func (s *Service) GetCleanVersion() int32 {
 	// TODO - actual version
 	return 1

internal/document/version_test.go

@@ -5,7 +5,6 @@ import (
 	"queryorchestration/internal/serviceconfig"
 	"testing"
 
-	"github.com/google/uuid"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -42,14 +41,3 @@ func TestGetTextVersion(t *testing.T) {
 
 	assert.Equal(t, int32(1), svc.GetTextVersion())
 }
-
-func TestIsTextExtracted(t *testing.T) {
-	cfg := &serviceconfig.BaseConfig{}
-	svc := document.New(cfg)
-
-	assert.Nil(t, svc.IsTextExtracted(&document.IsTextExtractedParams{
-		DocumentID:      uuid.New(),
-		MinCleanVersion: int32(1),
-		MinTextVersion:  int32(1),
-	}))
-}