Merged in feature/document (pull request #36)

Document CRUD * doccreate * dochashlocandget * depsandtodo
2025-01-24 16:12:25 +00:00
parent 4ec1d51a12
commit b16ff55afa
31 changed files with 381 additions and 82 deletions
@@ -0,0 +1,18 @@
+package documentclean
+
+import "errors"
+
+type Service struct {
+}
+
+func New() *Service {
+	return &Service{}
+}
+
+func (s *Service) IsValidVersion(v int32) error {
+	if v <= 0 {
+		return errors.New("document clean code version must be > 0")
+	}
+
+	return nil
+}
@@ -0,0 +1,13 @@
+package documentclean_test
+
+import (
+	documentclean "queryorchestration/internal/document/clean"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestService(t *testing.T) {
+	svc := documentclean.New()
+	assert.NotNil(t, svc)
+}
@@ -0,0 +1,41 @@
+package document
+
+import (
+	"context"
+	"queryorchestration/internal/database"
+	"queryorchestration/internal/database/repository"
+
+	"github.com/google/uuid"
+)
+
+type Location = string
+
+type Create struct {
+	JobID    uuid.UUID
+	Location Location
+}
+
+func (s *Service) Create(ctx context.Context, doc *Create) (uuid.UUID, error) {
+	hash, err := s.getHash()
+	if err != nil {
+		return uuid.Nil, err
+	}
+
+	dbid, err := s.db.Queries.CreateDocument(ctx, &repository.CreateDocumentParams{
+		Jobid:    database.MustToDBUUID(doc.JobID),
+		Hash:     hash,
+		Location: doc.Location,
+	})
+	if err != nil {
+		return uuid.Nil, err
+	}
+
+	id := database.MustToUUID(dbid)
+
+	return id, nil
+}
+
+func (s *Service) getHash() (string, error) {
+	// TODO
+	return "example_hash", nil
+}
@@ -0,0 +1,48 @@
+package document_test
+
+import (
+	"context"
+	"queryorchestration/internal/database"
+	"queryorchestration/internal/database/repository"
+	"queryorchestration/internal/document"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/pashagolub/pgxmock/v3"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCreate(t *testing.T) {
+	ctx := context.Background()
+
+	pool, err := pgxmock.NewPool()
+	if err != nil {
+		t.Fatalf("failed to open pgxmock database: %v", err)
+	}
+	queries := repository.New(pool)
+	db := &database.Connection{
+		Queries: queries,
+		Pool:    pool,
+	}
+
+	svc := document.New(db)
+
+	doc := document.Document{
+		ID:       uuid.New(),
+		JobID:    uuid.New(),
+		Location: "example_location",
+	}
+
+	pool.ExpectQuery("name: CreateDocument :one").WithArgs(database.MustToDBUUID(doc.JobID), pgxmock.AnyArg(), doc.Location).
+		WillReturnRows(
+			pgxmock.NewRows([]string{"id"}).
+				AddRow(database.MustToDBUUID(doc.ID)),
+		)
+
+	id, err := svc.Create(ctx, &document.Create{
+		JobID:    doc.JobID,
+		Location: doc.Location,
+	})
+	assert.Nil(t, err)
+	assert.Equal(t, doc.ID, id)
+}
@@ -0,0 +1,22 @@
+package document
+
+import (
+	"context"
+	"queryorchestration/internal/database"
+
+	"github.com/google/uuid"
+)
+
+func (s *Service) Get(ctx context.Context, id uuid.UUID) (*Document, error) {
+	doc, err := s.db.Queries.GetDocument(ctx, database.MustToDBUUID(id))
+	if err != nil {
+		return nil, err
+	}
+
+	return &Document{
+		ID:       database.MustToUUID(doc.ID),
+		JobID:    database.MustToUUID(doc.Jobid),
+		Hash:     doc.Hash,
+		Location: doc.Location,
+	}, nil
+}
@@ -0,0 +1,46 @@
+package document_test
+
+import (
+	"context"
+	"queryorchestration/internal/database"
+	"queryorchestration/internal/database/repository"
+	"queryorchestration/internal/document"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/pashagolub/pgxmock/v3"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGet(t *testing.T) {
+	ctx := context.Background()
+
+	pool, err := pgxmock.NewPool()
+	if err != nil {
+		t.Fatalf("failed to open pgxmock database: %v", err)
+	}
+	queries := repository.New(pool)
+	db := &database.Connection{
+		Queries: queries,
+		Pool:    pool,
+	}
+
+	svc := document.New(db)
+
+	doc := document.Document{
+		ID:       uuid.New(),
+		JobID:    uuid.New(),
+		Hash:     "example_hash",
+		Location: "example_location",
+	}
+
+	pool.ExpectQuery("name: GetDocument :one").WithArgs(database.MustToDBUUID(doc.ID)).
+		WillReturnRows(
+			pgxmock.NewRows([]string{"id", "jobId", "hash", "location"}).
+				AddRow(database.MustToDBUUID(doc.ID), database.MustToDBUUID(doc.JobID), doc.Hash, doc.Location),
+		)
+
+	adoc, err := svc.Get(ctx, doc.ID)
+	assert.Nil(t, err)
+	assert.Equal(t, &doc, adoc)
+}
@@ -0,0 +1,24 @@
+package document
+
+import (
+	"queryorchestration/internal/database"
+
+	"github.com/google/uuid"
+)
+
+type Document struct {
+	ID       uuid.UUID
+	JobID    uuid.UUID
+	Hash     string
+	Location Location
+}
+
+type Service struct {
+	db *database.Connection
+}
+
+func New(db *database.Connection) *Service {
+	return &Service{
+		db,
+	}
+}
@@ -0,0 +1,13 @@
+package document_test
+
+import (
+	"queryorchestration/internal/document"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNew(t *testing.T) {
+	svc := document.New(nil)
+	assert.NotNil(t, svc)
+}
@@ -0,0 +1,31 @@
+package documentsync
+
+import (
+	"queryorchestration/internal/database"
+	"queryorchestration/internal/job/collector"
+
+	"github.com/google/uuid"
+)
+
+type Document struct {
+	ID           uuid.UUID `json:"id" validate:"required,uuid"`
+	JobID        uuid.UUID `json:"jobId" validate:"required,uuid"`
+	CleanVersion int32     `json:"cleanVersion" validate:"required,gt=0"`
+	TextVersion  int32     `json:"textVersion" validate:"required,gt=0"`
+}
+
+type Services struct {
+	Collector *collector.Service
+}
+
+type Service struct {
+	db  *database.Connection
+	svc *Services
+}
+
+func New(db *database.Connection, svc *Services) *Service {
+	return &Service{
+		db,
+		svc,
+	}
+}
@@ -0,0 +1,13 @@
+package documentsync_test
+
+import (
+	documentsync "queryorchestration/internal/document/sync"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNew(t *testing.T) {
+	svc := documentsync.New(nil, nil)
+	assert.NotNil(t, svc)
+}
@@ -0,0 +1,63 @@
+package documentsync
+
+import (
+	"context"
+	"queryorchestration/internal/database"
+	"queryorchestration/internal/database/repository"
+	"queryorchestration/internal/job/collector"
+	queryqueue "queryorchestration/internal/query/queue"
+	"queryorchestration/internal/query/result"
+
+	"github.com/google/uuid"
+)
+
+func (s *Service) Sync(ctx context.Context, doc *Document) error {
+	coll, err := s.svc.Collector.GetByJobID(ctx, doc.JobID)
+	if err != nil {
+		return err
+	}
+
+	results, err := s.getResults(ctx, doc.ID, coll)
+	if err != nil {
+		return err
+	}
+
+	queue, err := queryqueue.New(ctx, &queryqueue.NewConfig{
+		DB:           s.db,
+		Collector:    coll,
+		Results:      results,
+		DocumentID:   doc.ID,
+		CleanVersion: doc.CleanVersion,
+		TextVersion:  doc.TextVersion,
+	})
+	if err != nil {
+		return err
+	}
+
+	err = queue.Execute(ctx)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *Service) getResults(ctx context.Context, id uuid.UUID, coll *collector.Collector) ([]*result.Result, error) {
+	docID := database.MustToDBUUID(id)
+
+	results, err := s.db.Queries.ListResultsByDocumentID(ctx, &repository.ListResultsByDocumentIDParams{
+		Documentid:   docID,
+		Textversion:  coll.MinTextVersion,
+		Cleanversion: coll.MinCleanVersion,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	cleanResults := make([]*result.Result, len(results))
+	for index, dbResult := range results {
+		cleanResults[index] = result.Parse(dbResult)
+	}
+
+	return cleanResults, nil
+}
@@ -0,0 +1,215 @@
+package documentsync_test
+
+import (
+	"context"
+	"errors"
+	"queryorchestration/internal/database"
+	"queryorchestration/internal/database/repository"
+	documentsync "queryorchestration/internal/document/sync"
+	"queryorchestration/internal/job/collector"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/jackc/pgx/v5/pgtype"
+	"github.com/pashagolub/pgxmock/v3"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSyncIsSynced(t *testing.T) {
+	ctx := context.Background()
+
+	pool, err := pgxmock.NewPool()
+	if err != nil {
+		t.Fatalf("failed to open pgxmock database: %v", err)
+	}
+	queries := repository.New(pool)
+	db := &database.Connection{
+		Queries: queries,
+		Pool:    pool,
+	}
+
+	doc := documentsync.Document{
+		ID:    uuid.New(),
+		JobID: uuid.New(),
+	}
+
+	queryVersion := int32(1)
+	coll := collector.Collector{
+		ID:              uuid.New(),
+		JobID:           doc.JobID,
+		MinCleanVersion: int32(1),
+		MinTextVersion:  int32(2),
+	}
+
+	pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(doc.JobID)).
+		WillReturnRows(
+			pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
+				AddRow(database.MustToDBUUID(coll.ID), database.MustToDBUUID(coll.JobID), &coll.MinCleanVersion, &coll.MinTextVersion, int32(1), int32(2), []byte("")),
+		)
+	pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), coll.MinCleanVersion, coll.MinTextVersion).
+		WillReturnRows(
+			pgxmock.NewRows([]string{"id", "queryId", "queryVersion"}).
+				AddRow(pgtype.UUID{}, pgtype.UUID{}, queryVersion),
+		)
+	pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(database.MustToDBUUID(coll.ID)).
+		WillReturnRows(
+			pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}).
+				AddRow(database.MustToDBUUID(coll.ID), pgtype.UUID{}, repository.QuerytypeJsonExtractor, queryVersion, []pgtype.UUID{}),
+		)
+
+	docSvc := documentsync.New(db, &documentsync.Services{
+		Collector: collector.New(db, &collector.Services{}),
+	})
+	err = docSvc.Sync(ctx, &doc)
+	assert.Nil(t, err)
+}
+
+func TestSyncDBFail(t *testing.T) {
+	ctx := context.Background()
+
+	pool, err := pgxmock.NewPool()
+	if err != nil {
+		t.Fatalf("failed to open pgxmock database: %v", err)
+	}
+	queries := repository.New(pool)
+	db := &database.Connection{
+		Queries: queries,
+		Pool:    pool,
+	}
+
+	doc := documentsync.Document{
+		ID:    uuid.New(),
+		JobID: uuid.New(),
+	}
+
+	dbCollectorId := database.MustToDBUUID(uuid.New())
+	dbJobID := database.MustToDBUUID(doc.JobID)
+	dbQueryID := database.MustToDBUUID(uuid.New())
+	minCleanVersion := int32(1)
+	minTextVersion := int32(1)
+
+	pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(dbJobID).
+		WillReturnRows(
+			pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}),
+		)
+
+	docSvc := documentsync.New(db, &documentsync.Services{
+		Collector: collector.New(db, &collector.Services{}),
+	})
+	err = docSvc.Sync(ctx, &doc)
+	assert.EqualError(t, err, "no rows in result set")
+
+	pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(dbJobID).
+		WillReturnRows(
+			pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
+				AddRow(dbCollectorId, dbJobID, &minCleanVersion, &minTextVersion, int32(1), int32(2), []byte("")),
+		)
+	dbErr := "database failure"
+	pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), minCleanVersion, minTextVersion).
+		WillReturnError(errors.New(dbErr))
+
+	docSvc = documentsync.New(db, &documentsync.Services{
+		Collector: collector.New(db, &collector.Services{}),
+	})
+	err = docSvc.Sync(ctx, &doc)
+	assert.EqualError(t, err, dbErr)
+
+	pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(dbJobID).
+		WillReturnRows(
+			pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
+				AddRow(dbCollectorId, dbJobID, &minCleanVersion, &minTextVersion, int32(1), int32(2), []byte("")),
+		)
+	pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), minCleanVersion, minTextVersion).
+		WillReturnRows(
+			pgxmock.NewRows([]string{"id", "queryId", "queryVersion"}),
+		)
+	dbErr = "database failure"
+	pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(dbCollectorId).
+		WillReturnError(errors.New(dbErr))
+
+	docSvc = documentsync.New(db, &documentsync.Services{
+		Collector: collector.New(db, &collector.Services{}),
+	})
+	err = docSvc.Sync(ctx, &doc)
+	assert.EqualError(t, err, dbErr)
+
+	pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(dbJobID).
+		WillReturnRows(
+			pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
+				AddRow(dbCollectorId, dbJobID, &minCleanVersion, &minTextVersion, int32(1), int32(2), []byte("")),
+		)
+	qV := int32(1)
+	reqID := database.MustToDBUUID(uuid.New())
+	resID := database.MustToDBUUID(uuid.New())
+	pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), minCleanVersion, minTextVersion).
+		WillReturnRows(
+			pgxmock.NewRows([]string{"id", "queryId", "queryVersion"}).
+				AddRow(resID, reqID, qV),
+		)
+	dbErr = "database failure"
+	pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(dbCollectorId).
+		WillReturnRows(
+			pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}).
+				AddRow(dbCollectorId, dbQueryID, repository.QuerytypeJsonExtractor, qV, []pgtype.UUID{reqID}).
+				AddRow(dbCollectorId, reqID, repository.QuerytypeContextFull, qV, []pgtype.UUID{}),
+		)
+	pool.ExpectQuery("name: ListResultValuesByID :many").WithArgs([]pgtype.UUID{resID}).
+		WillReturnError(errors.New(dbErr))
+
+	docSvc = documentsync.New(db, &documentsync.Services{
+		Collector: collector.New(db, &collector.Services{}),
+	})
+	err = docSvc.Sync(ctx, &doc)
+	assert.EqualError(t, err, dbErr)
+}
+
+func TestSync(t *testing.T) {
+	ctx := context.Background()
+
+	pool, err := pgxmock.NewPool()
+	if err != nil {
+		t.Fatalf("failed to open pgxmock database: %v", err)
+	}
+	queries := repository.New(pool)
+	db := &database.Connection{
+		Queries: queries,
+		Pool:    pool,
+	}
+
+	doc := documentsync.Document{
+		ID:    uuid.New(),
+		JobID: uuid.New(),
+	}
+
+	dbCollectorId := database.MustToDBUUID(uuid.New())
+	dbJobID := database.MustToDBUUID(doc.JobID)
+	dbQueryID := database.MustToDBUUID(uuid.New())
+	minCleanVersion := int32(1)
+	minTextVersion := int32(1)
+
+	pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(dbJobID).
+		WillReturnRows(
+			pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
+				AddRow(dbCollectorId, dbJobID, &minCleanVersion, &minTextVersion, int32(1), int32(2), []byte("")),
+		)
+	pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), minCleanVersion, minTextVersion).
+		WillReturnRows(
+			pgxmock.NewRows([]string{"id", "queryId", "queryVersion"}),
+		)
+	qV := int32(1)
+	pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(dbCollectorId).
+		WillReturnRows(
+			pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}).
+				AddRow(dbCollectorId, dbQueryID, repository.QuerytypeJsonExtractor, qV, []pgtype.UUID{}),
+		)
+	pool.ExpectQuery("name: ListResultValuesByID :many").WithArgs([]pgtype.UUID{}).
+		WillReturnRows(
+			pgxmock.NewRows([]string{"id", "queryId", "value"}),
+		)
+
+	docSvc := documentsync.New(db, &documentsync.Services{
+		Collector: collector.New(db, &collector.Services{}),
+	})
+	err = docSvc.Sync(ctx, &doc)
+	assert.EqualError(t, err, "JSON Extraction requires 1 result")
+}
@@ -0,0 +1,60 @@
+package documentsync
+
+import (
+	"context"
+	"queryorchestration/internal/database"
+	"queryorchestration/internal/database/repository"
+	"queryorchestration/internal/job/collector"
+	"queryorchestration/internal/query/result"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/pashagolub/pgxmock/v3"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetResults(t *testing.T) {
+	ctx := context.Background()
+
+	pool, err := pgxmock.NewPool()
+	if err != nil {
+		t.Fatalf("failed to open pgxmock database: %v", err)
+	}
+	queries := repository.New(pool)
+	db := &database.Connection{
+		Queries: queries,
+		Pool:    pool,
+	}
+
+	doc := Document{
+		ID:    uuid.New(),
+		JobID: uuid.New(),
+	}
+
+	coll := collector.Collector{
+		ID:              uuid.New(),
+		JobID:           doc.JobID,
+		MinCleanVersion: int32(1),
+		MinTextVersion:  int32(2),
+	}
+	results := []*result.Result{
+		{
+			ID:           uuid.New(),
+			QueryID:      uuid.New(),
+			QueryVersion: 2,
+		},
+	}
+
+	pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), coll.MinCleanVersion, coll.MinTextVersion).
+		WillReturnRows(
+			pgxmock.NewRows([]string{"id", "queryId", "queryVersion"}).
+				AddRow(database.MustToDBUUID(results[0].ID), database.MustToDBUUID(results[0].QueryID), results[0].QueryVersion),
+		)
+
+	docSvc := Service{
+		db: db,
+	}
+	res, err := docSvc.getResults(ctx, doc.ID, &coll)
+	assert.Nil(t, err)
+	assert.ElementsMatch(t, results, res)
+}