Merged in feature/document (pull request #36)

Document CRUD

* doccreate

* dochashlocandget

* depsandtodo

internal/database/repository/document.sql.go

@@ -12,29 +12,40 @@ import (
 )
 
 const createDocument = `-- name: CreateDocument :one
-INSERT INTO documents (jobId) VALUES ($1) RETURNING id
+INSERT INTO documents (jobId, hash, location) VALUES ($1, $2, $3) RETURNING id
 `
 
+type CreateDocumentParams struct {
+	Jobid    pgtype.UUID `db:"jobid"`
+	Hash     string      `db:"hash"`
+	Location string      `db:"location"`
+}
+
 // CreateDocument
 //
-//	INSERT INTO documents (jobId) VALUES ($1) RETURNING id
-func (q *Queries) CreateDocument(ctx context.Context, jobid pgtype.UUID) (pgtype.UUID, error) {
-	row := q.db.QueryRow(ctx, createDocument, jobid)
+//	INSERT INTO documents (jobId, hash, location) VALUES ($1, $2, $3) RETURNING id
+func (q *Queries) CreateDocument(ctx context.Context, arg *CreateDocumentParams) (pgtype.UUID, error) {
+	row := q.db.QueryRow(ctx, createDocument, arg.Jobid, arg.Hash, arg.Location)
 	var id pgtype.UUID
 	err := row.Scan(&id)
 	return id, err
 }
 
 const getDocument = `-- name: GetDocument :one
-SELECT id, jobId FROM documents WHERE id = $1 LIMIT 1
+SELECT id, jobId, hash, location FROM documents WHERE id = $1 LIMIT 1
 `
 
 // GetDocument
 //
-//	SELECT id, jobId FROM documents WHERE id = $1 LIMIT 1
+//	SELECT id, jobId, hash, location FROM documents WHERE id = $1 LIMIT 1
 func (q *Queries) GetDocument(ctx context.Context, id pgtype.UUID) (*Document, error) {
 	row := q.db.QueryRow(ctx, getDocument, id)
 	var i Document
-	err := row.Scan(&i.ID, &i.Jobid)
+	err := row.Scan(
+		&i.ID,
+		&i.Jobid,
+		&i.Hash,
+		&i.Location,
+	)
 	return &i, err
 }

internal/database/repository/document_test.go

@@ -27,7 +27,11 @@ func TestDocument(t *testing.T) {
 	jobId, err := queries.CreateJob(ctx, clientId)
 	assert.Nil(t, err)
 
-	id, err := queries.CreateDocument(ctx, jobId)
+	hash := "example_hash"
+	id, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
+		Jobid: jobId,
+		Hash:  hash,
+	})
 	assert.Nil(t, err)
 	assert.NotEmpty(t, id)
 
@@ -36,5 +40,6 @@ func TestDocument(t *testing.T) {
 	assert.EqualExportedValues(t, &repository.Document{
 		ID:    id,
 		Jobid: jobId,
+		Hash:  hash,
 	}, doc)
 }

internal/database/repository/models.go

@@ -102,8 +102,10 @@ type Collectorquerydependencytree struct {
 }
 
 type Document struct {
-	ID    pgtype.UUID `db:"id"`
-	Jobid pgtype.UUID `db:"jobid"`
+	ID       pgtype.UUID `db:"id"`
+	Jobid    pgtype.UUID `db:"jobid"`
+	Hash     string      `db:"hash"`
+	Location string      `db:"location"`
 }
 
 type Fullactivecollector struct {

internal/database/repository/result_test.go

@@ -31,7 +31,10 @@ func TestResults(t *testing.T) {
 	assert.Nil(t, err)
 	jobId, err := queries.CreateJob(ctx, clientId)
 	assert.Nil(t, err)
-	documentID, err := queries.CreateDocument(ctx, jobId)
+	documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
+		Jobid: jobId,
+		Hash:  "example_hash",
+	})
 	assert.Nil(t, err)
 
 	jsonQuery, err := queries.GetQuery(ctx, jsonQueryID)

internal/document/clean/service_test.go

@@ -1,7 +1,7 @@
 package documentclean_test
 
 import (
-	documentclean "queryorchestration/internal/document_clean"
+	documentclean "queryorchestration/internal/document/clean"
 	"testing"
 
 	"github.com/stretchr/testify/assert"

internal/document/create.go

@@ -0,0 +1,41 @@
+package document
+
+import (
+	"context"
+	"queryorchestration/internal/database"
+	"queryorchestration/internal/database/repository"
+
+	"github.com/google/uuid"
+)
+
+type Location = string
+
+type Create struct {
+	JobID    uuid.UUID
+	Location Location
+}
+
+func (s *Service) Create(ctx context.Context, doc *Create) (uuid.UUID, error) {
+	hash, err := s.getHash()
+	if err != nil {
+		return uuid.Nil, err
+	}
+
+	dbid, err := s.db.Queries.CreateDocument(ctx, &repository.CreateDocumentParams{
+		Jobid:    database.MustToDBUUID(doc.JobID),
+		Hash:     hash,
+		Location: doc.Location,
+	})
+	if err != nil {
+		return uuid.Nil, err
+	}
+
+	id := database.MustToUUID(dbid)
+
+	return id, nil
+}
+
+func (s *Service) getHash() (string, error) {
+	// TODO
+	return "example_hash", nil
+}

internal/document/create_test.go

@@ -0,0 +1,48 @@
+package document_test
+
+import (
+	"context"
+	"queryorchestration/internal/database"
+	"queryorchestration/internal/database/repository"
+	"queryorchestration/internal/document"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/pashagolub/pgxmock/v3"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCreate(t *testing.T) {
+	ctx := context.Background()
+
+	pool, err := pgxmock.NewPool()
+	if err != nil {
+		t.Fatalf("failed to open pgxmock database: %v", err)
+	}
+	queries := repository.New(pool)
+	db := &database.Connection{
+		Queries: queries,
+		Pool:    pool,
+	}
+
+	svc := document.New(db)
+
+	doc := document.Document{
+		ID:       uuid.New(),
+		JobID:    uuid.New(),
+		Location: "example_location",
+	}
+
+	pool.ExpectQuery("name: CreateDocument :one").WithArgs(database.MustToDBUUID(doc.JobID), pgxmock.AnyArg(), doc.Location).
+		WillReturnRows(
+			pgxmock.NewRows([]string{"id"}).
+				AddRow(database.MustToDBUUID(doc.ID)),
+		)
+
+	id, err := svc.Create(ctx, &document.Create{
+		JobID:    doc.JobID,
+		Location: doc.Location,
+	})
+	assert.Nil(t, err)
+	assert.Equal(t, doc.ID, id)
+}

internal/document/get.go

@@ -0,0 +1,22 @@
+package document
+
+import (
+	"context"
+	"queryorchestration/internal/database"
+
+	"github.com/google/uuid"
+)
+
+func (s *Service) Get(ctx context.Context, id uuid.UUID) (*Document, error) {
+	doc, err := s.db.Queries.GetDocument(ctx, database.MustToDBUUID(id))
+	if err != nil {
+		return nil, err
+	}
+
+	return &Document{
+		ID:       database.MustToUUID(doc.ID),
+		JobID:    database.MustToUUID(doc.Jobid),
+		Hash:     doc.Hash,
+		Location: doc.Location,
+	}, nil
+}

internal/document/get_test.go

@@ -0,0 +1,46 @@
+package document_test
+
+import (
+	"context"
+	"queryorchestration/internal/database"
+	"queryorchestration/internal/database/repository"
+	"queryorchestration/internal/document"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/pashagolub/pgxmock/v3"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGet(t *testing.T) {
+	ctx := context.Background()
+
+	pool, err := pgxmock.NewPool()
+	if err != nil {
+		t.Fatalf("failed to open pgxmock database: %v", err)
+	}
+	queries := repository.New(pool)
+	db := &database.Connection{
+		Queries: queries,
+		Pool:    pool,
+	}
+
+	svc := document.New(db)
+
+	doc := document.Document{
+		ID:       uuid.New(),
+		JobID:    uuid.New(),
+		Hash:     "example_hash",
+		Location: "example_location",
+	}
+
+	pool.ExpectQuery("name: GetDocument :one").WithArgs(database.MustToDBUUID(doc.ID)).
+		WillReturnRows(
+			pgxmock.NewRows([]string{"id", "jobId", "hash", "location"}).
+				AddRow(database.MustToDBUUID(doc.ID), database.MustToDBUUID(doc.JobID), doc.Hash, doc.Location),
+		)
+
+	adoc, err := svc.Get(ctx, doc.ID)
+	assert.Nil(t, err)
+	assert.Equal(t, &doc, adoc)
+}

internal/document/service.go

@@ -0,0 +1,24 @@
+package document
+
+import (
+	"queryorchestration/internal/database"
+
+	"github.com/google/uuid"
+)
+
+type Document struct {
+	ID       uuid.UUID
+	JobID    uuid.UUID
+	Hash     string
+	Location Location
+}
+
+type Service struct {
+	db *database.Connection
+}
+
+func New(db *database.Connection) *Service {
+	return &Service{
+		db,
+	}
+}

internal/document/service_test.go

@@ -0,0 +1,13 @@
+package document_test
+
+import (
+	"queryorchestration/internal/document"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNew(t *testing.T) {
+	svc := document.New(nil)
+	assert.NotNil(t, svc)
+}

internal/document/sync/service.go

@@ -0,0 +1,31 @@
+package documentsync
+
+import (
+	"queryorchestration/internal/database"
+	"queryorchestration/internal/job/collector"
+
+	"github.com/google/uuid"
+)
+
+type Document struct {
+	ID           uuid.UUID `json:"id" validate:"required,uuid"`
+	JobID        uuid.UUID `json:"jobId" validate:"required,uuid"`
+	CleanVersion int32     `json:"cleanVersion" validate:"required,gt=0"`
+	TextVersion  int32     `json:"textVersion" validate:"required,gt=0"`
+}
+
+type Services struct {
+	Collector *collector.Service
+}
+
+type Service struct {
+	db  *database.Connection
+	svc *Services
+}
+
+func New(db *database.Connection, svc *Services) *Service {
+	return &Service{
+		db,
+		svc,
+	}
+}

internal/document/sync/service_test.go

@@ -0,0 +1,13 @@
+package documentsync_test
+
+import (
+	documentsync "queryorchestration/internal/document/sync"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNew(t *testing.T) {
+	svc := documentsync.New(nil, nil)
+	assert.NotNil(t, svc)
+}

internal/document/sync/sync.go

@@ -1,4 +1,4 @@
-package document
+package documentsync
 
 import (
 	"context"
@@ -11,30 +11,6 @@ import (
 	"github.com/google/uuid"
 )
 
-type Document struct {
-	ID           uuid.UUID `json:"id" validate:"required,uuid"`
-	JobID        uuid.UUID `json:"jobId" validate:"required,uuid"`
-	Name         string    `json:"name" validate:"required"`
-	CleanVersion int32     `json:"cleanVersion" validate:"required,gt=0"`
-	TextVersion  int32     `json:"textVersion" validate:"required,gt=0"`
-}
-
-type Services struct {
-	Collector *collector.Service
-}
-
-type Service struct {
-	db  *database.Connection
-	svc *Services
-}
-
-func New(db *database.Connection, svc *Services) *Service {
-	return &Service{
-		db,
-		svc,
-	}
-}
-
 func (s *Service) Sync(ctx context.Context, doc *Document) error {
 	coll, err := s.svc.Collector.GetByJobID(ctx, doc.JobID)
 	if err != nil {

internal/document/sync/sync_test.go

@@ -1,12 +1,12 @@
-package document_test
+package documentsync_test
 
 import (
 	"context"
 	"errors"
 	"queryorchestration/internal/database"
 	"queryorchestration/internal/database/repository"
+	documentsync "queryorchestration/internal/document/sync"
 	"queryorchestration/internal/job/collector"
-	"queryorchestration/internal/query/document"
 	"testing"
 
 	"github.com/google/uuid"
@@ -28,10 +28,9 @@ func TestSyncIsSynced(t *testing.T) {
 		Pool:    pool,
 	}
 
-	doc := document.Document{
+	doc := documentsync.Document{
 		ID:    uuid.New(),
 		JobID: uuid.New(),
-		Name:  "document_name",
 	}
 
 	queryVersion := int32(1)
@@ -58,7 +57,7 @@ func TestSyncIsSynced(t *testing.T) {
 				AddRow(database.MustToDBUUID(coll.ID), pgtype.UUID{}, repository.QuerytypeJsonExtractor, queryVersion, []pgtype.UUID{}),
 		)
 
-	docSvc := document.New(db, &document.Services{
+	docSvc := documentsync.New(db, &documentsync.Services{
 		Collector: collector.New(db, &collector.Services{}),
 	})
 	err = docSvc.Sync(ctx, &doc)
@@ -78,10 +77,9 @@ func TestSyncDBFail(t *testing.T) {
 		Pool:    pool,
 	}
 
-	doc := document.Document{
+	doc := documentsync.Document{
 		ID:    uuid.New(),
 		JobID: uuid.New(),
-		Name:  "document_name",
 	}
 
 	dbCollectorId := database.MustToDBUUID(uuid.New())
@@ -95,7 +93,7 @@ func TestSyncDBFail(t *testing.T) {
 			pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}),
 		)
 
-	docSvc := document.New(db, &document.Services{
+	docSvc := documentsync.New(db, &documentsync.Services{
 		Collector: collector.New(db, &collector.Services{}),
 	})
 	err = docSvc.Sync(ctx, &doc)
@@ -110,7 +108,7 @@ func TestSyncDBFail(t *testing.T) {
 	pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), minCleanVersion, minTextVersion).
 		WillReturnError(errors.New(dbErr))
 
-	docSvc = document.New(db, &document.Services{
+	docSvc = documentsync.New(db, &documentsync.Services{
 		Collector: collector.New(db, &collector.Services{}),
 	})
 	err = docSvc.Sync(ctx, &doc)
@@ -129,7 +127,7 @@ func TestSyncDBFail(t *testing.T) {
 	pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(dbCollectorId).
 		WillReturnError(errors.New(dbErr))
 
-	docSvc = document.New(db, &document.Services{
+	docSvc = documentsync.New(db, &documentsync.Services{
 		Collector: collector.New(db, &collector.Services{}),
 	})
 	err = docSvc.Sync(ctx, &doc)
@@ -158,7 +156,7 @@ func TestSyncDBFail(t *testing.T) {
 	pool.ExpectQuery("name: ListResultValuesByID :many").WithArgs([]pgtype.UUID{resID}).
 		WillReturnError(errors.New(dbErr))
 
-	docSvc = document.New(db, &document.Services{
+	docSvc = documentsync.New(db, &documentsync.Services{
 		Collector: collector.New(db, &collector.Services{}),
 	})
 	err = docSvc.Sync(ctx, &doc)
@@ -178,10 +176,9 @@ func TestSync(t *testing.T) {
 		Pool:    pool,
 	}
 
-	doc := document.Document{
+	doc := documentsync.Document{
 		ID:    uuid.New(),
 		JobID: uuid.New(),
-		Name:  "document_name",
 	}
 
 	dbCollectorId := database.MustToDBUUID(uuid.New())
@@ -210,7 +207,7 @@ func TestSync(t *testing.T) {
 			pgxmock.NewRows([]string{"id", "queryId", "value"}),
 		)
 
-	docSvc := document.New(db, &document.Services{
+	docSvc := documentsync.New(db, &documentsync.Services{
 		Collector: collector.New(db, &collector.Services{}),
 	})
 	err = docSvc.Sync(ctx, &doc)

internal/document/sync/syncprivate_test.go

@@ -0,0 +1,60 @@
+package documentsync
+
+import (
+	"context"
+	"queryorchestration/internal/database"
+	"queryorchestration/internal/database/repository"
+	"queryorchestration/internal/job/collector"
+	"queryorchestration/internal/query/result"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/pashagolub/pgxmock/v3"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetResults(t *testing.T) {
+	ctx := context.Background()
+
+	pool, err := pgxmock.NewPool()
+	if err != nil {
+		t.Fatalf("failed to open pgxmock database: %v", err)
+	}
+	queries := repository.New(pool)
+	db := &database.Connection{
+		Queries: queries,
+		Pool:    pool,
+	}
+
+	doc := Document{
+		ID:    uuid.New(),
+		JobID: uuid.New(),
+	}
+
+	coll := collector.Collector{
+		ID:              uuid.New(),
+		JobID:           doc.JobID,
+		MinCleanVersion: int32(1),
+		MinTextVersion:  int32(2),
+	}
+	results := []*result.Result{
+		{
+			ID:           uuid.New(),
+			QueryID:      uuid.New(),
+			QueryVersion: 2,
+		},
+	}
+
+	pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), coll.MinCleanVersion, coll.MinTextVersion).
+		WillReturnRows(
+			pgxmock.NewRows([]string{"id", "queryId", "queryVersion"}).
+				AddRow(database.MustToDBUUID(results[0].ID), database.MustToDBUUID(results[0].QueryID), results[0].QueryVersion),
+		)
+
+	docSvc := Service{
+		db: db,
+	}
+	res, err := docSvc.getResults(ctx, doc.ID, &coll)
+	assert.Nil(t, err)
+	assert.ElementsMatch(t, results, res)
+}

internal/job/collector/service.go

@@ -2,7 +2,7 @@ package collector
 
 import (
 	"queryorchestration/internal/database"
-	documentclean "queryorchestration/internal/document_clean"
+	documentclean "queryorchestration/internal/document/clean"
 	"queryorchestration/internal/query"
 	textextraction "queryorchestration/internal/text_extraction"
 

internal/job/collector/update_test.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"queryorchestration/internal/database"
 	"queryorchestration/internal/database/repository"
-	documentclean "queryorchestration/internal/document_clean"
+	documentclean "queryorchestration/internal/document/clean"
 	"queryorchestration/internal/job/collector"
 	"queryorchestration/internal/query"
 	textextraction "queryorchestration/internal/text_extraction"

internal/job/collector/updateprivate_test.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"queryorchestration/internal/database"
 	"queryorchestration/internal/database/repository"
-	documentclean "queryorchestration/internal/document_clean"
+	documentclean "queryorchestration/internal/document/clean"
 	"queryorchestration/internal/query"
 	textextraction "queryorchestration/internal/text_extraction"
 	"testing"

internal/text_extraction/service_test.go

@@ -1,7 +1,7 @@
 package textextraction_test
 
 import (
-	documentclean "queryorchestration/internal/document_clean"
+	documentclean "queryorchestration/internal/document/clean"
 	"testing"
 
 	"github.com/stretchr/testify/assert"