Merged in feature/docinit (pull request #48)

Feature/docinit

* createunittests

* cleanup

* skip

* cleanup

api/docInitRunner/runner_test.go

@@ -7,23 +7,29 @@ import (
 	"queryorchestration/internal/client"
 	"queryorchestration/internal/database"
 	"queryorchestration/internal/database/repository"
+	"queryorchestration/internal/document"
 	documentinit "queryorchestration/internal/document/init"
 	"queryorchestration/internal/job"
 	"queryorchestration/internal/job/collector"
 	"queryorchestration/internal/serviceconfig"
+	"queryorchestration/internal/serviceconfig/objectstore"
 	"queryorchestration/internal/serviceconfig/queue/documentclean"
+	objectstoremock "queryorchestration/mocks/objectstore"
 	"testing"
 
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/sqs/types"
 	"github.com/go-playground/validator/v10"
 	"github.com/google/uuid"
 	"github.com/pashagolub/pgxmock/v3"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 )
 
 type DocInitConfig struct {
 	serviceconfig.BaseConfig
 	documentclean.DocCleanConfig
+	objectstore.ObjectStoreConfig
 }
 
 func TestDocInitRunner(t *testing.T) {
@@ -37,6 +43,8 @@ func TestDocInitRunner(t *testing.T) {
 	cfg := &DocInitConfig{}
 	cfg.DBPool = pool
 	cfg.DBQueries = repository.New(pool)
+	mockStore := objectstoremock.NewMockS3Client(t)
+	cfg.StoreClient = mockStore
 
 	runner := docinitrunner.New(validator.New(), &docinitrunner.Services{
 		Document: documentinit.New(cfg, &documentinit.Services{
@@ -54,6 +62,7 @@ func TestDocInitRunner(t *testing.T) {
 	}
 	doc := documentinit.Create{
 		JobID:    j.ID,
+		Bucket:   "bucket_name",
 		Location: "/I/am/here",
 	}
 	bodyBytes, err := json.Marshal(doc)
@@ -63,6 +72,11 @@ func TestDocInitRunner(t *testing.T) {
 		Body: &body,
 	}
 
+	docinfo := document.Document{
+		ID:   uuid.New(),
+		Hash: "example_hash",
+	}
+
 	pool.ExpectQuery("name: GetJob :one").WithArgs(database.MustToDBUUID(j.ID)).WillReturnRows(
 		pgxmock.NewRows([]string{"id", "clientId", "canSync"}).
 			AddRow(database.MustToDBUUID(j.ID), database.MustToDBUUID(j.ClientID), j.CanSync),
@@ -71,11 +85,30 @@ func TestDocInitRunner(t *testing.T) {
 		pgxmock.NewRows([]string{"id", "name", "canSync"}).
 			AddRow(database.MustToDBUUID(j.ClientID), "client_name", true),
 	)
-	pool.ExpectQuery("name: CreateDocument :one").WithArgs(database.MustToDBUUID(doc.JobID), pgxmock.AnyArg(), doc.Location).
+	pool.ExpectQuery("-- name: GetDocumentIDByHash :one").WithArgs(docinfo.Hash, database.MustToDBUUID(doc.JobID)).WillReturnRows(
+		pgxmock.NewRows([]string{"id"}),
+	)
+	pool.ExpectBegin()
+	pool.ExpectQuery("name: CreateDocument :one").WithArgs(database.MustToDBUUID(doc.JobID), docinfo.Hash).
 		WillReturnRows(
 			pgxmock.NewRows([]string{"id"}).
-				AddRow(database.MustToDBUUID(uuid.New())),
+				AddRow(database.MustToDBUUID(docinfo.ID)),
 		)
+	pool.ExpectExec("name: AddDocumentEntry :exec").WithArgs(database.MustToDBUUID(docinfo.ID), doc.Bucket, doc.Location).
+		WillReturnResult(pgxmock.NewResult("", 1))
+	pool.ExpectCommit()
+
+	mockStore.EXPECT().
+		HeadObject(
+			mock.Anything,
+			mock.MatchedBy(func(in *s3.HeadObjectInput) bool {
+				return *in.Bucket == doc.Bucket && *in.Key == doc.Location
+			}),
+			mock.Anything,
+		).
+		Return(&s3.HeadObjectOutput{
+			ETag: &docinfo.Hash,
+		}, nil)
 
 	err = runner.Process(ctx, msg)
 	assert.NoError(t, err)

api/queryService/query_test.go

@@ -226,10 +226,9 @@ func TestTestQuery(t *testing.T) {
 		JobID: uuid.New(),
 	}
 	doc := document.Document{
-		ID:       uuid.New(),
+		ID:    uuid.New(),
-		JobID:    coll.JobID,
+		JobID: coll.JobID,
-		Hash:     "example_hash",
+		Hash:  "example_hash",
-		Location: "example_location",
 	}
 	params := &query.Test{
 		QueryID:      uuid.New(),
@@ -252,8 +251,8 @@ func TestTestQuery(t *testing.T) {
 	reqID := database.MustToDBUUID(uuid.New())
 	pool.ExpectQuery("name: GetDocument :one").WithArgs(database.MustToDBUUID(doc.ID)).
 		WillReturnRows(
-			pgxmock.NewRows([]string{"id", "jobId", "hash", "location"}).
+			pgxmock.NewRows([]string{"id", "jobId", "hash"}).
-				AddRow(database.MustToDBUUID(doc.ID), database.MustToDBUUID(doc.JobID), doc.Hash, doc.Location),
+				AddRow(database.MustToDBUUID(doc.ID), database.MustToDBUUID(doc.JobID), doc.Hash),
 		)
 	pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(doc.JobID)).
 		WillReturnRows(

cmd/docInitRunner/main.go

@@ -62,7 +62,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	err = cfg.SetStoreClientAndPingByName(ctx, cfg.BucketName)
+	err = cfg.SetStoreClient(ctx)
 	if err != nil {
 		slog.Error(err.Error())
 		os.Exit(1)

database/migrations/00000000000006_documents.up.sql

@@ -2,6 +2,15 @@ CREATE TABLE documents (
     id uuid primary key DEFAULT gen_random_uuid(),
     jobId uuid not null,
     hash text not null,
+    foreign key (jobId) references jobs(id),
+    unique(jobId, hash)
+);
+
+CREATE TABLE documentEntries (
+    id uuid primary key DEFAULT gen_random_uuid(),
+    documentId uuid not null,
+    bucket text not null,
     location text not null,
-    foreign key (jobId) references jobs(id)
+    createdAt timestamp default now(),
+    foreign key (documentId) references documents(id)
 );

database/queries/document.sql

@@ -1,5 +1,11 @@
 -- name: GetDocument :one
-SELECT id, jobId, hash, location FROM documents WHERE id = $1 LIMIT 1;
+SELECT id, jobId, hash FROM documents WHERE id = $1 LIMIT 1;
 
 -- name: CreateDocument :one
-INSERT INTO documents (jobId, hash, location) VALUES ($1, $2, $3) RETURNING id;
+INSERT INTO documents (jobId, hash) VALUES ($1, $2) RETURNING id;
+
+-- name: AddDocumentEntry :exec
+INSERT INTO documentEntries (documentId, bucket, location) VALUES ($1, $2, $3);
+
+-- name: GetDocumentIDByHash :one
+SELECT id FROM documents WHERE hash = $1 and jobId = $2;

internal/database/repository/document.sql.go

@@ -11,41 +11,72 @@ import (
 	"github.com/jackc/pgx/v5/pgtype"
 )
 
+const addDocumentEntry = `-- name: AddDocumentEntry :exec
+INSERT INTO documentEntries (documentId, bucket, location) VALUES ($1, $2, $3)
+`
+
+type AddDocumentEntryParams struct {
+	Documentid pgtype.UUID `db:"documentid"`
+	Bucket     string      `db:"bucket"`
+	Location   string      `db:"location"`
+}
+
+// AddDocumentEntry
+//
+//	INSERT INTO documentEntries (documentId, bucket, location) VALUES ($1, $2, $3)
+func (q *Queries) AddDocumentEntry(ctx context.Context, arg *AddDocumentEntryParams) error {
+	_, err := q.db.Exec(ctx, addDocumentEntry, arg.Documentid, arg.Bucket, arg.Location)
+	return err
+}
+
 const createDocument = `-- name: CreateDocument :one
-INSERT INTO documents (jobId, hash, location) VALUES ($1, $2, $3) RETURNING id
+INSERT INTO documents (jobId, hash) VALUES ($1, $2) RETURNING id
 `
 
 type CreateDocumentParams struct {
-	Jobid    pgtype.UUID `db:"jobid"`
+	Jobid pgtype.UUID `db:"jobid"`
-	Hash     string      `db:"hash"`
+	Hash  string      `db:"hash"`
-	Location string      `db:"location"`
 }
 
 // CreateDocument
 //
-//	INSERT INTO documents (jobId, hash, location) VALUES ($1, $2, $3) RETURNING id
+//	INSERT INTO documents (jobId, hash) VALUES ($1, $2) RETURNING id
 func (q *Queries) CreateDocument(ctx context.Context, arg *CreateDocumentParams) (pgtype.UUID, error) {
-	row := q.db.QueryRow(ctx, createDocument, arg.Jobid, arg.Hash, arg.Location)
+	row := q.db.QueryRow(ctx, createDocument, arg.Jobid, arg.Hash)
 	var id pgtype.UUID
 	err := row.Scan(&id)
 	return id, err
 }
 
 const getDocument = `-- name: GetDocument :one
-SELECT id, jobId, hash, location FROM documents WHERE id = $1 LIMIT 1
+SELECT id, jobId, hash FROM documents WHERE id = $1 LIMIT 1
 `
 
 // GetDocument
 //
-//	SELECT id, jobId, hash, location FROM documents WHERE id = $1 LIMIT 1
+//	SELECT id, jobId, hash FROM documents WHERE id = $1 LIMIT 1
 func (q *Queries) GetDocument(ctx context.Context, id pgtype.UUID) (*Document, error) {
 	row := q.db.QueryRow(ctx, getDocument, id)
 	var i Document
-	err := row.Scan(
+	err := row.Scan(&i.ID, &i.Jobid, &i.Hash)
-		&i.ID,
-		&i.Jobid,
-		&i.Hash,
-		&i.Location,
-	)
 	return &i, err
 }
+
+const getDocumentIDByHash = `-- name: GetDocumentIDByHash :one
+SELECT id FROM documents WHERE hash = $1 and jobId = $2
+`
+
+type GetDocumentIDByHashParams struct {
+	Hash  string      `db:"hash"`
+	Jobid pgtype.UUID `db:"jobid"`
+}
+
+// GetDocumentIDByHash
+//
+//	SELECT id FROM documents WHERE hash = $1 and jobId = $2
+func (q *Queries) GetDocumentIDByHash(ctx context.Context, arg *GetDocumentIDByHashParams) (pgtype.UUID, error) {
+	row := q.db.QueryRow(ctx, getDocumentIDByHash, arg.Hash, arg.Jobid)
+	var id pgtype.UUID
+	err := row.Scan(&id)
+	return id, err
+}

internal/database/repository/document_test.go

@@ -39,6 +39,15 @@ func TestDocument(t *testing.T) {
 	assert.NoError(t, err)
 	assert.NotEmpty(t, id)
 
+	bucket := "example_bucket"
+	location := "/i/am/here"
+	err = queries.AddDocumentEntry(ctx, &repository.AddDocumentEntryParams{
+		Documentid: id,
+		Bucket:     bucket,
+		Location:   location,
+	})
+	assert.NoError(t, err)
+
 	doc, err := queries.GetDocument(ctx, id)
 	assert.NoError(t, err)
 	assert.EqualExportedValues(t, &repository.Document{
@@ -46,4 +55,11 @@ func TestDocument(t *testing.T) {
 		Jobid: jobId,
 		Hash:  hash,
 	}, doc)
+
+	docid, err := queries.GetDocumentIDByHash(ctx, &repository.GetDocumentIDByHashParams{
+		Hash:  hash,
+		Jobid: jobId,
+	})
+	assert.NoError(t, err)
+	assert.EqualExportedValues(t, id, docid)
 }

internal/database/repository/models.go

@@ -108,10 +108,17 @@ type Collectorquerydependencytree struct {
 }
 
 type Document struct {
-	ID       pgtype.UUID `db:"id"`
+	ID    pgtype.UUID `db:"id"`
-	Jobid    pgtype.UUID `db:"jobid"`
+	Jobid pgtype.UUID `db:"jobid"`
-	Hash     string      `db:"hash"`
+	Hash  string      `db:"hash"`
-	Location string      `db:"location"`
+}
+
+type Documententry struct {
+	ID         pgtype.UUID      `db:"id"`
+	Documentid pgtype.UUID      `db:"documentid"`
+	Bucket     string           `db:"bucket"`
+	Location   string           `db:"location"`
+	Createdat  pgtype.Timestamp `db:"createdat"`
 }
 
 type Fullactivecollector struct {

internal/document/get.go

@@ -14,9 +14,8 @@ func (s *Service) Get(ctx context.Context, id uuid.UUID) (*Document, error) {
 	}
 
 	return &Document{
-		ID:       database.MustToUUID(doc.ID),
+		ID:    database.MustToUUID(doc.ID),
-		JobID:    database.MustToUUID(doc.Jobid),
+		JobID: database.MustToUUID(doc.Jobid),
-		Hash:     doc.Hash,
+		Hash:  doc.Hash,
-		Location: doc.Location,
 	}, nil
 }

internal/document/get_test.go

@@ -27,16 +27,15 @@ func TestGet(t *testing.T) {
 	svc := document.New(cfg)
 
 	doc := document.Document{
-		ID:       uuid.New(),
+		ID:    uuid.New(),
-		JobID:    uuid.New(),
+		JobID: uuid.New(),
-		Hash:     "example_hash",
+		Hash:  "example_hash",
-		Location: "example_location",
 	}
 
 	pool.ExpectQuery("name: GetDocument :one").WithArgs(database.MustToDBUUID(doc.ID)).
 		WillReturnRows(
-			pgxmock.NewRows([]string{"id", "jobId", "hash", "location"}).
+			pgxmock.NewRows([]string{"id", "jobId", "hash"}).
-				AddRow(database.MustToDBUUID(doc.ID), database.MustToDBUUID(doc.JobID), doc.Hash, doc.Location),
+				AddRow(database.MustToDBUUID(doc.ID), database.MustToDBUUID(doc.JobID), doc.Hash),
 		)
 
 	adoc, err := svc.Get(ctx, doc.ID)

internal/document/init/create.go

@@ -2,6 +2,8 @@ package documentinit
 
 import (
 	"context"
+	"database/sql"
+	"errors"
 	"queryorchestration/internal/database"
 	"queryorchestration/internal/database/repository"
 	"queryorchestration/internal/document"
@@ -9,13 +11,16 @@ import (
 	"queryorchestration/internal/job"
 	"queryorchestration/internal/serviceconfig/queue"
 
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/sqs/types"
 	"github.com/google/uuid"
+	"github.com/jackc/pgx/v5/pgtype"
 )
 
 type Create struct {
 	JobID    uuid.UUID         `json:"jobId" validate:"required,uuid"`
 	Location document.Location `json:"location" validate:"required"`
+	Bucket   string            `json:"bucket" validate:"required"`
 }
 
 func (s *Service) Create(ctx context.Context, doc *Create) (uuid.UUID, error) {
@@ -42,38 +47,82 @@ func (s *Service) Create(ctx context.Context, doc *Create) (uuid.UUID, error) {
 	return id, nil
 }
 
-func (s *Service) getCreateParams(ctx context.Context, doc *Create) (*repository.CreateDocumentParams, error) {
+type createDocumentParams struct {
-	// TODO - get document
+	ID       *pgtype.UUID
+	JobID    pgtype.UUID
+	Hash     string
+	Bucket   string
+	Location string
+}
 
-	hash, err := s.getHash()
+func (s *Service) getCreateParams(ctx context.Context, doc *Create) (*createDocumentParams, error) {
+	res, err := s.cfg.GetStoreClient().HeadObject(ctx, &s3.HeadObjectInput{
+		Bucket: &doc.Bucket,
+		Key:    &doc.Location,
+	})
 	if err != nil {
 		return nil, err
 	}
 
-	return &repository.CreateDocumentParams{
+	idbyhash, err := s.cfg.GetDBQueries().GetDocumentIDByHash(ctx, &repository.GetDocumentIDByHashParams{
-		Jobid:    database.MustToDBUUID(doc.JobID),
+		Jobid: database.MustToDBUUID(doc.JobID),
-		Hash:     hash,
+		Hash:  *res.ETag,
+	})
+	var docID *pgtype.UUID
+	if err != nil && !errors.Is(err, sql.ErrNoRows) {
+		return nil, err
+	} else if err == nil {
+		docID = &idbyhash
+	}
+
+	return &createDocumentParams{
+		ID:       docID,
+		JobID:    database.MustToDBUUID(doc.JobID),
+		Hash:     *res.ETag,
+		Bucket:   doc.Bucket,
 		Location: doc.Location,
 	}, nil
 }
 
-func (s *Service) submitCreate(ctx context.Context, params *repository.CreateDocumentParams) (uuid.UUID, error) {
+func (s *Service) submitCreate(ctx context.Context, params *createDocumentParams) (uuid.UUID, error) {
-	// TODO create - or if hash exists log attempt
+	var id uuid.UUID
-	dbid, err := s.cfg.GetDBQueries().CreateDocument(ctx, params)
+
+	err := s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, q *repository.Queries) error {
+		var dbid pgtype.UUID
+		if params.ID == nil {
+			createid, err := s.cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
+				Jobid: params.JobID,
+				Hash:  params.Hash,
+			})
+			if err != nil {
+				return err
+			}
+
+			dbid = createid
+		} else {
+			dbid = *params.ID
+		}
+
+		err := s.cfg.GetDBQueries().AddDocumentEntry(ctx, &repository.AddDocumentEntryParams{
+			Documentid: dbid,
+			Bucket:     params.Bucket,
+			Location:   params.Location,
+		})
+		if err != nil {
+			return err
+		}
+
+		id = database.MustToUUID(dbid)
+
+		return nil
+	})
 	if err != nil {
 		return uuid.Nil, err
 	}
 
-	id := database.MustToUUID(dbid)
-
 	return id, nil
 }
 
-func (s *Service) getHash() (string, error) {
-	// TODO
-	return "example_hash", nil
-}
-
 func (s *Service) informCreate(ctx context.Context, id uuid.UUID, j *job.Job) error {
 	if !j.CanSync {
 		return nil

internal/document/init/create_test.go

@@ -2,6 +2,7 @@ package documentinit
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"queryorchestration/internal/client"
 	"queryorchestration/internal/database"
@@ -9,10 +10,13 @@ import (
 	"queryorchestration/internal/document"
 	"queryorchestration/internal/job"
 	"queryorchestration/internal/serviceconfig"
+	"queryorchestration/internal/serviceconfig/objectstore"
 	"queryorchestration/internal/serviceconfig/queue/documentclean"
+	objectstoremock "queryorchestration/mocks/objectstore"
 	queuemock "queryorchestration/mocks/queue"
 	"testing"
 
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/sqs"
 	"github.com/google/uuid"
 	"github.com/pashagolub/pgxmock/v3"
@@ -23,6 +27,7 @@ import (
 type DocInitConfig struct {
 	serviceconfig.BaseConfig
 	documentclean.DocCleanConfig
+	objectstore.ObjectStoreConfig
 }
 
 func TestCreate(t *testing.T) {
@@ -35,6 +40,8 @@ func TestCreate(t *testing.T) {
 	cfg := &DocInitConfig{}
 	cfg.DBPool = pool
 	cfg.DBQueries = repository.New(pool)
+	mockStore := objectstoremock.NewMockS3Client(t)
+	cfg.StoreClient = mockStore
 
 	svc := New(cfg, &Services{
 		Job: job.New(cfg, &job.Services{
@@ -47,10 +54,12 @@ func TestCreate(t *testing.T) {
 		ClientID: uuid.New(),
 	}
 	doc := document.Document{
-		ID:       uuid.New(),
+		ID:    uuid.New(),
-		JobID:    j.ID,
+		JobID: j.ID,
-		Location: "example_location",
+		Hash:  "example_hash",
 	}
+	bucket := "eample_bucket"
+	location := "/i/am/here"
 
 	pool.ExpectQuery("name: GetJob :one").WithArgs(database.MustToDBUUID(j.ID)).WillReturnRows(
 		pgxmock.NewRows([]string{"id", "clientId", "canSync"}).
@@ -60,15 +69,35 @@ func TestCreate(t *testing.T) {
 		pgxmock.NewRows([]string{"id", "name", "canSync"}).
 			AddRow(database.MustToDBUUID(j.ClientID), "client_name", true),
 	)
-	pool.ExpectQuery("name: CreateDocument :one").WithArgs(database.MustToDBUUID(doc.JobID), pgxmock.AnyArg(), doc.Location).
+	pool.ExpectQuery("-- name: GetDocumentIDByHash :one").WithArgs(doc.Hash, database.MustToDBUUID(doc.JobID)).WillReturnRows(
+		pgxmock.NewRows([]string{"id"}),
+	)
+	pool.ExpectBegin()
+	pool.ExpectQuery("name: CreateDocument :one").WithArgs(database.MustToDBUUID(doc.JobID), doc.Hash).
 		WillReturnRows(
 			pgxmock.NewRows([]string{"id"}).
 				AddRow(database.MustToDBUUID(doc.ID)),
 		)
+	pool.ExpectExec("name: AddDocumentEntry :exec").WithArgs(database.MustToDBUUID(doc.ID), bucket, location).
+		WillReturnResult(pgxmock.NewResult("", 1))
+	pool.ExpectCommit()
+
+	mockStore.EXPECT().
+		HeadObject(
+			mock.Anything,
+			mock.MatchedBy(func(in *s3.HeadObjectInput) bool {
+				return *in.Bucket == bucket && *in.Key == location
+			}),
+			mock.Anything,
+		).
+		Return(&s3.HeadObjectOutput{
+			ETag: &doc.Hash,
+		}, nil)
 
 	id, err := svc.Create(ctx, &Create{
 		JobID:    doc.JobID,
-		Location: doc.Location,
+		Location: location,
+		Bucket:   bucket,
 	})
 	assert.NoError(t, err)
 	assert.Equal(t, doc.ID, id)
@@ -104,3 +133,255 @@ func TestInformCreate(t *testing.T) {
 	err = svc.informCreate(ctx, id, j)
 	assert.NoError(t, err)
 }
+
+func TestGetCreateParams(t *testing.T) {
+	ctx := context.Background()
+
+	pool, err := pgxmock.NewPool()
+	if err != nil {
+		t.Fatalf("failed to open pgxmock database: %v", err)
+	}
+	cfg := &DocInitConfig{}
+	cfg.DBPool = pool
+	cfg.DBQueries = repository.New(pool)
+	mockStore := objectstoremock.NewMockS3Client(t)
+	cfg.StoreClient = mockStore
+
+	svc := New(cfg, &Services{
+		Job: job.New(cfg, &job.Services{
+			Client: client.New(cfg),
+		}),
+	})
+
+	doc := document.Document{
+		JobID: uuid.New(),
+		Hash:  "example_hash",
+	}
+	bucket := "eample_bucket"
+	location := "/i/am/here"
+
+	pool.ExpectQuery("-- name: GetDocumentIDByHash :one").WithArgs(doc.Hash, database.MustToDBUUID(doc.JobID)).WillReturnRows(
+		pgxmock.NewRows([]string{"id"}),
+	)
+
+	mockStore.EXPECT().
+		HeadObject(
+			mock.Anything,
+			mock.MatchedBy(func(in *s3.HeadObjectInput) bool {
+				return *in.Bucket == bucket && *in.Key == location
+			}),
+			mock.Anything,
+		).
+		Return(&s3.HeadObjectOutput{
+			ETag: &doc.Hash,
+		}, nil)
+
+	params, err := svc.getCreateParams(ctx, &Create{
+		JobID:    doc.JobID,
+		Location: location,
+		Bucket:   bucket,
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, &createDocumentParams{
+		JobID:    database.MustToDBUUID(doc.JobID),
+		Hash:     doc.Hash,
+		Bucket:   bucket,
+		Location: location,
+	}, params)
+}
+
+func TestGetCreateParamsExisting(t *testing.T) {
+	ctx := context.Background()
+
+	pool, err := pgxmock.NewPool()
+	if err != nil {
+		t.Fatalf("failed to open pgxmock database: %v", err)
+	}
+	cfg := &DocInitConfig{}
+	cfg.DBPool = pool
+	cfg.DBQueries = repository.New(pool)
+	mockStore := objectstoremock.NewMockS3Client(t)
+	cfg.StoreClient = mockStore
+
+	svc := New(cfg, &Services{
+		Job: job.New(cfg, &job.Services{
+			Client: client.New(cfg),
+		}),
+	})
+
+	doc := document.Document{
+		ID:    uuid.New(),
+		JobID: uuid.New(),
+		Hash:  "example_hash",
+	}
+	bucket := "eample_bucket"
+	location := "/i/am/here"
+
+	pool.ExpectQuery("-- name: GetDocumentIDByHash :one").WithArgs(doc.Hash, database.MustToDBUUID(doc.JobID)).WillReturnRows(
+		pgxmock.NewRows([]string{"id"}).
+			AddRow(database.MustToDBUUID(doc.ID)),
+	)
+
+	mockStore.EXPECT().
+		HeadObject(
+			mock.Anything,
+			mock.MatchedBy(func(in *s3.HeadObjectInput) bool {
+				return *in.Bucket == bucket && *in.Key == location
+			}),
+			mock.Anything,
+		).
+		Return(&s3.HeadObjectOutput{
+			ETag: &doc.Hash,
+		}, nil)
+
+	params, err := svc.getCreateParams(ctx, &Create{
+		JobID:    doc.JobID,
+		Location: location,
+		Bucket:   bucket,
+	})
+	assert.NoError(t, err)
+	dbid := database.MustToDBUUID(doc.ID)
+	assert.Equal(t, &createDocumentParams{
+		ID:       &dbid,
+		JobID:    database.MustToDBUUID(doc.JobID),
+		Hash:     doc.Hash,
+		Bucket:   bucket,
+		Location: location,
+	}, params)
+}
+
+func TestGetCreateParamsCurrentDocErr(t *testing.T) {
+	ctx := context.Background()
+
+	pool, err := pgxmock.NewPool()
+	if err != nil {
+		t.Fatalf("failed to open pgxmock database: %v", err)
+	}
+	cfg := &DocInitConfig{}
+	cfg.DBPool = pool
+	cfg.DBQueries = repository.New(pool)
+	mockStore := objectstoremock.NewMockS3Client(t)
+	cfg.StoreClient = mockStore
+
+	svc := New(cfg, &Services{
+		Job: job.New(cfg, &job.Services{
+			Client: client.New(cfg),
+		}),
+	})
+
+	doc := document.Document{
+		JobID: uuid.New(),
+		Hash:  "example_hash",
+	}
+	bucket := "eample_bucket"
+	location := "/i/am/here"
+
+	pool.ExpectQuery("-- name: GetDocumentIDByHash :one").WithArgs(doc.Hash, database.MustToDBUUID(doc.JobID)).
+		WillReturnError(errors.New("db err"))
+
+	mockStore.EXPECT().
+		HeadObject(
+			mock.Anything,
+			mock.MatchedBy(func(in *s3.HeadObjectInput) bool {
+				return *in.Bucket == bucket && *in.Key == location
+			}),
+			mock.Anything,
+		).
+		Return(&s3.HeadObjectOutput{
+			ETag: &doc.Hash,
+		}, nil)
+
+	_, err = svc.getCreateParams(ctx, &Create{
+		JobID:    doc.JobID,
+		Location: location,
+		Bucket:   bucket,
+	})
+	assert.Error(t, err)
+}
+
+func TestSubmitCreate(t *testing.T) {
+	ctx := context.Background()
+
+	pool, err := pgxmock.NewPool()
+	if err != nil {
+		t.Fatalf("failed to open pgxmock database: %v", err)
+	}
+	cfg := &DocInitConfig{}
+	cfg.DBPool = pool
+	cfg.DBQueries = repository.New(pool)
+
+	svc := New(cfg, &Services{
+		Job: job.New(cfg, &job.Services{
+			Client: client.New(cfg),
+		}),
+	})
+
+	doc := document.Document{
+		ID:    uuid.New(),
+		JobID: uuid.New(),
+		Hash:  "example_hash",
+	}
+	bucket := "eample_bucket"
+	location := "/i/am/here"
+
+	pool.ExpectBegin()
+	pool.ExpectQuery("name: CreateDocument :one").WithArgs(database.MustToDBUUID(doc.JobID), doc.Hash).
+		WillReturnRows(
+			pgxmock.NewRows([]string{"id"}).
+				AddRow(database.MustToDBUUID(doc.ID)),
+		)
+	pool.ExpectExec("name: AddDocumentEntry :exec").WithArgs(database.MustToDBUUID(doc.ID), bucket, location).
+		WillReturnResult(pgxmock.NewResult("", 1))
+	pool.ExpectCommit()
+
+	id, err := svc.submitCreate(ctx, &createDocumentParams{
+		JobID:    database.MustToDBUUID(doc.JobID),
+		Location: location,
+		Bucket:   bucket,
+		Hash:     doc.Hash,
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, doc.ID, id)
+}
+
+func TestSubmitCreateExists(t *testing.T) {
+	ctx := context.Background()
+
+	pool, err := pgxmock.NewPool()
+	if err != nil {
+		t.Fatalf("failed to open pgxmock database: %v", err)
+	}
+	cfg := &DocInitConfig{}
+	cfg.DBPool = pool
+	cfg.DBQueries = repository.New(pool)
+
+	svc := New(cfg, &Services{
+		Job: job.New(cfg, &job.Services{
+			Client: client.New(cfg),
+		}),
+	})
+
+	doc := document.Document{
+		ID:    uuid.New(),
+		JobID: uuid.New(),
+		Hash:  "example_hash",
+	}
+	bucket := "eample_bucket"
+	location := "/i/am/here"
+
+	pool.ExpectBegin()
+	pool.ExpectExec("name: AddDocumentEntry :exec").WithArgs(database.MustToDBUUID(doc.ID), bucket, location).
+		WillReturnResult(pgxmock.NewResult("", 1))
+	pool.ExpectCommit()
+
+	docid := database.MustToDBUUID(doc.ID)
+	id, err := svc.submitCreate(ctx, &createDocumentParams{
+		ID:       &docid,
+		JobID:    database.MustToDBUUID(doc.JobID),
+		Location: location,
+		Bucket:   bucket,
+		Hash:     doc.Hash,
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, doc.ID, id)
+}

internal/document/init/service.go

@@ -3,6 +3,7 @@ package documentinit
 import (
 	"queryorchestration/internal/job"
 	"queryorchestration/internal/serviceconfig"
+	"queryorchestration/internal/serviceconfig/objectstore"
 	"queryorchestration/internal/serviceconfig/queue/documentclean"
 )
 
@@ -13,6 +14,7 @@ type Services struct {
 type ConfigProvider interface {
 	serviceconfig.ConfigProvider
 	documentclean.ConfigProvider
+	objectstore.ConfigProvider
 }
 
 type Service struct {

internal/document/service.go

@@ -9,10 +9,9 @@ import (
 type Location = string
 
 type Document struct {
-	ID       uuid.UUID
+	ID    uuid.UUID
-	JobID    uuid.UUID
+	JobID uuid.UUID
-	Hash     string
+	Hash  string
-	Location Location
 }
 
 type Service struct {

internal/job/collector/create.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"queryorchestration/internal/database"
 	"queryorchestration/internal/database/repository"
-	"queryorchestration/internal/server/validation"
+	"queryorchestration/internal/validation"
 
 	"github.com/google/uuid"
 	"github.com/jackc/pgx/v5/pgtype"

internal/job/collector/update.go

@@ -5,7 +5,7 @@ import (
 	"errors"
 	"queryorchestration/internal/database"
 	"queryorchestration/internal/database/repository"
-	"queryorchestration/internal/server/validation"
+	"queryorchestration/internal/validation"
 
 	"github.com/google/uuid"
 	"github.com/jackc/pgx/v5/pgtype"

internal/query/normalize.go

@@ -7,7 +7,7 @@ import (
 	"fmt"
 	"queryorchestration/internal/database"
 	resultprocessor "queryorchestration/internal/query/result/processor"
-	"queryorchestration/internal/server/validation"
+	"queryorchestration/internal/validation"
 	"strings"
 
 	"github.com/google/uuid"

internal/query/test_test.go

@@ -48,10 +48,9 @@ func TestTest(t *testing.T) {
 		JobID: uuid.New(),
 	}
 	doc := document.Document{
-		ID:       uuid.New(),
+		ID:    uuid.New(),
-		JobID:    coll.JobID,
+		JobID: coll.JobID,
-		Hash:     "example_hash",
+		Hash:  "example_hash",
-		Location: "example_location",
 	}
 	params := &query.Test{
 		QueryID:      uuid.New(),
@@ -62,8 +61,8 @@ func TestTest(t *testing.T) {
 	reqID := database.MustToDBUUID(uuid.New())
 	pool.ExpectQuery("name: GetDocument :one").WithArgs(database.MustToDBUUID(doc.ID)).
 		WillReturnRows(
-			pgxmock.NewRows([]string{"id", "jobId", "hash", "location"}).
+			pgxmock.NewRows([]string{"id", "jobId", "hash"}).
-				AddRow(database.MustToDBUUID(doc.ID), database.MustToDBUUID(doc.JobID), doc.Hash, doc.Location),
+				AddRow(database.MustToDBUUID(doc.ID), database.MustToDBUUID(doc.JobID), doc.Hash),
 		)
 	pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(doc.JobID)).
 		WillReturnRows(

internal/server/runner/listener_test.go

@@ -44,8 +44,8 @@ func TestNew(t *testing.T) {
 	cfg.QueueURL = test.CreateQueue(t, ctx, cfg, "queueName")
 
 	srvPtr, err := New(ctx, cfg)
-	assert.NotNil(t, srvPtr)
 	assert.NoError(t, err)
+	assert.NotNil(t, srvPtr)
 }
 
 func TestListen(t *testing.T) {

internal/serviceconfig/database/pool.go

@@ -88,7 +88,7 @@ func (b *DBConfig) DBPing(ctx context.Context) error {
 				slog.Info("Successful database ping")
 				return nil
 			}
-			slog.Info("Attempted database ping")
+			slog.Info("Attempted database ping", "host", b.DBHost, "port", b.DBPort)
 		case <-ctx.Done():
 			return ctx.Err()
 		}

internal/serviceconfig/objectstore/config.go

@@ -12,7 +12,6 @@ import (
 )
 
 type ObjectStoreConfig struct {
-	BucketName        string `env:"BUCKET_NAME,required,notEmpty"`
 	AWSEndpointUrlS3  string `env:"AWS_ENDPOINT_URL_S3"`
 	AWSS3UsePathStyle bool   `env:"AWS_S3_USE_PATH_STYLE" envDefault:"false"`
 	StoreClient       S3Client
@@ -74,7 +73,7 @@ func (c *ObjectStoreConfig) PingStoreByName(ctx context.Context, name string) er
 				slog.Info("Successful bucket ping", "name", name)
 				return nil
 			}
-			slog.Info("Attempted bucket ping", "name", name)
+			slog.Info("Attempted bucket ping", "name", name, "address", c.GetS3Endpoint())
 		case <-ctx.Done():
 			return ctx.Err()
 		}

internal/serviceconfig/queue/config.go

@@ -58,7 +58,7 @@ func (c *QueueConfig) PingQueueByURL(ctx context.Context, url string) error {
 				slog.Info("Successful queue ping", "url", url, "approx_msgs", r.Attributes[string(types.QueueAttributeNameApproximateNumberOfMessages)])
 				return nil
 			}
-			slog.Info("Attempted queue ping", "url", url)
+			slog.Info("Attempted queue ping", "url", url, "address", c.GetSQSEndpoint())
 		case <-ctx.Done():
 			return ctx.Err()
 		}

internal/serviceconfig/queue/receive.go

@@ -15,8 +15,8 @@ func (c *QueueConfig) ReceiveFromQueue(ctx context.Context, params *ReceiveParam
 	return c.QueueClient.ReceiveMessage(ctx, &sqs.ReceiveMessageInput{
 		QueueUrl:              &params.QueueURL,
 		MaxNumberOfMessages:   1,
-		WaitTimeSeconds:       20,
+		WaitTimeSeconds:       30,
-		VisibilityTimeout:     30,
+		VisibilityTimeout:     5,
 		MessageAttributeNames: params.Attributes,
 	})
 }

internal/test/queue.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"queryorchestration/internal/serviceconfig"
 	"queryorchestration/internal/serviceconfig/queue"
+	"regexp"
 	"testing"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
@@ -38,20 +39,20 @@ func AssertMessage(t *testing.T, ctx context.Context, cfg serviceconfig.ConfigPr
 	return result.Messages[0]
 }
 
-func AssertMessageBody(t *testing.T, ctx context.Context, cfg serviceconfig.ConfigProvider, url string, body string) {
+func AssertMessageBody(t *testing.T, ctx context.Context, cfg serviceconfig.ConfigProvider, url string, body *regexp.Regexp) {
 	message := AssertMessage(t, ctx, cfg, &queue.ReceiveParams{
 		QueueURL: url,
 	})
 
-	assert.Equal(t, body, *message.Body)
+	assert.Regexp(t, body, *message.Body)
 }
 
-func AssertMessageAttr(t *testing.T, ctx context.Context, cfg serviceconfig.ConfigProvider, url string, name string, value string) {
+func AssertMessageAttr(t *testing.T, ctx context.Context, cfg serviceconfig.ConfigProvider, url string, name string, value *regexp.Regexp) {
 	message := AssertMessage(t, ctx, cfg, &queue.ReceiveParams{
 		QueueURL:   url,
 		Attributes: []string{name},
 	})
 
 	assert.NotNil(t, message.MessageAttributes[name])
-	assert.Equal(t, value, *(message.MessageAttributes[name]).StringValue)
+	assert.Regexp(t, value, *(message.MessageAttributes[name]).StringValue)
 }

internal/test/queue_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"queryorchestration/internal/serviceconfig"
 	"queryorchestration/internal/serviceconfig/queue"
+	"regexp"
 	"testing"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
@@ -84,7 +85,7 @@ func TestAssertMessageBodyWait(t *testing.T) {
 	})
 	assert.NoError(t, err)
 
-	AssertMessageBody(t, ctx, cfg, url, "\"body\"")
+	AssertMessageBody(t, ctx, cfg, url, regexp.MustCompile("\"body\""))
 }
 
 func TestAssertMessageAttrWait(t *testing.T) {
@@ -119,5 +120,5 @@ func TestAssertMessageAttrWait(t *testing.T) {
 	})
 	assert.NoError(t, err)
 
-	AssertMessageAttr(t, ctx, cfg, url, name, value)
+	AssertMessageAttr(t, ctx, cfg, url, name, regexp.MustCompile(value))
 }

internal/validation/validation_test.go

@@ -1,7 +1,7 @@
 package validation_test
 
 import (
-	"queryorchestration/internal/server/validation"
+	"queryorchestration/internal/validation"
 	"testing"
 
 	"github.com/google/uuid"

test/docInitRunner/docinitrunner_test.go

@@ -11,8 +11,11 @@ import (
 	documentcleanc "queryorchestration/internal/serviceconfig/queue/documentclean"
 	"queryorchestration/internal/test"
 	queryservice "queryorchestration/pkg/queryService"
+	"regexp"
+	"strings"
 	"testing"
 
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -55,7 +58,6 @@ func TestDocInitRunner(t *testing.T) {
 				Name: test.DocInitRunner,
 				Env: map[string]string{
 					"DOCUMENT_CLEAN_URL": doccleanurl,
-					"BUCKET_NAME":        bucketName,
 				},
 			},
 		},
@@ -89,21 +91,24 @@ func TestDocInitRunner(t *testing.T) {
 	})
 	assert.NoError(t, err)
 
-	// TODO - upload example doc
 	location := "/i/am/here"
-
+	body := strings.NewReader("hello world")
-	document := documentinit.Create{
+	_, err = cfg.StoreClient.PutObject(ctx, &s3.PutObjectInput{
-		JobID:    jobRes.JSON201.Id,
+		Bucket: &bucketName,
-		Location: location,
+		Key:    &location,
-	}
+		Body:   body,
-
-	err = cfg.SendToQueue(ctx, &queue.SendParams{
-		QueueURL: net.Runners[test.DocInitRunner].URI,
-		Body:     document,
 	})
 	assert.NoError(t, err)
 
-	_ = test.AssertMessage(t, ctx, cfg, &queue.ReceiveParams{
+	err = cfg.SendToQueue(ctx, &queue.SendParams{
-		QueueURL: doccleanurl,
+		QueueURL: net.Runners[test.DocInitRunner].URI,
+		Body: documentinit.Create{
+			JobID:    jobRes.JSON201.Id,
+			Location: location,
+		},
 	})
+	assert.NoError(t, err)
+
+	resRegex := regexp.MustCompile(`{"id": "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"`)
+	test.AssertMessageBody(t, ctx, cfg, doccleanurl, resRegex)
 }