jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/docclean (pull request #55)

Browse Source 
Doc Clean Structure

* outline

* isdocclean

* placeholder for clean log

* versionplustesting

* versionplustesting

* testspassed
This commit is contained in:
Michael McGuinness
2025-02-07 12:12:51 +00:00
parent e146725825
commit 90353d8161
56 changed files with 1106 additions and 316 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/docInitRunner/runner.go
+33 -23
View File
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"log/slog"
"queryorchestration/internal/document"
documentinit "queryorchestration/internal/document/init"
"github.com/go-playground/validator/v10"
@@ -31,26 +32,33 @@ func New(validator *validator.Validate, svc *Services) Runner {
}
}
type S3Bucket struct {
Name string `json:"name"`
Arn string `json:"arn"`
}
type S3Object struct {
Key string `json:"key"`
Size int64 `json:"size"`
ETag string `json:"eTag"`
VersionId string `json:"versionId"`
}
type S3EventRecordDetails struct {
Bucket S3Bucket `json:"bucket"`
Object S3Object `json:"object"`
}
type S3EventRecord struct {
EventVersion string `json:"eventVersion"`
EventSource string `json:"eventSource"`
AwsRegion string `json:"awsRegion"`
EventTime string `json:"eventTime"`
EventName string `json:"eventName"`
S3 S3EventRecordDetails `json:"s3"`
}
type S3EventNotification struct {
Records []struct {
EventVersion string `json:"eventVersion"`
EventSource string `json:"eventSource"`
AwsRegion string `json:"awsRegion"`
EventTime string `json:"eventTime"`
EventName string `json:"eventName"`
S3 struct {
Bucket struct {
Name string `json:"name"`
Arn string `json:"arn"`
} `json:"bucket"`
Object struct {
Key string `json:"key"`
Size int64 `json:"size"`
ETag string `json:"eTag"`
VersionId string `json:"versionId"`
} `json:"object"`
} `json:"s3"`
} `json:"Records"`
Records []S3EventRecord `json:"Records"`
}
func (s Runner) Process(ctx context.Context, req *types.Message) error {
@@ -79,10 +87,12 @@ func (s Runner) Process(ctx context.Context, req *types.Message) error {
}
_, err = s.svc.Document.Create(ctx, &documentinit.Create{
JobID: jobID,
Location: record.S3.Object.Key,
Bucket: record.S3.Bucket.Name,
Hash: record.S3.Object.ETag,
JobID: jobID,
Location: document.Location{
Bucket: record.S3.Bucket.Name,
Key: record.S3.Object.Key,
},
Hash: record.S3.Object.ETag,
})
if err != nil {
return err
api/docInitRunner/runner_test.go
+26 -13
View File
@@ -3,6 +3,7 @@ package docinitrunner_test
import (
"context"
"encoding/json"
"fmt"
docinitrunner "queryorchestration/api/docInitRunner"
"queryorchestration/internal/client"
"queryorchestration/internal/database"
@@ -58,10 +59,27 @@ func TestDocInitRunner(t *testing.T) {
ID: uuid.New(),
ClientID: uuid.New(),
}
doc := documentinit.Create{
JobID: j.ID,
Bucket: "bucket_name",
Location: "/I/am/here",
bucketName := "bucketName"
location := fmt.Sprintf("aaa/%s/aaa", j.ID.String())
docinfo := document.Document{
ID: uuid.New(),
Hash: "example_hash",
}
doc := docinitrunner.S3EventNotification{
Records: []docinitrunner.S3EventRecord{
{
EventName: "ObjectCreated:Put",
S3: docinitrunner.S3EventRecordDetails{
Bucket: docinitrunner.S3Bucket{
Name: bucketName,
},
Object: docinitrunner.S3Object{
Key: location,
ETag: docinfo.Hash,
},
},
},
},
}
bodyBytes, err := json.Marshal(doc)
assert.NoError(t, err)
@@ -70,29 +88,24 @@ func TestDocInitRunner(t *testing.T) {
Body: &body,
}
docinfo := document.Document{
ID: uuid.New(),
Hash: "example_hash",
}
pool.ExpectQuery("name: GetJob :one").WithArgs(database.MustToDBUUID(j.ID)).WillReturnRows(
pgxmock.NewRows([]string{"id", "clientId", "canSync"}).
AddRow(database.MustToDBUUID(j.ID), database.MustToDBUUID(j.ClientID), j.CanSync),
)
pool.ExpectQuery("-- name: GetClient :one").WithArgs(database.MustToDBUUID(j.ClientID)).WillReturnRows(
pool.ExpectQuery("name: GetClient :one").WithArgs(database.MustToDBUUID(j.ClientID)).WillReturnRows(
pgxmock.NewRows([]string{"id", "name", "canSync"}).
AddRow(database.MustToDBUUID(j.ClientID), "client_name", true),
)
pool.ExpectQuery("-- name: GetDocumentIDByHash :one").WithArgs(docinfo.Hash, database.MustToDBUUID(doc.JobID)).WillReturnRows(
pool.ExpectQuery("name: GetDocumentIDByHash :one").WithArgs(docinfo.Hash, database.MustToDBUUID(j.ID)).WillReturnRows(
pgxmock.NewRows([]string{"id"}),
)
pool.ExpectBegin()
pool.ExpectQuery("name: CreateDocument :one").WithArgs(database.MustToDBUUID(doc.JobID), docinfo.Hash).
pool.ExpectQuery("name: CreateDocument :one").WithArgs(database.MustToDBUUID(j.ID), docinfo.Hash).
WillReturnRows(
pgxmock.NewRows([]string{"id"}).
AddRow(database.MustToDBUUID(docinfo.ID)),
)
pool.ExpectExec("name: AddDocumentEntry :exec").WithArgs(database.MustToDBUUID(docinfo.ID), doc.Bucket, doc.Location).
pool.ExpectExec("name: AddDocumentEntry :exec").WithArgs(database.MustToDBUUID(docinfo.ID), bucketName, location).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()