jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/localstackands3object (pull request #54)

Browse Source 
LocalStack and S3 Objects

* firstround

* cleanupintegration

* tidyports
This commit is contained in:
Michael McGuinness
2025-02-06 15:00:26 +00:00
parent 9e6328d8f9
commit e146725825
17 changed files with 381 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/document/init/create.go
+28 -17
View File
@@ -4,23 +4,25 @@ import (
"context"
"database/sql"
"errors"
"fmt"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/document"
documentclean "queryorchestration/internal/document/clean"
"queryorchestration/internal/job"
"queryorchestration/internal/serviceconfig/queue"
"regexp"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/sqs/types"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
)
type Create struct {
JobID uuid.UUID `json:"jobId" validate:"required,uuid"`
Location document.Location `json:"location" validate:"required"`
Bucket string `json:"bucket" validate:"required"`
JobID uuid.UUID
Location document.Location
Bucket string
Hash string
}
func (s *Service) Create(ctx context.Context, doc *Create) (uuid.UUID, error) {
@@ -39,9 +41,11 @@ func (s *Service) Create(ctx context.Context, doc *Create) (uuid.UUID, error) {
return uuid.Nil, err
}
err = s.informCreate(ctx, id, j)
if err != nil {
return uuid.Nil, err
if params.ID == nil {
err = s.informCreate(ctx, id, j)
if err != nil {
return uuid.Nil, err
}
}
return id, nil
@@ -56,17 +60,9 @@ type createDocumentParams struct {
}
func (s *Service) getCreateParams(ctx context.Context, doc *Create) (*createDocumentParams, error) {
res, err := s.cfg.GetStoreClient().HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &doc.Bucket,
Key: &doc.Location,
})
if err != nil {
return nil, err
}
idbyhash, err := s.cfg.GetDBQueries().GetDocumentIDByHash(ctx, &repository.GetDocumentIDByHashParams{
Jobid: database.MustToDBUUID(doc.JobID),
Hash: *res.ETag,
Hash: doc.Hash,
})
var docID *pgtype.UUID
if err != nil && !errors.Is(err, sql.ErrNoRows) {
@@ -78,7 +74,7 @@ func (s *Service) getCreateParams(ctx context.Context, doc *Create) (*createDocu
return &createDocumentParams{
ID: docID,
JobID: database.MustToDBUUID(doc.JobID),
Hash: *res.ETag,
Hash: doc.Hash,
Bucket: doc.Bucket,
Location: doc.Location,
}, nil
@@ -141,3 +137,18 @@ func (s *Service) informCreate(ctx context.Context, id uuid.UUID, j *job.Job) er
return nil
}
func (s *Service) GetJobIDFromKey(key string) (uuid.UUID, error) {
re := regexp.MustCompile(`^.+/(.+)/.+$`)
match := re.FindStringSubmatch(key)
if match == nil || len(match) != 2 {
return uuid.Nil, fmt.Errorf("no job id match found in key")
}
id, err := uuid.Parse(match[1])
if err != nil {
return uuid.Nil, err
}
return id, nil
}