Merged in feature/localstackands3object (pull request #54)

LocalStack and S3 Objects

* firstround

* cleanupintegration

* tidyports

api/docInitRunner/runner.go

@@ -3,6 +3,9 @@ package docinitrunner
 import (
 	"context"
 	"encoding/json"
+	"errors"
+	"fmt"
+	"log/slog"
 	documentinit "queryorchestration/internal/document/init"
 
 	"github.com/go-playground/validator/v10"
@@ -28,8 +31,30 @@ func New(validator *validator.Validate, svc *Services) Runner {
 	}
 }
 
+type S3EventNotification struct {
+	Records []struct {
+		EventVersion string `json:"eventVersion"`
+		EventSource  string `json:"eventSource"`
+		AwsRegion    string `json:"awsRegion"`
+		EventTime    string `json:"eventTime"`
+		EventName    string `json:"eventName"`
+		S3           struct {
+			Bucket struct {
+				Name string `json:"name"`
+				Arn  string `json:"arn"`
+			} `json:"bucket"`
+			Object struct {
+				Key       string `json:"key"`
+				Size      int64  `json:"size"`
+				ETag      string `json:"eTag"`
+				VersionId string `json:"versionId"`
+			} `json:"object"`
+		} `json:"s3"`
+	} `json:"Records"`
+}
+
 func (s Runner) Process(ctx context.Context, req *types.Message) error {
-	var body documentinit.Create
+	var body S3EventNotification
 	err := json.Unmarshal([]byte(*req.Body), &body)
 	if err != nil {
 		return err
@@ -40,9 +65,39 @@ func (s Runner) Process(ctx context.Context, req *types.Message) error {
 		return err
 	}
 
-	_, err = s.svc.Document.Create(ctx, &body)
-	if err != nil {
-		return err
+	hasErr := false
+
+	for _, record := range body.Records {
+		err := func() error {
+			if record.EventName != "ObjectCreated:Put" {
+				return fmt.Errorf("invalid event name: %s", record.EventName)
+			}
+
+			jobID, err := s.svc.Document.GetJobIDFromKey(record.S3.Object.Key)
+			if err != nil {
+				return err
+			}
+
+			_, err = s.svc.Document.Create(ctx, &documentinit.Create{
+				JobID:    jobID,
+				Location: record.S3.Object.Key,
+				Bucket:   record.S3.Bucket.Name,
+				Hash:     record.S3.Object.ETag,
+			})
+			if err != nil {
+				return err
+			}
+
+			return nil
+		}()
+		if err != nil {
+			hasErr = true
+			slog.Error("Error processing record", "err", err)
+		}
+	}
+
+	if hasErr {
+		return errors.New("error processing records")
 	}
 
 	return nil

api/docInitRunner/runner_test.go

@@ -17,13 +17,11 @@ import (
 	objectstoremock "queryorchestration/mocks/objectstore"
 	"testing"
 
-	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/sqs/types"
 	"github.com/go-playground/validator/v10"
 	"github.com/google/uuid"
 	"github.com/pashagolub/pgxmock/v3"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/mock"
 )
 
 type DocInitConfig struct {
@@ -98,18 +96,6 @@ func TestDocInitRunner(t *testing.T) {
 		WillReturnResult(pgxmock.NewResult("", 1))
 	pool.ExpectCommit()
 
-	mockStore.EXPECT().
-		HeadObject(
-			mock.Anything,
-			mock.MatchedBy(func(in *s3.HeadObjectInput) bool {
-				return *in.Bucket == doc.Bucket && *in.Key == doc.Location
-			}),
-			mock.Anything,
-		).
-		Return(&s3.HeadObjectOutput{
-			ETag: &docinfo.Hash,
-		}, nil)
-
 	err = runner.Process(ctx, msg)
 	assert.NoError(t, err)
 }