Merged in feature/preserve-folder-paths (pull request #179)

support folder paths

* support folder paths

* s3 storage docs

api/docInitRunner/runner.go

@@ -2,8 +2,11 @@ package docinitrunner
 
 import (
 	"context"
+	"database/sql"
+	"errors"
 	"log/slog"
 
+	"queryorchestration/internal/database/repository"
 	documentinit "queryorchestration/internal/document/init"
 	"queryorchestration/internal/serviceconfig/objectstore"
 )
@@ -12,6 +15,7 @@ const Name = "docInitRunner"
 
 type Services struct {
 	Document *documentinit.Service
+	Queries  *repository.Queries
 }
 
 type Runner struct {
@@ -37,15 +41,30 @@ func (s Runner) Process(ctx context.Context, body Body) bool {
 		return false
 	}
 
-	id, err := s.svc.Document.Create(ctx, &documentinit.Create{
-		Key:    key,
+	// Get filename from documentUploads table
+	var filename *string
+	upload, err := s.svc.Queries.GetDocumentUploadByKey(ctx, &repository.GetDocumentUploadByKeyParams{
 		Bucket: body.Bucket,
-		Hash:   body.Hash,
+		Key:    body.Key,
+	})
+	if err != nil && !errors.Is(err, sql.ErrNoRows) {
+		slog.Error("unable to get document upload", "bucket", body.Bucket, "key", body.Key, "error", err)
+		return false
+	}
+	if err == nil && upload.Filename != nil {
+		filename = upload.Filename
+	}
+
+	id, err := s.svc.Document.Create(ctx, &documentinit.Create{
+		Key:      key,
+		Bucket:   body.Bucket,
+		Hash:     body.Hash,
+		Filename: filename,
 	})
 	if err != nil {
 		return false
 	}
-	slog.Debug("created document", "id", id)
+	slog.Debug("created document", "id", id, "filename", filename)
 
 	return true
 }

api/docInitRunner/runner_test.go

@@ -14,6 +14,8 @@ import (
 	"queryorchestration/internal/serviceconfig/queue/documentsync"
 	"queryorchestration/internal/test"
 
+	"github.com/google/uuid"
+	"github.com/jackc/pgx/v5/pgtype"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -25,6 +27,11 @@ type DocInitConfig struct {
 	objectstore.ObjectStoreConfig
 }
 
+// stringPtr returns a pointer to a string
+func stringPtr(s string) *string {
+	return &s
+}
+
 func TestDocInitRunner(t *testing.T) {
 	cfg := &DocInitConfig{}
 	test.CreateDB(t, cfg)
@@ -35,6 +42,7 @@ func TestDocInitRunner(t *testing.T) {
 
 	runner := docinitrunner.New(&docinitrunner.Services{
 		Document: documentinit.New(cfg),
+		Queries:  cfg.GetDBQueries(),
 	})
 
 	err := cfg.GetDBQueries().CreateClient(t.Context(), &repository.CreateClientParams{
@@ -48,6 +56,24 @@ func TestDocInitRunner(t *testing.T) {
 		ClientID:  "clientid",
 		CreatedAt: time.Now().UTC(),
 	}
+
+	// Create a document upload record to simulate the upload pipeline
+	uploadID := uuid.New()
+	err = cfg.GetDBQueries().AddDocumentUpload(t.Context(), &repository.AddDocumentUploadParams{
+		ID:       uploadID,
+		Clientid: "clientid",
+		Bucket:   "bucket",
+		Key:      location.String(),
+		Part:     1,
+		Createdat: pgtype.Timestamp{
+			Valid: true,
+			Time:  time.Now().UTC(),
+		},
+		Filename: stringPtr("test-document.pdf"),
+		BatchID:  nil,
+	})
+	require.NoError(t, err)
+
 	doc := docinitrunner.Body{
 		Bucket: "bucket",
 		Key:    location.String(),