Merged in feature/preserve-folder-paths (pull request #179)

support folder paths

* support folder paths

* s3 storage docs

CLAUDE.MD

@@ -23,7 +23,7 @@ When running `task lint` output should always be clean. 0 errors.
 Warnings about variable not set are ok.  
 Lint output output must contain ‘Linting passed, A perfect score! well done!’ or you must fix an issue to resume.
 
-Use `task go:lint -- --fix` to fix go (file not formmatted) types of errors from lint. 
+Use `task go:lint -- --fix` to fix go (file not formmatted) types of errors from lint. Always run this before running `task fullsuite:ci`
 
 ## Essential Commands
 
@@ -31,6 +31,7 @@ Use `task go:lint -- --fix` to fix go (file not formmatted) types of errors from
 - `devbox shell` - Enter development environment (Nix-based)  I will start you already in the devbox shell for the project if there is one. 
 - `touch .env` - Create environment variables file
 - `task fullsuite:ci` - Complete development workflow (generate, build, lint, test) This must pass clean or its not working. If its output has the word `FAIL` then investigate what the issue is before calling the change complete.  
+- when running fullsuite:ci make sure that you have stopped the containers that run from a previous `task compose:refresh' if this applies. (use task compose:down to clean those up first) 
 
 ### Core Development Tasks
 - `task generate` - Generate all code (DB queries via SQLC, OpenAPI clients/servers, mocks, docs)

README.md

@@ -226,3 +226,50 @@ Key changes from production configuration:
 2. Set breakpoints and start debugging
 
 The debugger will connect to the running container and pause execution when breakpoints are hit, allowing you to inspect variables, step through code, and debug issues in the containerized environment.
+
+## S3 Storage Format
+
+The system uses a structured naming convention for S3 objects that embeds metadata for parsing, recovery, and organization.
+
+### Object Key Structure
+
+S3 object keys follow this format:
+```
+{client_id}/{location}/{date}/{part}/{filename}
+```
+
+#### Example
+```
+test_client_1757026403/import/20250904/0/2025-09-04T225324Z~test_client_1757026403~import~56df8aa9-873c-48ea-a0e4-bc4f6b8ad69e~019916ef-46ab-73f7-b97b-5709df5c3985
+```
+
+### Components
+
+#### Directory Path (Prefix)
+- **Client ID**: Unique identifier for the client (e.g., `test_client_1757026403`)
+- **Location**: Processing stage - `import`, `text`, or `export`
+- **Date**: Date in `YYYYMMDD` format (e.g., `20250904`)
+- **Part**: Partition number for storage distribution (e.g., `0`)
+
+#### Filename
+The filename uses tilde (`~`) as a delimiter and contains:
+- **Timestamp**: Full ISO timestamp (e.g., `2025-09-04T225324Z`)
+- **Client ID**: Repeated for self-contained metadata
+- **Location**: Repeated for self-contained metadata
+- **Entity ID**: UUID for the specific document (e.g., `56df8aa9-873c-48ea-a0e4-bc4f6b8ad69e`)
+- **Batch ID**: Optional UUID for batch uploads (e.g., `019916ef-46ab-73f7-b97b-5709df5c3985`)
+- **File Extension**: Optional file type extension (e.g., `.pdf`)
+
+### Location Types
+- **import**: Initial document upload location
+- **text**: Text extraction results storage
+- **export**: Export results storage
+
+### Partitioning Strategy
+The part number (`0-n`) is used to distribute files across multiple directories, preventing performance issues with too many objects in a single S3 prefix. Export location does not use partitioning.
+
+### Metadata Encoding
+The filename contains all necessary metadata for document recovery and processing:
+- Self-contained recovery capability from filename alone
+- Support for both path-based and filename-based queries
+- Human-readable format for debugging and troubleshooting

api/docInitRunner/runner.go

@@ -2,8 +2,11 @@ package docinitrunner
 
 import (
 	"context"
+	"database/sql"
+	"errors"
 	"log/slog"
 
+	"queryorchestration/internal/database/repository"
 	documentinit "queryorchestration/internal/document/init"
 	"queryorchestration/internal/serviceconfig/objectstore"
 )
@@ -12,6 +15,7 @@ const Name = "docInitRunner"
 
 type Services struct {
 	Document *documentinit.Service
+	Queries  *repository.Queries
 }
 
 type Runner struct {
@@ -37,15 +41,30 @@ func (s Runner) Process(ctx context.Context, body Body) bool {
 		return false
 	}
 
-	id, err := s.svc.Document.Create(ctx, &documentinit.Create{
-		Key:    key,
+	// Get filename from documentUploads table
+	var filename *string
+	upload, err := s.svc.Queries.GetDocumentUploadByKey(ctx, &repository.GetDocumentUploadByKeyParams{
 		Bucket: body.Bucket,
-		Hash:   body.Hash,
+		Key:    body.Key,
+	})
+	if err != nil && !errors.Is(err, sql.ErrNoRows) {
+		slog.Error("unable to get document upload", "bucket", body.Bucket, "key", body.Key, "error", err)
+		return false
+	}
+	if err == nil && upload.Filename != nil {
+		filename = upload.Filename
+	}
+
+	id, err := s.svc.Document.Create(ctx, &documentinit.Create{
+		Key:      key,
+		Bucket:   body.Bucket,
+		Hash:     body.Hash,
+		Filename: filename,
 	})
 	if err != nil {
 		return false
 	}
-	slog.Debug("created document", "id", id)
+	slog.Debug("created document", "id", id, "filename", filename)
 
 	return true
 }

api/docInitRunner/runner_test.go

@@ -14,6 +14,8 @@ import (
 	"queryorchestration/internal/serviceconfig/queue/documentsync"
 	"queryorchestration/internal/test"
 
+	"github.com/google/uuid"
+	"github.com/jackc/pgx/v5/pgtype"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -25,6 +27,11 @@ type DocInitConfig struct {
 	objectstore.ObjectStoreConfig
 }
 
+// stringPtr returns a pointer to a string
+func stringPtr(s string) *string {
+	return &s
+}
+
 func TestDocInitRunner(t *testing.T) {
 	cfg := &DocInitConfig{}
 	test.CreateDB(t, cfg)
@@ -35,6 +42,7 @@ func TestDocInitRunner(t *testing.T) {
 
 	runner := docinitrunner.New(&docinitrunner.Services{
 		Document: documentinit.New(cfg),
+		Queries:  cfg.GetDBQueries(),
 	})
 
 	err := cfg.GetDBQueries().CreateClient(t.Context(), &repository.CreateClientParams{
@@ -48,6 +56,24 @@ func TestDocInitRunner(t *testing.T) {
 		ClientID:  "clientid",
 		CreatedAt: time.Now().UTC(),
 	}
+
+	// Create a document upload record to simulate the upload pipeline
+	uploadID := uuid.New()
+	err = cfg.GetDBQueries().AddDocumentUpload(t.Context(), &repository.AddDocumentUploadParams{
+		ID:       uploadID,
+		Clientid: "clientid",
+		Bucket:   "bucket",
+		Key:      location.String(),
+		Part:     1,
+		Createdat: pgtype.Timestamp{
+			Valid: true,
+			Time:  time.Now().UTC(),
+		},
+		Filename: stringPtr("test-document.pdf"),
+		BatchID:  nil,
+	})
+	require.NoError(t, err)
+
 	doc := docinitrunner.Body{
 		Bucket: "bucket",
 		Key:    location.String(),

cmd/docInitRunner/main.go

@@ -43,6 +43,7 @@ func main() {
 
 		return docinitrunner.New(&docinitrunner.Services{
 			Document: docinit,
+			Queries:  cfg.GetDBQueries(),
 		})
 	}
 

internal/database/migrations/00000000000105_add_document_filename.down.sql

@@ -0,0 +1,3 @@
+-- Remove filename column and index
+DROP INDEX IF EXISTS idx_documents_filename;
+ALTER TABLE documents DROP COLUMN IF EXISTS filename;

internal/database/migrations/00000000000105_add_document_filename.up.sql

@@ -0,0 +1,5 @@
+-- Add filename column to documents table to store original filename with path
+ALTER TABLE documents ADD COLUMN filename TEXT;
+
+-- Add index for efficient filename searches
+CREATE INDEX idx_documents_filename ON documents(filename);

internal/database/migrations/00000000000106_add_documentuploads_filename.down.sql

@@ -0,0 +1,2 @@
+DROP INDEX IF EXISTS idx_documentuploads_filename;
+ALTER TABLE documentUploads DROP COLUMN IF EXISTS filename;

internal/database/migrations/00000000000106_add_documentuploads_filename.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE documentUploads ADD COLUMN filename TEXT;
+CREATE INDEX idx_documentuploads_filename ON documentUploads(filename);

internal/database/migrations/00000000000107_add_documentuploads_batch_id.down.sql

@@ -0,0 +1,2 @@
+DROP INDEX IF EXISTS idx_documentuploads_batch_id;
+ALTER TABLE documentUploads DROP COLUMN IF EXISTS batch_id;

internal/database/migrations/00000000000107_add_documentuploads_batch_id.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE documentUploads ADD COLUMN batch_id uuid;
+CREATE INDEX idx_documentuploads_batch_id ON documentUploads(batch_id);

internal/database/migrations/00000000000108_add_documentuploads_clientid_index.down.sql

@@ -0,0 +1 @@
+DROP INDEX IF EXISTS idx_documentuploads_clientid;

internal/database/migrations/00000000000108_add_documentuploads_clientid_index.up.sql

@@ -0,0 +1 @@
+CREATE INDEX idx_documentuploads_clientid ON documentUploads(clientId);

internal/database/queries/document.sql

@@ -30,8 +30,11 @@ GROUP BY d.id, d.clientId, d.hash;
 -- name: ListDocumentsByClient :many
 SELECT id, hash from documents where clientId = @clientId;
 
+-- name: ListDocumentsByBatch :many
+SELECT id, hash, filename from documents where batch_id = @batch_id;
+
 -- name: CreateDocument :one
-INSERT INTO documents (clientId, hash, batch_id) VALUES ($1, $2, $3) RETURNING id;
+INSERT INTO documents (clientId, hash, batch_id, filename) VALUES ($1, $2, $3, $4) RETURNING id;
 
 -- name: AddDocumentEntry :exec
 INSERT INTO documentEntries (documentId, bucket, key) VALUES ($1, $2, $3);
@@ -46,7 +49,13 @@ SELECT id FROM documents WHERE hash = $1 and clientId = $2;
 SELECT id, totalCount FROM listDocumentIDs(@clientId, @batchSize, @pageOffset);
 
 -- name: AddDocumentUpload :exec
-INSERT INTO documentUploads (id, clientId, bucket, key, part, createdAt) VALUES ($1, $2, $3, $4, $5, $6);
+INSERT INTO documentUploads (id, clientId, bucket, key, part, createdAt, filename, batch_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8);
+
+-- name: GetDocumentUploadByKey :one
+SELECT id, clientId, bucket, key, part, createdAt, filename, batch_id FROM documentUploads WHERE bucket = $1 AND key = $2 LIMIT 1;
+
+-- name: ListDocumentUploads :many
+SELECT id, clientId, bucket, key, part, createdAt, filename, batch_id FROM documentUploads WHERE clientId = $1 ORDER BY createdAt;
 
 -- name: GetDocumentUploadCurrentPart :one
 WITH client as (

internal/database/repository/document.sql.go

@@ -31,7 +31,7 @@ func (q *Queries) AddDocumentEntry(ctx context.Context, arg *AddDocumentEntryPar
 }
 
 const addDocumentUpload = `-- name: AddDocumentUpload :exec
-INSERT INTO documentUploads (id, clientId, bucket, key, part, createdAt) VALUES ($1, $2, $3, $4, $5, $6)
+INSERT INTO documentUploads (id, clientId, bucket, key, part, createdAt, filename, batch_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
 `
 
 type AddDocumentUploadParams struct {
@@ -41,11 +41,13 @@ type AddDocumentUploadParams struct {
 	Key       string           `db:"key"`
 	Part      uint16           `db:"part"`
 	Createdat pgtype.Timestamp `db:"createdat"`
+	Filename  *string          `db:"filename"`
+	BatchID   *uuid.UUID       `db:"batch_id"`
 }
 
 // AddDocumentUpload
 //
-//	INSERT INTO documentUploads (id, clientId, bucket, key, part, createdAt) VALUES ($1, $2, $3, $4, $5, $6)
+//	INSERT INTO documentUploads (id, clientId, bucket, key, part, createdAt, filename, batch_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
 func (q *Queries) AddDocumentUpload(ctx context.Context, arg *AddDocumentUploadParams) error {
 	_, err := q.db.Exec(ctx, addDocumentUpload,
 		arg.ID,
@@ -54,25 +56,33 @@ func (q *Queries) AddDocumentUpload(ctx context.Context, arg *AddDocumentUploadP
 		arg.Key,
 		arg.Part,
 		arg.Createdat,
+		arg.Filename,
+		arg.BatchID,
 	)
 	return err
 }
 
 const createDocument = `-- name: CreateDocument :one
-INSERT INTO documents (clientId, hash, batch_id) VALUES ($1, $2, $3) RETURNING id
+INSERT INTO documents (clientId, hash, batch_id, filename) VALUES ($1, $2, $3, $4) RETURNING id
 `
 
 type CreateDocumentParams struct {
 	Clientid string     `db:"clientid"`
 	Hash     string     `db:"hash"`
 	BatchID  *uuid.UUID `db:"batch_id"`
+	Filename *string    `db:"filename"`
 }
 
 // CreateDocument
 //
-//	INSERT INTO documents (clientId, hash, batch_id) VALUES ($1, $2, $3) RETURNING id
+//	INSERT INTO documents (clientId, hash, batch_id, filename) VALUES ($1, $2, $3, $4) RETURNING id
 func (q *Queries) CreateDocument(ctx context.Context, arg *CreateDocumentParams) (uuid.UUID, error) {
-	row := q.db.QueryRow(ctx, createDocument, arg.Clientid, arg.Hash, arg.BatchID)
+	row := q.db.QueryRow(ctx, createDocument,
+		arg.Clientid,
+		arg.Hash,
+		arg.BatchID,
+		arg.Filename,
+	)
 	var id uuid.UUID
 	err := row.Scan(&id)
 	return id, err
@@ -209,6 +219,45 @@ func (q *Queries) GetDocumentSummary(ctx context.Context, id uuid.UUID) (*GetDoc
 	return &i, err
 }
 
+const getDocumentUploadByKey = `-- name: GetDocumentUploadByKey :one
+SELECT id, clientId, bucket, key, part, createdAt, filename, batch_id FROM documentUploads WHERE bucket = $1 AND key = $2 LIMIT 1
+`
+
+type GetDocumentUploadByKeyParams struct {
+	Bucket string `db:"bucket"`
+	Key    string `db:"key"`
+}
+
+type GetDocumentUploadByKeyRow struct {
+	ID        uuid.UUID        `db:"id"`
+	Clientid  string           `db:"clientid"`
+	Bucket    string           `db:"bucket"`
+	Key       string           `db:"key"`
+	Part      uint16           `db:"part"`
+	Createdat pgtype.Timestamp `db:"createdat"`
+	Filename  *string          `db:"filename"`
+	BatchID   *uuid.UUID       `db:"batch_id"`
+}
+
+// GetDocumentUploadByKey
+//
+//	SELECT id, clientId, bucket, key, part, createdAt, filename, batch_id FROM documentUploads WHERE bucket = $1 AND key = $2 LIMIT 1
+func (q *Queries) GetDocumentUploadByKey(ctx context.Context, arg *GetDocumentUploadByKeyParams) (*GetDocumentUploadByKeyRow, error) {
+	row := q.db.QueryRow(ctx, getDocumentUploadByKey, arg.Bucket, arg.Key)
+	var i GetDocumentUploadByKeyRow
+	err := row.Scan(
+		&i.ID,
+		&i.Clientid,
+		&i.Bucket,
+		&i.Key,
+		&i.Part,
+		&i.Createdat,
+		&i.Filename,
+		&i.BatchID,
+	)
+	return &i, err
+}
+
 const getDocumentUploadCurrentPart = `-- name: GetDocumentUploadCurrentPart :one
 WITH client as (
     select clientId from clients where clientId = $1
@@ -305,6 +354,86 @@ func (q *Queries) ListDocumentIDsBatch(ctx context.Context, arg *ListDocumentIDs
 	return items, nil
 }
 
+const listDocumentUploads = `-- name: ListDocumentUploads :many
+SELECT id, clientId, bucket, key, part, createdAt, filename, batch_id FROM documentUploads WHERE clientId = $1 ORDER BY createdAt
+`
+
+type ListDocumentUploadsRow struct {
+	ID        uuid.UUID        `db:"id"`
+	Clientid  string           `db:"clientid"`
+	Bucket    string           `db:"bucket"`
+	Key       string           `db:"key"`
+	Part      uint16           `db:"part"`
+	Createdat pgtype.Timestamp `db:"createdat"`
+	Filename  *string          `db:"filename"`
+	BatchID   *uuid.UUID       `db:"batch_id"`
+}
+
+// ListDocumentUploads
+//
+//	SELECT id, clientId, bucket, key, part, createdAt, filename, batch_id FROM documentUploads WHERE clientId = $1 ORDER BY createdAt
+func (q *Queries) ListDocumentUploads(ctx context.Context, clientid string) ([]*ListDocumentUploadsRow, error) {
+	rows, err := q.db.Query(ctx, listDocumentUploads, clientid)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	items := []*ListDocumentUploadsRow{}
+	for rows.Next() {
+		var i ListDocumentUploadsRow
+		if err := rows.Scan(
+			&i.ID,
+			&i.Clientid,
+			&i.Bucket,
+			&i.Key,
+			&i.Part,
+			&i.Createdat,
+			&i.Filename,
+			&i.BatchID,
+		); err != nil {
+			return nil, err
+		}
+		items = append(items, &i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
+const listDocumentsByBatch = `-- name: ListDocumentsByBatch :many
+SELECT id, hash, filename from documents where batch_id = $1
+`
+
+type ListDocumentsByBatchRow struct {
+	ID       uuid.UUID `db:"id"`
+	Hash     string    `db:"hash"`
+	Filename *string   `db:"filename"`
+}
+
+// ListDocumentsByBatch
+//
+//	SELECT id, hash, filename from documents where batch_id = $1
+func (q *Queries) ListDocumentsByBatch(ctx context.Context, batchID *uuid.UUID) ([]*ListDocumentsByBatchRow, error) {
+	rows, err := q.db.Query(ctx, listDocumentsByBatch, batchID)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	items := []*ListDocumentsByBatchRow{}
+	for rows.Next() {
+		var i ListDocumentsByBatchRow
+		if err := rows.Scan(&i.ID, &i.Hash, &i.Filename); err != nil {
+			return nil, err
+		}
+		items = append(items, &i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
 const listDocumentsByClient = `-- name: ListDocumentsByClient :many
 SELECT id, hash from documents where clientId = $1
 `

internal/database/repository/models.go

@@ -369,6 +369,7 @@ type Document struct {
 	Clientid string     `db:"clientid"`
 	Hash     string     `db:"hash"`
 	BatchID  *uuid.UUID `db:"batch_id"`
+	Filename *string    `db:"filename"`
 }
 
 type Documentclean struct {
@@ -417,6 +418,8 @@ type Documentupload struct {
 	Clientid  string           `db:"clientid"`
 	Part      uint16           `db:"part"`
 	Createdat pgtype.Timestamp `db:"createdat"`
+	Filename  *string          `db:"filename"`
+	BatchID   *uuid.UUID       `db:"batch_id"`
 }
 
 type Fullactivecollector struct {

internal/document/init/create.go

@@ -15,9 +15,10 @@ import (
 )
 
 type Create struct {
-	Bucket string
-	Key    objectstore.BucketKey
-	Hash   string
+	Bucket   string
+	Key      objectstore.BucketKey
+	Hash     string
+	Filename *string // Optional filename for batch processing
 }
 
 func (s *Service) Create(ctx context.Context, doc *Create) (uuid.UUID, error) {
@@ -47,10 +48,11 @@ func (s *Service) Create(ctx context.Context, doc *Create) (uuid.UUID, error) {
 }
 
 type createDocumentParams struct {
-	ID     *uuid.UUID
-	Hash   string
-	Bucket string
-	Key    objectstore.BucketKey
+	ID       *uuid.UUID
+	Hash     string
+	Bucket   string
+	Key      objectstore.BucketKey
+	Filename *string
 }
 
 func (s *Service) getCreateParams(ctx context.Context, doc *Create) (*createDocumentParams, error) {
@@ -66,10 +68,11 @@ func (s *Service) getCreateParams(ctx context.Context, doc *Create) (*createDocu
 	}
 
 	return &createDocumentParams{
-		ID:     docID,
-		Hash:   doc.Hash,
-		Key:    doc.Key,
-		Bucket: doc.Bucket,
+		ID:       docID,
+		Hash:     doc.Hash,
+		Key:      doc.Key,
+		Bucket:   doc.Bucket,
+		Filename: doc.Filename,
 	}, nil
 }
 
@@ -83,6 +86,7 @@ func (s *Service) submitCreate(ctx context.Context, params *createDocumentParams
 				Clientid: params.Key.ClientID,
 				Hash:     params.Hash,
 				BatchID:  params.Key.BatchID,
+				Filename: params.Filename,
 			})
 			if err != nil {
 				return err

internal/document/init/create_test.go

@@ -52,7 +52,7 @@ func TestCreate(t *testing.T) {
 		pgxmock.NewRows([]string{"id"}),
 	)
 	pool.ExpectBegin()
-	pool.ExpectQuery("name: CreateDocument :one").WithArgs(doc.ClientID, doc.Hash, (*uuid.UUID)(nil)).
+	pool.ExpectQuery("name: CreateDocument :one").WithArgs(doc.ClientID, doc.Hash, (*uuid.UUID)(nil), (*string)(nil)).
 		WillReturnRows(
 			pgxmock.NewRows([]string{"id"}).
 				AddRow(doc.ID),
@@ -80,14 +80,14 @@ func TestCreate(t *testing.T) {
 			mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
 				return *in.QueueUrl == cfg.DocumentSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", doc.ID.String())
 			}),
-			mock.Anything,
 		).
 		Return(&sqs.SendMessageOutput{}, nil)
 
 	id, err := svc.Create(ctx, &Create{
-		Hash:   doc.Hash,
-		Key:    key,
-		Bucket: bucket,
+		Hash:     doc.Hash,
+		Key:      key,
+		Bucket:   bucket,
+		Filename: nil, // No filename for this test
 	})
 	require.NoError(t, err)
 	assert.Equal(t, doc.ID, id)
@@ -235,7 +235,7 @@ func TestSubmitCreate(t *testing.T) {
 	}
 
 	pool.ExpectBegin()
-	pool.ExpectQuery("name: CreateDocument :one").WithArgs(doc.ClientID, doc.Hash, (*uuid.UUID)(nil)).
+	pool.ExpectQuery("name: CreateDocument :one").WithArgs(doc.ClientID, doc.Hash, (*uuid.UUID)(nil), (*string)(nil)).
 		WillReturnRows(
 			pgxmock.NewRows([]string{"id"}).
 				AddRow(doc.ID),
@@ -245,9 +245,10 @@ func TestSubmitCreate(t *testing.T) {
 	pool.ExpectCommit()
 
 	id, err := svc.submitCreate(ctx, &createDocumentParams{
-		Hash:   doc.Hash,
-		Bucket: bucket,
-		Key:    key,
+		Hash:     doc.Hash,
+		Bucket:   bucket,
+		Key:      key,
+		Filename: nil, // No filename for this test
 	})
 	require.NoError(t, err)
 	assert.Equal(t, doc.ID, id)

internal/document/upload/get.go

@@ -34,7 +34,14 @@ func detectContentType(filename string) string {
 }
 
 func (s *Service) Upload(ctx context.Context, file File) error {
-	return s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, q *repository.Queries) error {
+	_, err := s.UploadAndReturnKey(ctx, file)
+	return err
+}
+
+// UploadAndReturnKey uploads a file and returns the BucketKey used
+func (s *Service) UploadAndReturnKey(ctx context.Context, file File) (objectstore.BucketKey, error) {
+	var key objectstore.BucketKey
+	err := s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, q *repository.Queries) error {
 		now := time.Now().UTC()
 		part, err := s.cfg.GetDBQueries().GetDocumentUploadCurrentPart(ctx, &repository.GetDocumentUploadCurrentPartParams{
 			Clientid: &file.ClientID,
@@ -49,7 +56,7 @@ func (s *Service) Upload(ctx context.Context, file File) error {
 
 		uploadId := uuid.New()
 		newPart := s.cfg.GetDirectoryPart(part.Part, part.Count)
-		key := objectstore.BucketKey{
+		key = objectstore.BucketKey{
 			ClientID:  file.ClientID,
 			Location:  objectstore.Import,
 			CreatedAt: now,
@@ -70,6 +77,8 @@ func (s *Service) Upload(ctx context.Context, file File) error {
 				Valid: true,
 				Time:  now,
 			},
+			Filename: &file.Filename,
+			BatchID:  file.BatchID,
 		})
 		if err != nil {
 			return err
@@ -78,11 +87,18 @@ func (s *Service) Upload(ctx context.Context, file File) error {
 		// Detect content type from original filename
 		contentType := detectContentType(file.Filename)
 
+		// Prepare metadata for S3 object
+		metadata := make(map[string]string)
+		if file.Filename != "" {
+			metadata["original-filename"] = file.Filename
+		}
+
 		_, err = s.cfg.GetStoreClient().PutObject(ctx, &s3.PutObjectInput{
 			Bucket:      &bucket,
 			Key:         &keyStr,
 			Body:        file.Content,
 			ContentType: &contentType,
+			Metadata:    metadata,
 		})
 		if err != nil {
 			return err
@@ -90,4 +106,6 @@ func (s *Service) Upload(ctx context.Context, file File) error {
 
 		return nil
 	})
+
+	return key, err
 }

internal/query/result/process_test.go

@@ -31,6 +31,8 @@ func TestProcess(t *testing.T) {
 	docId, err := cfg.GetDBQueries().CreateDocument(t.Context(), &repository.CreateDocumentParams{
 		Clientid: "client_id",
 		Hash:     "hash",
+		BatchID:  nil,
+		Filename: nil,
 	})
 	require.NoError(t, err)
 	fill := "fill"

internal/server/api/batch_worker.go

@@ -25,6 +25,43 @@ const (
 	ConfigKeyUploadHandler = "uploadHandler"
 )
 
+// sanitizeZipPath cleans a path from a zip file to prevent path traversal attacks
+// while preserving the internal folder structure.
+// Returns empty string if the path is invalid or attempts traversal.
+func sanitizeZipPath(p string) string {
+	// Reject paths with backslashes (Windows path traversal)
+	if strings.Contains(p, "\\") {
+		return ""
+	}
+
+	// Clean the path to resolve . and .. elements
+	cleaned := path.Clean(p)
+
+	// Reject paths that try to escape (start with .. or /)
+	if strings.HasPrefix(cleaned, "../") || strings.HasPrefix(cleaned, "/") || cleaned == ".." {
+		return ""
+	}
+
+	// Check for net upward traversal
+	// We need to ensure that we don't escape the root level
+	parts := strings.Split(p, "/")
+	depth := 0
+	for _, part := range parts {
+		if part == ".." {
+			depth--
+			if depth < 0 {
+				// Trying to escape root
+				return ""
+			}
+		} else if part != "." && part != "" {
+			depth++
+		}
+	}
+
+	// The cleaned path is safe and preserves internal structure
+	return cleaned
+}
+
 // processBatchWork is the background worker function that processes batch uploads
 func processBatchWork(ctx context.Context, logger *slog.Logger, config map[string]any) error {
 	// Extract the batch service from config
@@ -232,14 +269,23 @@ func processZipFile(
 		return 0, 0, 0
 	}
 
-	// Skip system files and sanitize filename
+	// Skip system files and hidden files (but allow relative path prefixes like ./file.pdf)
 	filename := file.Name
-	if strings.HasPrefix(filename, "__MACOSX/") || strings.HasPrefix(filename, ".") {
+	if strings.HasPrefix(filename, "__MACOSX/") {
 		return 0, 0, 0
 	}
 
-	// Security: Clean filename to prevent path traversal attacks
-	filename = path.Base(filename)
+	// Skip hidden files (files that start with . but are not relative paths like ./file)
+	if strings.HasPrefix(filename, ".") && !strings.HasPrefix(filename, "./") {
+		return 0, 0, 0
+	}
+
+	// Security: Clean filename to prevent path traversal attacks while preserving internal structure
+	filename = sanitizeZipPath(filename)
+	if filename == "" {
+		logger.Warn("Skipping file with invalid path", "original", file.Name)
+		return 0, 0, 0
+	}
 
 	// Extract file content
 	fileReader, err := file.Open()
@@ -257,7 +303,7 @@ func processZipFile(
 		return 0, 1, 0
 	}
 
-	// Store extracted file in S3 temporary directory (filename is already sanitized with path.Base)
+	// Store extracted file in S3 temporary directory (filename now includes sanitized path)
 	extractedKey := extractPath + "/" + filename
 	if err := uploadToS3(ctx, s3Client, bucket, extractedKey, fileData); err != nil {
 		logger.Error("Failed to upload extracted file to S3", "filename", filename, "error", err)

internal/server/api/batch_worker_test.go

@@ -23,12 +23,6 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-// TestConfig embeds objectstore for S3 operations
-type TestConfig struct {
-	serviceconfig.BaseConfig
-	objectstore.ObjectStoreConfig
-}
-
 // TestProcessBatchWork tests the main batch processing work function
 func TestProcessBatchWork(t *testing.T) {
 	if testing.Short() {
@@ -36,7 +30,7 @@ func TestProcessBatchWork(t *testing.T) {
 	}
 
 	ctx := t.Context()
-	cfg := &TestConfig{}
+	cfg := &BaseConfig{}
 	_ = serviceconfig.InitializeConfig(cfg)
 	test.CreateDB(t, cfg)
 	test.CreateAWSResources(t, cfg)
@@ -115,7 +109,7 @@ func TestProcessSingleBatch(t *testing.T) {
 	}
 
 	ctx := t.Context()
-	cfg := &TestConfig{}
+	cfg := &BaseConfig{}
 	_ = serviceconfig.InitializeConfig(cfg)
 	test.CreateDB(t, cfg)
 	test.CreateAWSResources(t, cfg)
@@ -191,7 +185,7 @@ func TestProcessSingleBatchWithFailure(t *testing.T) {
 	}
 
 	ctx := t.Context()
-	cfg := &TestConfig{}
+	cfg := &BaseConfig{}
 	_ = serviceconfig.InitializeConfig(cfg)
 	test.CreateDB(t, cfg)
 	test.CreateAWSResources(t, cfg)
@@ -269,7 +263,7 @@ func TestDownloadZipFromS3(t *testing.T) {
 	}
 
 	ctx := t.Context()
-	cfg := &TestConfig{}
+	cfg := &BaseConfig{}
 	_ = serviceconfig.InitializeConfig(cfg)
 	test.CreateAWSResources(t, cfg)
 
@@ -302,7 +296,7 @@ func TestUploadToS3(t *testing.T) {
 	}
 
 	ctx := t.Context()
-	cfg := &TestConfig{}
+	cfg := &BaseConfig{}
 	_ = serviceconfig.InitializeConfig(cfg)
 	test.CreateAWSResources(t, cfg)
 
@@ -380,3 +374,422 @@ func createMixedContentZIP(t *testing.T) []byte {
 
 	return buf.Bytes()
 }
+
+// Helper function to create a ZIP with nested folder structure
+func createNestedFolderZIP(t *testing.T) []byte {
+	var buf bytes.Buffer
+	zipWriter := zip.NewWriter(&buf)
+
+	// Create files in various nested folder structures
+	testFiles := []struct {
+		path    string
+		content string
+	}{
+		{"document1.pdf", "%%PDF-1.4\n%%Root level PDF\n%%%%EOF"},
+		{"foldera/file1.pdf", "%%PDF-1.4\n%%FolderA File1\n%%%%EOF"},
+		{"foldera/file2.pdf", "%%PDF-1.4\n%%FolderA File2\n%%%%EOF"},
+		{"folderb/folderc/file3.pdf", "%%PDF-1.4\n%%Nested File3\n%%%%EOF"},
+		{"folderb/folderc/file4.pdf", "%%PDF-1.4\n%%Nested File4\n%%%%EOF"},
+		{"folderb/file5.pdf", "%%PDF-1.4\n%%FolderB File5\n%%%%EOF"},
+		{"deep/nested/structure/test/file6.pdf", "%%PDF-1.4\n%%Deep nested\n%%%%EOF"},
+		// Add a non-PDF to verify it's skipped
+		{"foldera/readme.txt", "This should be skipped"},
+	}
+
+	for _, tf := range testFiles {
+		fileWriter, err := zipWriter.Create(tf.path)
+		require.NoError(t, err)
+		_, err = fileWriter.Write([]byte(tf.content))
+		require.NoError(t, err)
+	}
+
+	err := zipWriter.Close()
+	require.NoError(t, err)
+
+	return buf.Bytes()
+}
+
+// Helper function to create a ZIP with path traversal attempts
+func createMaliciousPathZIP(t *testing.T) []byte {
+	var buf bytes.Buffer
+	zipWriter := zip.NewWriter(&buf)
+
+	// Create files with various path traversal attempts
+	testFiles := []struct {
+		path    string
+		content string
+	}{
+		{"../escape.pdf", "%%PDF-1.4\n%%Escape attempt\n%%%%EOF"},
+		{"../../etc/passwd", "malicious content"},
+		{"./valid.pdf", "%%PDF-1.4\n%%Valid PDF\n%%%%EOF"},
+		{"folder/../sibling.pdf", "%%PDF-1.4\n%%Sibling\n%%%%EOF"},
+		{"normal/file.pdf", "%%PDF-1.4\n%%Normal\n%%%%EOF"},
+	}
+
+	for _, tf := range testFiles {
+		// Note: zip.Writer doesn't validate paths, so we can create these entries
+		header := &zip.FileHeader{
+			Name:   tf.path,
+			Method: zip.Deflate,
+		}
+		fileWriter, err := zipWriter.CreateHeader(header)
+		require.NoError(t, err)
+		_, err = fileWriter.Write([]byte(tf.content))
+		require.NoError(t, err)
+	}
+
+	err := zipWriter.Close()
+	require.NoError(t, err)
+
+	return buf.Bytes()
+}
+
+// TestSanitizeZipPath tests the path sanitization function
+func TestSanitizeZipPath(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{"simple file", "document.pdf", "document.pdf"},
+		{"file in folder", "folder/document.pdf", "folder/document.pdf"},
+		{"nested folders", "folder/subfolder/document.pdf", "folder/subfolder/document.pdf"},
+		{"deep nesting", "a/b/c/d/e/f/document.pdf", "a/b/c/d/e/f/document.pdf"},
+		{"dot prefix cleaned", "./document.pdf", "document.pdf"},
+		{"dot in path cleaned", "folder/./document.pdf", "folder/document.pdf"},
+		{"parent traversal rejected", "../document.pdf", ""},
+		{"parent in middle cleaned", "folder/../document.pdf", "document.pdf"},
+		{"absolute path rejected", "/etc/passwd", ""},
+		{"backslash rejected", "folder\\document.pdf", ""},
+		{"double parent rejected", "../../document.pdf", ""},
+		{"complex traversal attempt", "folder/../../etc/passwd", ""},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := sanitizeZipPath(tt.input)
+			assert.Equal(t, tt.expected, result, "sanitizeZipPath(%q) = %q, want %q", tt.input, result, tt.expected)
+		})
+	}
+}
+
+// TestProcessBatchWithNestedFolders tests processing a ZIP with nested folder structure
+func TestProcessBatchWithNestedFolders(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+	}
+
+	ctx := t.Context()
+	cfg := &BaseConfig{}
+	_ = serviceconfig.InitializeConfig(cfg)
+	test.CreateDB(t, cfg)
+	test.CreateAWSResources(t, cfg)
+
+	// Create test client
+	clientID := "test_nested_folders"
+	err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
+		Clientid: clientID,
+		Name:     "Test Nested Folders Client",
+	})
+	require.NoError(t, err)
+
+	// Create services
+	batchService := batch.New(cfg)
+	s3ClientInterface := cfg.GetStoreClient()
+	s3Client, ok := s3ClientInterface.(*s3.Client)
+	require.True(t, ok, "s3Client should be *s3.Client")
+	bucket := cfg.GetBucket()
+
+	// Upload test ZIP with nested folders
+	zipContent := createNestedFolderZIP(t)
+	archiveKey := fmt.Sprintf("test/%s/nested.zip", clientID)
+
+	_, err = s3Client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(archiveKey),
+		Body:   bytes.NewReader(zipContent),
+	})
+	require.NoError(t, err)
+
+	// Create batch
+	batchID, err := batchService.CreateWithStorage(ctx, clientID, "nested.zip", 7, bucket, archiveKey, int64(len(zipContent)))
+	require.NoError(t, err)
+
+	// Use the real upload handler that creates documents in the database
+	uploadHandler := createDocumentUploadHandler(cfg)
+
+	// Create batch details
+	batchDetails := &batch.BatchUploadDetails{
+		BatchUploadSummary: batch.BatchUploadSummary{
+			ID:               batchID,
+			ClientID:         clientID,
+			OriginalFilename: "nested.zip",
+			Status:           "processing",
+		},
+		ArchiveKey:    archiveKey,
+		ArchiveBucket: bucket,
+	}
+
+	// Process the batch
+	logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
+	err = processSingleBatch(ctx, logger, batchService, s3Client, bucket, uploadHandler, batchDetails)
+	require.NoError(t, err)
+
+	// Verify batch status
+	batchInfo, err := batchService.Get(ctx, clientID, batchID)
+	require.NoError(t, err)
+	assert.Equal(t, int32(7), batchInfo.ProcessedDocuments)   // 7 PDFs
+	assert.Equal(t, int32(1), batchInfo.InvalidTypeDocuments) // 1 txt file
+	assert.Equal(t, "completed", batchInfo.Status)
+
+	// Manually trigger document initialization for uploaded files since S3 notifications
+	// don't work in test environment. Get all document uploads for this batch.
+	uploads, err := cfg.GetDBQueries().ListDocumentUploads(ctx, clientID)
+	require.NoError(t, err)
+
+	// Process each uploaded file through the document initialization pipeline
+	for _, upload := range uploads {
+		if upload.BatchID != nil && *upload.BatchID == batchID {
+			// Parse the bucket key
+			key, err := objectstore.ParseBucketKey(upload.Key)
+			require.NoError(t, err)
+
+			// Get file hash from S3 object metadata (normally provided by S3 event)
+			headResult, err := s3Client.HeadObject(ctx, &s3.HeadObjectInput{
+				Bucket: aws.String(upload.Bucket),
+				Key:    aws.String(upload.Key),
+			})
+			require.NoError(t, err)
+
+			// Create document directly in database since full pipeline requires additional setup
+			_, err = cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
+				Clientid: key.ClientID,
+				Hash:     *headResult.ETag, // Use ETag as hash for testing
+				BatchID:  upload.BatchID,
+				Filename: upload.Filename,
+			})
+			require.NoError(t, err)
+		}
+	}
+
+	// Get documents for this specific batch from database
+	batchDocs, err := cfg.GetDBQueries().ListDocumentsByBatch(ctx, &batchID)
+	require.NoError(t, err)
+
+	// Verify we have the expected number of documents
+	assert.Equal(t, 7, len(batchDocs), "Should have exactly 7 documents in database")
+
+	// Verify filenames are preserved with their paths
+	foundFilenames := make(map[string]bool)
+	for _, doc := range batchDocs {
+		if doc.Filename != nil {
+			foundFilenames[*doc.Filename] = true
+		}
+	}
+
+	// Check that we found all expected filenames with their paths
+	expectedFilenames := []string{
+		"document1.pdf",
+		"foldera/file1.pdf",
+		"foldera/file2.pdf",
+		"folderb/folderc/file3.pdf",
+		"folderb/folderc/file4.pdf",
+		"folderb/file5.pdf",
+		"deep/nested/structure/test/file6.pdf",
+	}
+
+	for _, expectedFilename := range expectedFilenames {
+		assert.True(t, foundFilenames[expectedFilename],
+			"Should have preserved filename with path: %s", expectedFilename)
+	}
+}
+
+// TestProcessBatchWithPathTraversal tests that path traversal attempts are handled safely
+func TestProcessBatchWithPathTraversal(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+	}
+
+	ctx := t.Context()
+	cfg := &BaseConfig{}
+	_ = serviceconfig.InitializeConfig(cfg)
+	test.CreateDB(t, cfg)
+	test.CreateAWSResources(t, cfg)
+
+	// Create test client
+	clientID := "test_path_traversal"
+	err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
+		Clientid: clientID,
+		Name:     "Test Path Traversal Client",
+	})
+	require.NoError(t, err)
+
+	// Create services
+	batchService := batch.New(cfg)
+	s3ClientInterface := cfg.GetStoreClient()
+	s3Client, ok := s3ClientInterface.(*s3.Client)
+	require.True(t, ok, "s3Client should be *s3.Client")
+	bucket := cfg.GetBucket()
+
+	// Upload test ZIP with path traversal attempts
+	zipContent := createMaliciousPathZIP(t)
+	archiveKey := fmt.Sprintf("test/%s/malicious.zip", clientID)
+
+	_, err = s3Client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(archiveKey),
+		Body:   bytes.NewReader(zipContent),
+	})
+	require.NoError(t, err)
+
+	// Create batch
+	batchID, err := batchService.CreateWithStorage(ctx, clientID, "malicious.zip", 5, bucket, archiveKey, int64(len(zipContent)))
+	require.NoError(t, err)
+
+	// Track uploaded documents
+	uploadedDocs := make(map[string][]byte)
+	uploadHandler := func(ctx context.Context, clientID string, docData io.Reader, filename string, batchID *uuid.UUID) error {
+		data, err := io.ReadAll(docData)
+		if err != nil {
+			return err
+		}
+		uploadedDocs[filename] = data
+		return nil
+	}
+
+	// Create batch details
+	batchDetails := &batch.BatchUploadDetails{
+		BatchUploadSummary: batch.BatchUploadSummary{
+			ID:               batchID,
+			ClientID:         clientID,
+			OriginalFilename: "malicious.zip",
+			Status:           "processing",
+		},
+		ArchiveKey:    archiveKey,
+		ArchiveBucket: bucket,
+	}
+
+	// Process the batch
+	logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
+	err = processSingleBatch(ctx, logger, batchService, s3Client, bucket, uploadHandler, batchDetails)
+	require.NoError(t, err)
+
+	// Verify that path traversal attempts were blocked
+	assert.NotContains(t, uploadedDocs, "../escape.pdf", "Should reject parent traversal")
+	assert.NotContains(t, uploadedDocs, "../../etc/passwd", "Should reject multiple parent traversal")
+	assert.NotContains(t, uploadedDocs, "/etc/passwd", "Should reject absolute paths")
+
+	// Verify that safe files were processed correctly
+	assert.Contains(t, uploadedDocs, "valid.pdf", "Should process ./valid.pdf as valid.pdf")
+	assert.Contains(t, uploadedDocs, "sibling.pdf", "Should clean folder/../sibling.pdf to sibling.pdf")
+	assert.Contains(t, uploadedDocs, "normal/file.pdf", "Should process normal paths correctly")
+
+	// Verify only safe PDFs were processed
+	assert.Equal(t, 3, len(uploadedDocs), "Should have processed exactly 3 safe PDFs")
+}
+
+// TestProcessBatchWithDuplicateFilenames tests handling of duplicate filenames in different folders
+func TestProcessBatchWithDuplicateFilenames(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+	}
+
+	ctx := t.Context()
+	cfg := &BaseConfig{}
+	_ = serviceconfig.InitializeConfig(cfg)
+	test.CreateDB(t, cfg)
+	test.CreateAWSResources(t, cfg)
+
+	// Create test client
+	clientID := "test_duplicate_names"
+	err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
+		Clientid: clientID,
+		Name:     "Test Duplicate Names Client",
+	})
+	require.NoError(t, err)
+
+	// Create services
+	batchService := batch.New(cfg)
+	s3ClientInterface := cfg.GetStoreClient()
+	s3Client, ok := s3ClientInterface.(*s3.Client)
+	require.True(t, ok, "s3Client should be *s3.Client")
+	bucket := cfg.GetBucket()
+
+	// Create ZIP with duplicate filenames in different folders
+	var buf bytes.Buffer
+	zipWriter := zip.NewWriter(&buf)
+
+	// Create files with same names in different folders
+	testFiles := []struct {
+		path    string
+		content string
+	}{
+		{"document.pdf", "%%PDF-1.4\n%%Root document\n%%%%EOF"},
+		{"folder1/document.pdf", "%%PDF-1.4\n%%Folder1 document\n%%%%EOF"},
+		{"folder2/document.pdf", "%%PDF-1.4\n%%Folder2 document\n%%%%EOF"},
+		{"folder1/subfolder/document.pdf", "%%PDF-1.4\n%%Subfolder document\n%%%%EOF"},
+	}
+
+	for _, tf := range testFiles {
+		fileWriter, err := zipWriter.Create(tf.path)
+		require.NoError(t, err)
+		_, err = fileWriter.Write([]byte(tf.content))
+		require.NoError(t, err)
+	}
+
+	err = zipWriter.Close()
+	require.NoError(t, err)
+	zipContent := buf.Bytes()
+
+	// Upload the ZIP
+	archiveKey := fmt.Sprintf("test/%s/duplicates.zip", clientID)
+	_, err = s3Client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(archiveKey),
+		Body:   bytes.NewReader(zipContent),
+	})
+	require.NoError(t, err)
+
+	// Create batch
+	batchID, err := batchService.CreateWithStorage(ctx, clientID, "duplicates.zip", 4, bucket, archiveKey, int64(len(zipContent)))
+	require.NoError(t, err)
+
+	// Track uploaded documents
+	uploadedDocs := make(map[string]string)
+	uploadHandler := func(ctx context.Context, clientID string, docData io.Reader, filename string, batchID *uuid.UUID) error {
+		data, err := io.ReadAll(docData)
+		if err != nil {
+			return err
+		}
+		// Extract the content identifier from the PDF
+		content := string(data)
+		uploadedDocs[filename] = content
+		return nil
+	}
+
+	// Create batch details
+	batchDetails := &batch.BatchUploadDetails{
+		BatchUploadSummary: batch.BatchUploadSummary{
+			ID:               batchID,
+			ClientID:         clientID,
+			OriginalFilename: "duplicates.zip",
+			Status:           "processing",
+		},
+		ArchiveKey:    archiveKey,
+		ArchiveBucket: bucket,
+	}
+
+	// Process the batch
+	logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
+	err = processSingleBatch(ctx, logger, batchService, s3Client, bucket, uploadHandler, batchDetails)
+	require.NoError(t, err)
+
+	// Verify all files were uploaded with correct paths
+	assert.Equal(t, 4, len(uploadedDocs), "Should have uploaded all 4 PDFs")
+
+	// Verify each file has unique path and correct content
+	assert.Contains(t, uploadedDocs["document.pdf"], "Root document")
+	assert.Contains(t, uploadedDocs["folder1/document.pdf"], "Folder1 document")
+	assert.Contains(t, uploadedDocs["folder2/document.pdf"], "Folder2 document")
+	assert.Contains(t, uploadedDocs["folder1/subfolder/document.pdf"], "Subfolder document")
+}

internal/server/api/listener.go

@@ -327,9 +327,10 @@ func New(ctx context.Context, cfg Config) (*Server, error) {
 }
 
 // createDocumentUploadHandler creates a function that can upload documents
-// directly through the service layer, bypassing HTTP
+// directly through the service layer, bypassing HTTP. For batch uploads,
+// it creates the document record immediately with filename information.
 func createDocumentUploadHandler(cfg Config) func(ctx context.Context, clientID string, docData io.Reader, filename string, batchID *uuid.UUID) error {
-	// Import the document upload service
+	// Import the upload service
 	uploadService := documentupload.New(cfg)
 
 	return func(ctx context.Context, clientID string, docData io.Reader, filename string, batchID *uuid.UUID) error {
@@ -341,7 +342,8 @@ func createDocumentUploadHandler(cfg Config) func(ctx context.Context, clientID
 			Filename: filename,
 		}
 
-		// Call the upload service directly
+		// Use the unified upload path for both batch and single uploads
+		// The filename will be stored in documentUploads table and used by the async pipeline
 		return uploadService.Upload(ctx, file)
 	}
 }

scripts/manual_tests/test_zip_upload.sh

@@ -65,9 +65,9 @@ check_service() {
     log_info "Service is running"
 }
 
-# Create test ZIP file with real PDFs from test.pdf.batch directory
+# Create test ZIP file with real PDFs from test.pdf.batch directory, each in its own subfolder
 create_test_zip() {
-    log_info "Creating test ZIP file with real PDFs from ./test.pdf.batch..."
+    log_info "Creating test ZIP file with real PDFs from ./test.pdf.batch, each in its own subfolder..."
 
     # Remove existing file if it exists
     rm -f "$ZIP_FILE"
@@ -86,14 +86,34 @@ create_test_zip() {
         exit 1
     fi
 
-    # Create ZIP file with all PDFs from the batch directory
+    # Create temporary directory structure
+    temp_dir="temp_zip_structure"
+    rm -rf "$temp_dir"
+    mkdir -p "$temp_dir"
+
+    # Copy each PDF to its own subfolder
+    folder_num=1
+    for pdf_file in "$pdf_dir"/*.pdf; do
+        if [ -f "$pdf_file" ]; then
+            folder_name="folder$folder_num"
+            mkdir -p "$temp_dir/$folder_name"
+            cp "$pdf_file" "$temp_dir/$folder_name/"
+            log_info "Placed $(basename "$pdf_file") in $folder_name/"
+            folder_num=$((folder_num + 1))
+        fi
+    done
+
+    # Create ZIP file with the folder structure
     current_dir=$(pwd)
-    cd "$pdf_dir"
-    zip -q "$current_dir/$ZIP_FILE" *.pdf
+    cd "$temp_dir"
+    zip -q -r "$current_dir/$ZIP_FILE" .
     cd "$current_dir"
 
+    # Cleanup temporary directory
+    rm -rf "$temp_dir"
+
     file_size=$(stat -f%z "$ZIP_FILE" 2>/dev/null || stat -c%s "$ZIP_FILE" 2>/dev/null || echo "unknown")
-    log_info "Created $ZIP_FILE with $pdf_count real PDFs ($file_size bytes)"
+    log_info "Created $ZIP_FILE with $pdf_count real PDFs in separate subfolders ($file_size bytes)"
 }
 
 # Create client (idempotent)
@@ -261,6 +281,7 @@ cancel_batch() {
 cleanup() {
     log_info "Cleaning up..."
     rm -f "$ZIP_FILE" invalid.txt
+    rm -rf temp_zip_structure
     log_info "Cleanup complete"
 }
 

scripts/manual_tests/upload_single_document.sh

@@ -30,14 +30,38 @@ fi
 
 echo -e "${GREEN}API is running${NC}"
 
-# Verify client exists by trying to get its details
+# Function to create client
+create_client() {
+    echo "Creating client with ID '$CLIENT_ID'..."
+    CREATE_RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "$API_URL/client" \
+        -H "Content-Type: application/json" \
+        -d "{\"id\":\"$CLIENT_ID\",\"name\":\"Test Client $CLIENT_ID\"}")
+    
+    CREATE_HTTP_CODE=$(echo "$CREATE_RESPONSE" | tail -n1)
+    CREATE_RESPONSE_BODY=$(echo "$CREATE_RESPONSE" | sed '$d')
+    
+    if [ "$CREATE_HTTP_CODE" = "201" ] || [ "$CREATE_HTTP_CODE" = "200" ]; then
+        echo -e "${GREEN}Client '$CLIENT_ID' created successfully${NC}"
+        return 0
+    elif [ "$CREATE_HTTP_CODE" = "409" ] || [ "$CREATE_HTTP_CODE" = "400" ]; then
+        echo -e "${GREEN}Client '$CLIENT_ID' already exists${NC}"
+        return 0
+    else
+        echo -e "${RED}Error: Failed to create client (HTTP $CREATE_HTTP_CODE)${NC}"
+        echo "Response: $CREATE_RESPONSE_BODY"
+        return 1
+    fi
+}
+
+# Verify client exists or create it
 echo "Verifying client with ID '$CLIENT_ID' exists..."
 CLIENT_CHECK=$(curl -s -o /dev/null -w "%{http_code}" "$API_URL/client/$CLIENT_ID")
 
 if [ "$CLIENT_CHECK" != "200" ]; then
-    echo -e "${RED}Error: Client with ID '$CLIENT_ID' not found${NC}"
-    echo "Please ensure the client exists in the database"
-    exit 1
+    echo "Client '$CLIENT_ID' not found, attempting to create it..."
+    if ! create_client; then
+        exit 1
+    fi
 else
     echo -e "${GREEN}Client '$CLIENT_ID' verified${NC}"
 fi