Merged in feature/textExtractionsPart1 (pull request #192)

all schema and rest apis for text extraction support

* in progress

* stage 8 complete

* phase 9 completed

* phase 9 complete

* ongoing - s3 path fix

* working

* optimize ci build

* e2e tests

* missing test

internal/document/init/create.go

@@ -15,10 +15,12 @@ import (
 )
 
 type Create struct {
-	Bucket   string
-	Key      objectstore.BucketKey
-	Hash     string
-	Filename *string // Optional filename for batch processing
+	Bucket       string
+	Key          objectstore.BucketKey
+	Hash         string
+	Filename     *string    // Optional filename for batch processing
+	OriginalPath *string    // Original path provided during upload (immutable after creation)
+	FolderID     *uuid.UUID // Optional folder ID if folder hierarchy was pre-created
 }
 
 func (s *Service) Create(ctx context.Context, doc *Create) (uuid.UUID, error) {
@@ -48,11 +50,13 @@ func (s *Service) Create(ctx context.Context, doc *Create) (uuid.UUID, error) {
 }
 
 type createDocumentParams struct {
-	ID       *uuid.UUID
-	Hash     string
-	Bucket   string
-	Key      objectstore.BucketKey
-	Filename *string
+	ID           *uuid.UUID
+	Hash         string
+	Bucket       string
+	Key          objectstore.BucketKey
+	Filename     *string
+	OriginalPath *string
+	FolderID     *uuid.UUID
 }
 
 func (s *Service) getCreateParams(ctx context.Context, doc *Create) (*createDocumentParams, error) {
@@ -68,11 +72,13 @@ func (s *Service) getCreateParams(ctx context.Context, doc *Create) (*createDocu
 	}
 
 	return &createDocumentParams{
-		ID:       docID,
-		Hash:     doc.Hash,
-		Key:      doc.Key,
-		Bucket:   doc.Bucket,
-		Filename: doc.Filename,
+		ID:           docID,
+		Hash:         doc.Hash,
+		Key:          doc.Key,
+		Bucket:       doc.Bucket,
+		Filename:     doc.Filename,
+		OriginalPath: doc.OriginalPath,
+		FolderID:     doc.FolderID,
 	}, nil
 }
 
@@ -83,10 +89,12 @@ func (s *Service) submitCreate(ctx context.Context, params *createDocumentParams
 		var dbid uuid.UUID
 		if params.ID == nil {
 			createid, err := s.cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
-				Clientid: params.Key.ClientID,
-				Hash:     params.Hash,
-				BatchID:  params.Key.BatchID,
-				Filename: params.Filename,
+				Clientid:     params.Key.ClientID,
+				Hash:         params.Hash,
+				BatchID:      params.Key.BatchID,
+				Filename:     params.Filename,
+				Folderid:     params.FolderID,
+				Originalpath: params.OriginalPath,
 			})
 			if err != nil {
 				return err

internal/document/init/create_test.go

@@ -52,7 +52,7 @@ func TestCreate(t *testing.T) {
 		pgxmock.NewRows([]string{"id"}),
 	)
 	pool.ExpectBegin()
-	pool.ExpectQuery("name: CreateDocument :one").WithArgs(doc.ClientID, doc.Hash, (*uuid.UUID)(nil), (*string)(nil)).
+	pool.ExpectQuery("name: CreateDocument :one").WithArgs(doc.ClientID, doc.Hash, (*uuid.UUID)(nil), (*string)(nil), (*uuid.UUID)(nil), (*string)(nil)).
 		WillReturnRows(
 			pgxmock.NewRows([]string{"id"}).
 				AddRow(doc.ID),
@@ -80,6 +80,7 @@ func TestCreate(t *testing.T) {
 			mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
 				return *in.QueueUrl == cfg.DocumentSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", doc.ID.String())
 			}),
+			mock.Anything,
 		).
 		Return(&sqs.SendMessageOutput{}, nil)
 
@@ -235,7 +236,7 @@ func TestSubmitCreate(t *testing.T) {
 	}
 
 	pool.ExpectBegin()
-	pool.ExpectQuery("name: CreateDocument :one").WithArgs(doc.ClientID, doc.Hash, (*uuid.UUID)(nil), (*string)(nil)).
+	pool.ExpectQuery("name: CreateDocument :one").WithArgs(doc.ClientID, doc.Hash, (*uuid.UUID)(nil), (*string)(nil), (*uuid.UUID)(nil), (*string)(nil)).
 		WillReturnRows(
 			pgxmock.NewRows([]string{"id"}).
 				AddRow(doc.ID),

internal/document/upload/get.go

@@ -2,9 +2,11 @@ package documentupload
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"mime"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"queryorchestration/internal/database/repository"
@@ -19,9 +21,20 @@ type File struct {
 	ClientID string
 	Content  io.Reader
 	BatchID  *uuid.UUID // Optional batch ID for batch processing
-	Filename string     // Original filename for MIME type detection
+	Filename string     // Original filename (just the file name, not path) for MIME type detection
+	Path     string     // Full path including folders (e.g., "contracts/2025/Q1/agreement.pdf"). Optional - if empty, uses Filename only.
 }
 
+// UploadResult contains all information from an upload operation
+type UploadResult struct {
+	Key          objectstore.BucketKey
+	OriginalPath string    // The original path provided (or filename if no path)
+	FolderID     uuid.UUID // The folder ID where the document was placed (root folder if no path specified)
+}
+
+// RootFolderPath is the path used for the auto-created root folder per client.
+const RootFolderPath = "/"
+
 // detectContentType determines the MIME type based on file extension
 func detectContentType(filename string) string {
 	// Use Go's standard library to detect MIME type from extension
@@ -33,14 +46,142 @@ func detectContentType(filename string) string {
 	return contentType
 }
 
+// parsePath extracts the folder path and filename from a full path.
+// Returns (folderPath, filename). If no folder path, folderPath is empty.
+// Examples:
+//   - "contracts/2025/Q1/agreement.pdf" -> ("contracts/2025/Q1", "agreement.pdf")
+//   - "agreement.pdf" -> ("", "agreement.pdf")
+//   - "/contracts/agreement.pdf" -> ("contracts", "agreement.pdf") (leading slash stripped)
+func parsePath(path string) (folderPath string, filename string) {
+	// Normalize: strip leading/trailing slashes
+	path = strings.TrimPrefix(path, "/")
+	path = strings.TrimSuffix(path, "/")
+
+	if path == "" {
+		return "", ""
+	}
+
+	// Split by last separator
+	lastSlash := strings.LastIndex(path, "/")
+	if lastSlash == -1 {
+		// No folder path, just filename
+		return "", path
+	}
+
+	return path[:lastSlash], path[lastSlash+1:]
+}
+
+// createFolderHierarchy creates all folders in the given path and returns the leaf folder ID.
+// For path "contracts/2025/Q1", it creates:
+//   - /contracts (parent: root folder "/")
+//   - /contracts/2025 (parent: /contracts)
+//   - /contracts/2025/Q1 (parent: /contracts/2025)
+//
+// Returns the UUID of the leaf folder (/contracts/2025/Q1 in this example).
+// If folderPath is empty, returns the root folder ID (documents without paths go to root).
+// The root folder "/" must exist (created when the client was created).
+func (s *Service) createFolderHierarchy(ctx context.Context, q *repository.Queries, clientID string, folderPath string, createdBy string) (uuid.UUID, error) {
+	// Get the root folder for this client (must exist - created when client was created)
+	rootFolder, err := q.GetFolderByPath(ctx, &repository.GetFolderByPathParams{
+		Clientid: clientID,
+		Path:     RootFolderPath,
+	})
+	if err != nil {
+		return uuid.Nil, fmt.Errorf("root folder not found for client %s: %w", clientID, err)
+	}
+
+	// Normalize path
+	folderPath = strings.TrimPrefix(folderPath, "/")
+	folderPath = strings.TrimSuffix(folderPath, "/")
+
+	// If no folder path provided, use root folder
+	if folderPath == "" {
+		return rootFolder.ID, nil
+	}
+
+	// Split path into parts
+	parts := strings.Split(folderPath, "/")
+
+	// Start with root as the parent
+	parentID := &rootFolder.ID
+	var currentPath string
+	var lastFolderID uuid.UUID
+
+	for _, part := range parts {
+		if part == "" {
+			continue
+		}
+
+		// Build the full path for this folder
+		if currentPath == "" {
+			currentPath = "/" + part
+		} else {
+			currentPath = currentPath + "/" + part
+		}
+
+		// Upsert the folder (create if not exists, return existing if exists)
+		folder, err := q.UpsertFolder(ctx, &repository.UpsertFolderParams{
+			Path:      currentPath,
+			Parentid:  parentID,
+			Clientid:  clientID,
+			Createdby: createdBy,
+		})
+		if err != nil {
+			return uuid.Nil, err
+		}
+
+		// This folder becomes the parent for the next level
+		lastFolderID = folder.ID
+		parentID = &folder.ID
+	}
+
+	return lastFolderID, nil
+}
+
 func (s *Service) Upload(ctx context.Context, file File) error {
-	_, err := s.UploadAndReturnKey(ctx, file)
+	_, err := s.UploadWithResult(ctx, file)
 	return err
 }
 
-// UploadAndReturnKey uploads a file and returns the BucketKey used
+// UploadAndReturnKey uploads a file and returns the BucketKey used (for backward compatibility)
 func (s *Service) UploadAndReturnKey(ctx context.Context, file File) (objectstore.BucketKey, error) {
-	var key objectstore.BucketKey
+	result, err := s.UploadWithResult(ctx, file)
+	if err != nil {
+		return objectstore.BucketKey{}, err
+	}
+	return result.Key, nil
+}
+
+// UploadWithResult uploads a file and returns the full upload result including folder information.
+// If file.Path is provided, it will:
+//  1. Extract the folder path and filename from the path
+//  2. Create the folder hierarchy in the database
+//  3. Return the folder ID in the result for use when creating the document
+func (s *Service) UploadWithResult(ctx context.Context, file File) (UploadResult, error) {
+	var result UploadResult
+
+	// Determine the original path and folder path
+	var originalPath string
+	var folderPath string
+	var filename string
+
+	if file.Path != "" {
+		// Use the provided path
+		originalPath = file.Path
+		folderPath, filename = parsePath(file.Path)
+		// If filename from path is empty but Filename is set, use Filename
+		if filename == "" && file.Filename != "" {
+			filename = file.Filename
+		}
+	} else {
+		// Fall back to just the filename (no folder)
+		originalPath = file.Filename
+		filename = file.Filename
+		folderPath = ""
+	}
+
+	result.OriginalPath = originalPath
+
 	err := s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, q *repository.Queries) error {
 		now := time.Now().UTC()
 		part, err := s.cfg.GetDBQueries().GetDocumentUploadCurrentPart(ctx, &repository.GetDocumentUploadCurrentPartParams{
@@ -54,9 +195,17 @@ func (s *Service) UploadAndReturnKey(ctx context.Context, file File) (objectstor
 			return err
 		}
 
+		// Create folder hierarchy if folder path is provided
+		// Use system email as createdBy since this is an automated upload process
+		folderID, err := s.createFolderHierarchy(ctx, q, file.ClientID, folderPath, "system@doczy.local")
+		if err != nil {
+			return err
+		}
+		result.FolderID = folderID
+
 		uploadId := uuid.New()
 		newPart := s.cfg.GetDirectoryPart(part.Part, part.Count)
-		key = objectstore.BucketKey{
+		result.Key = objectstore.BucketKey{
 			ClientID:  file.ClientID,
 			Location:  objectstore.Import,
 			CreatedAt: now,
@@ -64,9 +213,15 @@ func (s *Service) UploadAndReturnKey(ctx context.Context, file File) (objectstor
 			Part:      &newPart,
 			BatchID:   file.BatchID,
 		}
-		keyStr := key.String()
+		keyStr := result.Key.String()
 		bucket := s.cfg.GetBucket()
 
+		// Store the full original path in Filename to preserve the complete path from ZIP uploads
+		// The folder hierarchy is separately maintained via FolderID
+		filenameToStore := originalPath
+		if filenameToStore == "" {
+			filenameToStore = filename
+		}
 		err = q.AddDocumentUpload(ctx, &repository.AddDocumentUploadParams{
 			ID:       uploadId,
 			Clientid: file.ClientID,
@@ -77,20 +232,24 @@ func (s *Service) UploadAndReturnKey(ctx context.Context, file File) (objectstor
 				Valid: true,
 				Time:  now,
 			},
-			Filename: &file.Filename,
+			Filename: &filenameToStore,
 			BatchID:  file.BatchID,
+			FolderID: &folderID,
 		})
 		if err != nil {
 			return err
 		}
 
-		// Detect content type from original filename
-		contentType := detectContentType(file.Filename)
+		// Detect content type from filename
+		contentType := detectContentType(filename)
 
-		// Prepare metadata for S3 object
+		// Prepare metadata for S3 object - store original path for debugging
 		metadata := make(map[string]string)
-		if file.Filename != "" {
-			metadata["original-filename"] = file.Filename
+		if originalPath != "" {
+			metadata["original-path"] = originalPath
+		}
+		if filename != "" {
+			metadata["original-filename"] = filename
 		}
 
 		_, err = s.cfg.GetStoreClient().PutObject(ctx, &s3.PutObjectInput{
@@ -107,5 +266,5 @@ func (s *Service) UploadAndReturnKey(ctx context.Context, file File) (objectstor
 		return nil
 	})
 
-	return key, err
+	return result, err
 }

internal/document/upload/get_test.go

@@ -6,13 +6,13 @@ import (
 	"strings"
 	"testing"
 
+	"queryorchestration/internal/client"
 	"queryorchestration/internal/database/repository"
+	documentupload "queryorchestration/internal/document/upload"
 	"queryorchestration/internal/serviceconfig"
 	"queryorchestration/internal/serviceconfig/objectstore"
 	"queryorchestration/internal/test"
 
-	documentupload "queryorchestration/internal/document/upload"
-
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -23,6 +23,17 @@ type Config struct {
 	objectstore.ObjectStoreConfig
 }
 
+// createTestClient creates a client using the client service, which automatically creates the root folder.
+func createTestClient(t *testing.T, cfg *Config, clientID, clientName string) {
+	t.Helper()
+	clientSvc := client.New(cfg)
+	_, err := clientSvc.Create(t.Context(), client.CreateParams{
+		ID:   clientID,
+		Name: clientName,
+	})
+	require.NoError(t, err)
+}
+
 func TestUpload(t *testing.T) {
 	t.Parallel()
 	cfg := &Config{}
@@ -32,11 +43,8 @@ func TestUpload(t *testing.T) {
 	test.SetStoreClient(t, t.Context(), cfg, acfg.ExternalEndpoint)
 	test.CreateBucket(t, cfg)
 
-	err := cfg.GetDBQueries().CreateClient(t.Context(), &repository.CreateClientParams{
-		Clientid: "client_id",
-		Name:     "client",
-	})
-	require.NoError(t, err)
+	// Use client service to create client (which auto-creates root folder)
+	createTestClient(t, cfg, "client_id", "client")
 
 	svc := documentupload.New(cfg)
 
@@ -47,7 +55,7 @@ func TestUpload(t *testing.T) {
 	}
 	log.Print("test")
 
-	err = svc.Upload(t.Context(), file)
+	err := svc.Upload(t.Context(), file)
 	require.NoError(t, err)
 
 	os, err := cfg.StoreClient.ListObjects(t.Context(), &s3.ListObjectsInput{
@@ -67,3 +75,142 @@ func TestUpload(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, []byte("abc"), body)
 }
+
+func TestUploadWithPath(t *testing.T) {
+	t.Parallel()
+	cfg := &Config{}
+	test.CreateDB(t, cfg)
+	acfg := test.CreateAWSContainer(t, cfg)
+
+	test.SetStoreClient(t, t.Context(), cfg, acfg.ExternalEndpoint)
+	test.CreateBucket(t, cfg)
+
+	// Use client service to create client (which auto-creates root folder)
+	createTestClient(t, cfg, "client_id", "client")
+
+	svc := documentupload.New(cfg)
+
+	// Upload with a full path including folders
+	file := documentupload.File{
+		ClientID: "client_id",
+		Content:  strings.NewReader("content with path"),
+		Filename: "agreement.pdf",
+		Path:     "contracts/2025/Q1/agreement.pdf",
+	}
+
+	result, err := svc.UploadWithResult(t.Context(), file)
+	require.NoError(t, err)
+
+	// Verify original path is set correctly
+	assert.Equal(t, "contracts/2025/Q1/agreement.pdf", result.OriginalPath)
+
+	// Verify folder was assigned (always non-nil now since we always have at least root folder)
+	assert.NotEqual(t, result.FolderID.String(), "00000000-0000-0000-0000-000000000000")
+
+	// Verify S3 key is set
+	assert.NotEmpty(t, result.Key.String())
+
+	// Verify folder hierarchy was created in database
+	folder, err := cfg.GetDBQueries().GetFolderByID(t.Context(), result.FolderID)
+	require.NoError(t, err)
+	assert.Equal(t, "/contracts/2025/Q1", folder.Path)
+
+	// Verify parent folders exist (root "/" + /contracts, /contracts/2025, /contracts/2025/Q1)
+	folders, err := cfg.GetDBQueries().GetFoldersByClientID(t.Context(), "client_id")
+	require.NoError(t, err)
+	assert.Len(t, folders, 4) // /, /contracts, /contracts/2025, /contracts/2025/Q1
+
+	// Verify the root folder exists and is the parent of /contracts
+	rootFolder, err := cfg.GetDBQueries().GetFolderByPath(t.Context(), &repository.GetFolderByPathParams{
+		Clientid: "client_id",
+		Path:     "/",
+	})
+	require.NoError(t, err)
+	assert.Nil(t, rootFolder.Parentid, "Root folder should have no parent")
+
+	// Verify /contracts has root as parent
+	contractsFolder, err := cfg.GetDBQueries().GetFolderByPath(t.Context(), &repository.GetFolderByPathParams{
+		Clientid: "client_id",
+		Path:     "/contracts",
+	})
+	require.NoError(t, err)
+	assert.Equal(t, rootFolder.ID, *contractsFolder.Parentid, "/contracts should have root as parent")
+
+	// Verify S3 object exists
+	os, err := cfg.StoreClient.ListObjects(t.Context(), &s3.ListObjectsInput{
+		Bucket: &cfg.Bucket,
+	})
+	require.NoError(t, err)
+	assert.Len(t, os.Contents, 1)
+}
+
+func TestUploadWithPathNoFolder(t *testing.T) {
+	t.Parallel()
+	cfg := &Config{}
+	test.CreateDB(t, cfg)
+	acfg := test.CreateAWSContainer(t, cfg)
+
+	test.SetStoreClient(t, t.Context(), cfg, acfg.ExternalEndpoint)
+	test.CreateBucket(t, cfg)
+
+	// Use client service to create client (which auto-creates root folder)
+	createTestClient(t, cfg, "client_id", "client")
+
+	svc := documentupload.New(cfg)
+
+	// Upload with just a filename (no folder path)
+	file := documentupload.File{
+		ClientID: "client_id",
+		Content:  strings.NewReader("root content"),
+		Filename: "root_file.pdf",
+	}
+
+	result, err := svc.UploadWithResult(t.Context(), file)
+	require.NoError(t, err)
+
+	// Verify original path is just the filename
+	assert.Equal(t, "root_file.pdf", result.OriginalPath)
+
+	// Verify document is assigned to root folder
+	rootFolder, err := cfg.GetDBQueries().GetFolderByPath(t.Context(), &repository.GetFolderByPathParams{
+		Clientid: "client_id",
+		Path:     "/",
+	})
+	require.NoError(t, err)
+	assert.Equal(t, rootFolder.ID, result.FolderID, "Document without path should be in root folder")
+
+	// Verify only root folder exists in database
+	folders, err := cfg.GetDBQueries().GetFoldersByClientID(t.Context(), "client_id")
+	require.NoError(t, err)
+	assert.Len(t, folders, 1) // Only root folder "/"
+	assert.Equal(t, "/", folders[0].Path)
+}
+
+func TestUploadAndReturnKey(t *testing.T) {
+	t.Parallel()
+	cfg := &Config{}
+	test.CreateDB(t, cfg)
+	acfg := test.CreateAWSContainer(t, cfg)
+
+	test.SetStoreClient(t, t.Context(), cfg, acfg.ExternalEndpoint)
+	test.CreateBucket(t, cfg)
+
+	// Use client service to create client (which auto-creates root folder)
+	createTestClient(t, cfg, "client_id", "client")
+
+	svc := documentupload.New(cfg)
+
+	file := documentupload.File{
+		ClientID: "client_id",
+		Content:  strings.NewReader("key test content"),
+		Filename: "keytest.pdf",
+		Path:     "folder/keytest.pdf",
+	}
+
+	key, err := svc.UploadAndReturnKey(t.Context(), file)
+	require.NoError(t, err)
+
+	// Verify the key is valid
+	assert.Equal(t, "client_id", key.ClientID)
+	assert.NotEmpty(t, key.String())
+}