jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/textExtractionsPart1 (pull request #192)

Browse Source 
all schema and rest apis for text extraction support

* in progress

* stage 8 complete

* phase 9 completed

* phase 9 complete

* ongoing - s3 path fix

* working

* optimize ci build

* e2e tests

* missing test
This commit is contained in:
Jay Brown
2025-11-26 19:23:42 +00:00
parent 2b43799f56
commit c45e1dd427
67 changed files with 16120 additions and 817 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/server/api/batch_worker_test.go
+18 -42
View File
@@ -35,13 +35,9 @@ func TestProcessBatchWork(t *testing.T) {
test.CreateDB(t, cfg)
test.CreateAWSResources(t, cfg)
// Create test client
// Create test client with root folder
clientID := "test_batch_worker"
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Batch Worker Client",
})
require.NoError(t, err)
test.CreateTestClient(t, cfg, clientID, "Test Batch Worker Client")
// Create a test batch service
batchService := batch.New(cfg)
@@ -54,7 +50,7 @@ func TestProcessBatchWork(t *testing.T) {
bucket := cfg.GetBucket()
archiveKey := fmt.Sprintf("test/%s/batch.zip", clientID)
_, err = s3Client.PutObject(ctx, &s3.PutObjectInput{
_, err := s3Client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(archiveKey),
Body: bytes.NewReader(zipContent),
@@ -114,13 +110,9 @@ func TestProcessSingleBatch(t *testing.T) {
test.CreateDB(t, cfg)
test.CreateAWSResources(t, cfg)
// Create test client
// Create test client with root folder
clientID := "test_single_batch"
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Single Batch Client",
})
require.NoError(t, err)
test.CreateTestClient(t, cfg, clientID, "Test Single Batch Client")
// Create services
batchService := batch.New(cfg)
@@ -133,7 +125,7 @@ func TestProcessSingleBatch(t *testing.T) {
zipContent := createMixedContentZIP(t)
archiveKey := fmt.Sprintf("test/%s/mixed.zip", clientID)
_, err = s3Client.PutObject(ctx, &s3.PutObjectInput{
_, err := s3Client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(archiveKey),
Body: bytes.NewReader(zipContent),
@@ -190,13 +182,9 @@ func TestProcessSingleBatchWithFailure(t *testing.T) {
test.CreateDB(t, cfg)
test.CreateAWSResources(t, cfg)
// Create test client
// Create test client with root folder
clientID := "test_batch_failure"
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Batch Failure Client",
})
require.NoError(t, err)
test.CreateTestClient(t, cfg, clientID, "Test Batch Failure Client")
// Create services
batchService := batch.New(cfg)
@@ -209,7 +197,7 @@ func TestProcessSingleBatchWithFailure(t *testing.T) {
zipContent := createTestZIPForWorker(t, 3)
archiveKey := fmt.Sprintf("test/%s/fail.zip", clientID)
_, err = s3Client.PutObject(ctx, &s3.PutObjectInput{
_, err := s3Client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(archiveKey),
Body: bytes.NewReader(zipContent),
@@ -485,13 +473,9 @@ func TestProcessBatchWithNestedFolders(t *testing.T) {
test.CreateDB(t, cfg)
test.CreateAWSResources(t, cfg)
// Create test client
// Create test client with root folder
clientID := "test_nested_folders"
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Nested Folders Client",
})
require.NoError(t, err)
test.CreateTestClient(t, cfg, clientID, "Test Nested Folders Client")
// Create services
batchService := batch.New(cfg)
@@ -504,7 +488,7 @@ func TestProcessBatchWithNestedFolders(t *testing.T) {
zipContent := createNestedFolderZIP(t)
archiveKey := fmt.Sprintf("test/%s/nested.zip", clientID)
_, err = s3Client.PutObject(ctx, &s3.PutObjectInput{
_, err := s3Client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(archiveKey),
Body: bytes.NewReader(zipContent),
@@ -616,13 +600,9 @@ func TestProcessBatchWithPathTraversal(t *testing.T) {
test.CreateDB(t, cfg)
test.CreateAWSResources(t, cfg)
// Create test client
// Create test client with root folder
clientID := "test_path_traversal"
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Path Traversal Client",
})
require.NoError(t, err)
test.CreateTestClient(t, cfg, clientID, "Test Path Traversal Client")
// Create services
batchService := batch.New(cfg)
@@ -635,7 +615,7 @@ func TestProcessBatchWithPathTraversal(t *testing.T) {
zipContent := createMaliciousPathZIP(t)
archiveKey := fmt.Sprintf("test/%s/malicious.zip", clientID)
_, err = s3Client.PutObject(ctx, &s3.PutObjectInput{
_, err := s3Client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(archiveKey),
Body: bytes.NewReader(zipContent),
@@ -700,13 +680,9 @@ func TestProcessBatchWithDuplicateFilenames(t *testing.T) {
test.CreateDB(t, cfg)
test.CreateAWSResources(t, cfg)
// Create test client
// Create test client with root folder
clientID := "test_duplicate_names"
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Duplicate Names Client",
})
require.NoError(t, err)
test.CreateTestClient(t, cfg, clientID, "Test Duplicate Names Client")
// Create services
batchService := batch.New(cfg)
@@ -737,7 +713,7 @@ func TestProcessBatchWithDuplicateFilenames(t *testing.T) {
require.NoError(t, err)
}
err = zipWriter.Close()
err := zipWriter.Close()
require.NoError(t, err)
zipContent := buf.Bytes()
internal/server/api/listener.go
+9 -2
View File
@@ -360,21 +360,28 @@ func New(ctx context.Context, cfg Config) (*Server, error) {
// createDocumentUploadHandler creates a function that can upload documents
// directly through the service layer, bypassing HTTP. For batch uploads,
// it creates the document record immediately with filename information.
// The filename parameter may contain folder structure (e.g., "folder1/subfolder2/file.pdf")
// which will be used to create folder records in the database.
func createDocumentUploadHandler(cfg Config) func(ctx context.Context, clientID string, docData io.Reader, filename string, batchID *uuid.UUID) error {
// Import the upload service
uploadService := documentupload.New(cfg)
return func(ctx context.Context, clientID string, docData io.Reader, filename string, batchID *uuid.UUID) error {
// Create file struct for upload service
// Pass the filename as Path to enable folder hierarchy creation.
// The upload service will parse the path and create folder records automatically.
file := documentupload.File{
ClientID: clientID,
Content: docData,
BatchID: batchID,
Filename: filename,
Path: filename, // Use as Path to trigger folder creation
}
// Use the unified upload path for both batch and single uploads
// The filename will be stored in documentUploads table and used by the async pipeline
// The upload service will:
// 1. Parse the path to extract folder structure
// 2. Create folder records in the database
// 3. Store the file with proper folder association
return uploadService.Upload(ctx, file)
}
}
internal/server/api/listener_test.go
+3 -8
View File
@@ -11,7 +11,6 @@ import (
"testing"
"queryorchestration/internal/backgroundtask"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/build"
"queryorchestration/internal/serviceconfig/logger"
@@ -72,15 +71,11 @@ func TestCreateDocumentUploadHandler(t *testing.T) {
filename := "test.pdf"
content := strings.NewReader("test pdf content")
// Create client for testing
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Client",
})
require.NoError(t, err)
// Create client with root folder for testing
test.CreateTestClient(t, cfg, clientID, "Test Client")
// Test upload handler
err = uploadHandler(ctx, clientID, content, filename, nil)
err := uploadHandler(ctx, clientID, content, filename, nil)
assert.NoError(t, err)
// Test upload handler with error (invalid client)