jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/textExtractionsPart1 (pull request #192)

Browse Source 
all schema and rest apis for text extraction support

* in progress

* stage 8 complete

* phase 9 completed

* phase 9 complete

* ongoing - s3 path fix

* working

* optimize ci build

* e2e tests

* missing test
This commit is contained in:
Jay Brown
2025-11-26 19:23:42 +00:00
parent 2b43799f56
commit c45e1dd427
67 changed files with 16120 additions and 817 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/server/api/batch_worker_test.go
+18 -42
View File
@@ -35,13 +35,9 @@ func TestProcessBatchWork(t *testing.T) {
test.CreateDB(t, cfg)
test.CreateAWSResources(t, cfg)
// Create test client
// Create test client with root folder
clientID := "test_batch_worker"
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Batch Worker Client",
})
require.NoError(t, err)
test.CreateTestClient(t, cfg, clientID, "Test Batch Worker Client")
// Create a test batch service
batchService := batch.New(cfg)
@@ -54,7 +50,7 @@ func TestProcessBatchWork(t *testing.T) {
bucket := cfg.GetBucket()
archiveKey := fmt.Sprintf("test/%s/batch.zip", clientID)
_, err = s3Client.PutObject(ctx, &s3.PutObjectInput{
_, err := s3Client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(archiveKey),
Body: bytes.NewReader(zipContent),
@@ -114,13 +110,9 @@ func TestProcessSingleBatch(t *testing.T) {
test.CreateDB(t, cfg)
test.CreateAWSResources(t, cfg)
// Create test client
// Create test client with root folder
clientID := "test_single_batch"
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Single Batch Client",
})
require.NoError(t, err)
test.CreateTestClient(t, cfg, clientID, "Test Single Batch Client")
// Create services
batchService := batch.New(cfg)
@@ -133,7 +125,7 @@ func TestProcessSingleBatch(t *testing.T) {
zipContent := createMixedContentZIP(t)
archiveKey := fmt.Sprintf("test/%s/mixed.zip", clientID)
_, err = s3Client.PutObject(ctx, &s3.PutObjectInput{
_, err := s3Client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(archiveKey),
Body: bytes.NewReader(zipContent),
@@ -190,13 +182,9 @@ func TestProcessSingleBatchWithFailure(t *testing.T) {
test.CreateDB(t, cfg)
test.CreateAWSResources(t, cfg)
// Create test client
// Create test client with root folder
clientID := "test_batch_failure"
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Batch Failure Client",
})
require.NoError(t, err)
test.CreateTestClient(t, cfg, clientID, "Test Batch Failure Client")
// Create services
batchService := batch.New(cfg)
@@ -209,7 +197,7 @@ func TestProcessSingleBatchWithFailure(t *testing.T) {
zipContent := createTestZIPForWorker(t, 3)
archiveKey := fmt.Sprintf("test/%s/fail.zip", clientID)
_, err = s3Client.PutObject(ctx, &s3.PutObjectInput{
_, err := s3Client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(archiveKey),
Body: bytes.NewReader(zipContent),
@@ -485,13 +473,9 @@ func TestProcessBatchWithNestedFolders(t *testing.T) {
test.CreateDB(t, cfg)
test.CreateAWSResources(t, cfg)
// Create test client
// Create test client with root folder
clientID := "test_nested_folders"
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Nested Folders Client",
})
require.NoError(t, err)
test.CreateTestClient(t, cfg, clientID, "Test Nested Folders Client")
// Create services
batchService := batch.New(cfg)
@@ -504,7 +488,7 @@ func TestProcessBatchWithNestedFolders(t *testing.T) {
zipContent := createNestedFolderZIP(t)
archiveKey := fmt.Sprintf("test/%s/nested.zip", clientID)
_, err = s3Client.PutObject(ctx, &s3.PutObjectInput{
_, err := s3Client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(archiveKey),
Body: bytes.NewReader(zipContent),
@@ -616,13 +600,9 @@ func TestProcessBatchWithPathTraversal(t *testing.T) {
test.CreateDB(t, cfg)
test.CreateAWSResources(t, cfg)
// Create test client
// Create test client with root folder
clientID := "test_path_traversal"
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Path Traversal Client",
})
require.NoError(t, err)
test.CreateTestClient(t, cfg, clientID, "Test Path Traversal Client")
// Create services
batchService := batch.New(cfg)
@@ -635,7 +615,7 @@ func TestProcessBatchWithPathTraversal(t *testing.T) {
zipContent := createMaliciousPathZIP(t)
archiveKey := fmt.Sprintf("test/%s/malicious.zip", clientID)
_, err = s3Client.PutObject(ctx, &s3.PutObjectInput{
_, err := s3Client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(archiveKey),
Body: bytes.NewReader(zipContent),
@@ -700,13 +680,9 @@ func TestProcessBatchWithDuplicateFilenames(t *testing.T) {
test.CreateDB(t, cfg)
test.CreateAWSResources(t, cfg)
// Create test client
// Create test client with root folder
clientID := "test_duplicate_names"
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Duplicate Names Client",
})
require.NoError(t, err)
test.CreateTestClient(t, cfg, clientID, "Test Duplicate Names Client")
// Create services
batchService := batch.New(cfg)
@@ -737,7 +713,7 @@ func TestProcessBatchWithDuplicateFilenames(t *testing.T) {
require.NoError(t, err)
}
err = zipWriter.Close()
err := zipWriter.Close()
require.NoError(t, err)
zipContent := buf.Bytes()