jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/support-more-types (pull request #219)

Browse Source 
support new types for import

* tests pass

* missing file

* bug fixes
This commit is contained in:
Jay Brown
2026-03-31 17:40:42 +00:00
parent 8e9889accc
commit b71a28d3c0
144 changed files with 28352 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/document/batch/service.go
+64
View File
@@ -141,6 +141,10 @@ func (s *Service) Get(ctx context.Context, clientID string, batchID uuid.UUID) (
}
details.DocumentOutcomes = outcomes
// Derive summary counts from actual outcome states rather than
// the extraction-phase counters stored in batch_uploads.
s.deriveSummaryCounts(details)
return details, nil
}
@@ -197,6 +201,23 @@ func (s *Service) UpdateProgress(ctx context.Context, clientID string, batchID u
})
}
// SetTotalDocuments updates the total document count for a batch.
// Called after ZIP extraction when the actual file count is known.
//
// Parameters:
// - ctx: request context
// - batchID: the batch UUID
// - total: the total number of files extracted from the ZIP
//
// Returns:
// - error: database error, or nil
func (s *Service) SetTotalDocuments(ctx context.Context, batchID uuid.UUID, total int32) error {
return s.cfg.GetDBQueries().SetBatchTotalDocuments(ctx, &repository.SetBatchTotalDocumentsParams{
ID: batchID,
TotalDocuments: total,
})
}
// UpdateStatus updates the status of a batch
func (s *Service) UpdateStatus(ctx context.Context, batchID uuid.UUID, status string) error {
// Convert string status to repository.BatchStatus
@@ -407,6 +428,49 @@ func (s *Service) CheckDuplicate(ctx context.Context, clientID string, hash stri
return &id, nil
}
// deriveSummaryCounts recomputes ProcessedDocuments, FailedDocuments, and
// InvalidTypeDocuments from the actual document outcome states. This ensures the
// summary reflects final pipeline results rather than extraction-phase counters.
//
// Terminal outcome bucketing:
//
// processed: clean_passed, sync_skipped, duplicate, init_duplicate
// failed: clean_failed, failed_open, failed_read, failed_s3_upload, failed_upload
// invalid_type: invalid_type
// in-progress: submitted, init_complete, sync_complete (not expected at completion)
func (s *Service) deriveSummaryCounts(details *BatchUploadDetails) {
if len(details.DocumentOutcomes) == 0 {
return
}
var processed, failed, invalidType int32
for _, o := range details.DocumentOutcomes {
switch o.Outcome {
case "clean_passed", "sync_skipped", "duplicate", "init_duplicate":
processed++
case "clean_failed", "failed_open", "failed_read",
"failed_s3_upload", "failed_upload":
failed++
case "invalid_type":
invalidType++
default:
// Non-terminal states (submitted, init_complete, sync_complete)
// are in-progress; don't count them in any bucket.
}
}
details.ProcessedDocuments = processed
details.FailedDocuments = failed
details.InvalidTypeDocuments = invalidType
// Recalculate progress percent from derived counts
total := details.TotalDocuments
if total > 0 {
done := processed + failed + invalidType
details.ProgressPercent = int32(float64(done) / float64(total) * 100)
}
}
// ListUnprocessed retrieves all unprocessed batch uploads across all clients
func (s *Service) ListUnprocessed(ctx context.Context) ([]*BatchUploadSummary, error) {
batches, err := s.cfg.GetDBQueries().GetUnprocessedBatches(ctx)
internal/document/batch/service_test.go
+220
View File
@@ -716,6 +716,88 @@ func TestListOutcomes_WithCleanFail(t *testing.T) {
assert.Equal(t, "invalid_mimetype", *outcomes[0].CleanFail)
}
func TestSetTotalDocuments(t *testing.T) {
cfg := &TestConfig{}
test.CreateDB(t, cfg)
service := documentbatch.New(cfg)
ctx := t.Context()
// Create test client
clientID := "test_client_set_total"
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Client Set Total",
})
require.NoError(t, err)
// Create batch with totalDocs=0
batchID, err := service.Create(ctx, clientID, "test.zip", 0)
require.NoError(t, err)
// Set total documents to 15
err = service.SetTotalDocuments(ctx, batchID, 15)
require.NoError(t, err)
// Fetch and assert TotalDocuments == 15
batch, err := cfg.GetDBQueries().GetBatchUpload(ctx, &repository.GetBatchUploadParams{
ID: batchID,
ClientID: clientID,
})
require.NoError(t, err)
assert.Equal(t, int32(15), batch.TotalDocuments)
// Update progress and verify all fields
err = service.UpdateProgress(ctx, clientID, batchID, 10, 3, 2)
require.NoError(t, err)
batch, err = cfg.GetDBQueries().GetBatchUpload(ctx, &repository.GetBatchUploadParams{
ID: batchID,
ClientID: clientID,
})
require.NoError(t, err)
assert.Equal(t, int32(10), batch.ProcessedDocuments)
assert.Equal(t, int32(3), batch.FailedDocuments)
assert.Equal(t, int32(2), batch.InvalidTypeDocuments)
assert.Equal(t, int32(100), batch.ProgressPercent) // (10+3+2)/15 * 100 = 100%
}
func TestSetTotalDocuments_ThenUpdateProgress(t *testing.T) {
cfg := &TestConfig{}
test.CreateDB(t, cfg)
service := documentbatch.New(cfg)
ctx := t.Context()
// Create test client
clientID := "test_client_set_total_flow"
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Client Set Total Flow",
})
require.NoError(t, err)
// Create batch with totalDocs=0
batchID, err := service.Create(ctx, clientID, "flow.zip", 0)
require.NoError(t, err)
// Set total to 20
err = service.SetTotalDocuments(ctx, batchID, 20)
require.NoError(t, err)
// Update progress (10 processed, 5 failed, 3 invalid)
err = service.UpdateProgress(ctx, clientID, batchID, 10, 5, 3)
require.NoError(t, err)
// Assert ProgressPercent == 90 (18/20 * 100)
batch, err := cfg.GetDBQueries().GetBatchUpload(ctx, &repository.GetBatchUploadParams{
ID: batchID,
ClientID: clientID,
})
require.NoError(t, err)
assert.Equal(t, int32(90), batch.ProgressPercent)
}
// TestCheckDuplicate creates a document, then verifies that CheckDuplicate
// returns its ID when the hash matches.
func TestCheckDuplicate(t *testing.T) {
@@ -751,3 +833,141 @@ func TestCheckDuplicate(t *testing.T) {
require.NoError(t, err)
assert.Nil(t, noMatch)
}
// TestGet_DerivesSummaryCountsFromOutcomes verifies that the Get endpoint
// derives ProcessedDocuments, FailedDocuments, and InvalidTypeDocuments from
// actual document outcome states rather than the extraction-phase counters
// stored in batch_uploads.
func TestGet_DerivesSummaryCountsFromOutcomes(t *testing.T) {
cfg := &TestConfig{}
test.CreateDB(t, cfg)
ctx := t.Context()
service := documentbatch.New(cfg)
clientID := "test_derive_counts_" + uuid.New().String()[:8]
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Client Derive Counts",
})
require.NoError(t, err)
// Create batch -- extraction-phase counters will all be zero
batchID, err := service.Create(ctx, clientID, "mixed.zip", 0)
require.NoError(t, err)
// Set total so progress percent can be calculated
err = service.SetTotalDocuments(ctx, batchID, 7)
require.NoError(t, err)
// Create real documents for outcomes that require document_id
docA, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientID, Hash: "hash_a", BatchID: &batchID,
})
require.NoError(t, err)
docB, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientID, Hash: "hash_b", BatchID: &batchID,
})
require.NoError(t, err)
docC, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientID, Hash: "hash_c", BatchID: &batchID,
})
require.NoError(t, err)
// Record outcomes: 3 processed, 3 failed, 1 invalid_type
err = service.RecordOutcome(ctx, batchID, "a.pdf",
repository.BatchOutcomeStatusCleanPassed, nil, &docA)
require.NoError(t, err)
err = service.RecordOutcome(ctx, batchID, "b.pdf",
repository.BatchOutcomeStatusDuplicate, nil, &docB)
require.NoError(t, err)
err = service.RecordOutcome(ctx, batchID, "c.pdf",
repository.BatchOutcomeStatusInitDuplicate, nil, &docC)
require.NoError(t, err)
err = service.RecordOutcome(ctx, batchID, "d.pdf",
repository.BatchOutcomeStatusCleanFailed, nil, nil)
require.NoError(t, err)
errOpen := "cannot open"
err = service.RecordOutcome(ctx, batchID, "e.pdf",
repository.BatchOutcomeStatusFailedOpen, &errOpen, nil)
require.NoError(t, err)
errRead := "truncated"
err = service.RecordOutcome(ctx, batchID, "f.pdf",
repository.BatchOutcomeStatusFailedRead, &errRead, nil)
require.NoError(t, err)
err = service.RecordOutcome(ctx, batchID, "g.exe",
repository.BatchOutcomeStatusInvalidType, nil, nil)
require.NoError(t, err)
// Call Get and assert derived summary counts
details, err := service.Get(ctx, clientID, batchID)
require.NoError(t, err)
assert.Equal(t, int32(3), details.ProcessedDocuments,
"processed should count clean_passed + duplicate + init_duplicate")
assert.Equal(t, int32(3), details.FailedDocuments,
"failed should count clean_failed + failed_open + failed_read")
assert.Equal(t, int32(1), details.InvalidTypeDocuments,
"invalid_type should count invalid_type outcomes")
assert.Equal(t, int32(100), details.ProgressPercent,
"progress should be 100%% when all 7 outcomes are terminal")
assert.Equal(t, int32(7), details.TotalDocuments)
}
// TestGet_DerivedCountsWithInProgressOutcomes verifies that non-terminal
// outcomes (submitted, init_complete, sync_complete) are not counted in
// any summary bucket.
func TestGet_DerivedCountsWithInProgressOutcomes(t *testing.T) {
cfg := &TestConfig{}
test.CreateDB(t, cfg)
ctx := t.Context()
service := documentbatch.New(cfg)
clientID := "test_inprog_counts_" + uuid.New().String()[:8]
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Client InProgress Counts",
})
require.NoError(t, err)
batchID, err := service.Create(ctx, clientID, "partial.zip", 0)
require.NoError(t, err)
err = service.SetTotalDocuments(ctx, batchID, 3)
require.NoError(t, err)
// 1 terminal, 2 non-terminal
docA, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientID, Hash: "hash_term", BatchID: &batchID,
})
require.NoError(t, err)
err = service.RecordOutcome(ctx, batchID, "a.pdf",
repository.BatchOutcomeStatusCleanPassed, nil, &docA)
require.NoError(t, err)
err = service.RecordOutcome(ctx, batchID, "b.pdf",
repository.BatchOutcomeStatusSubmitted, nil, nil)
require.NoError(t, err)
err = service.RecordOutcome(ctx, batchID, "c.pdf",
repository.BatchOutcomeStatusInitComplete, nil, nil)
require.NoError(t, err)
details, err := service.Get(ctx, clientID, batchID)
require.NoError(t, err)
assert.Equal(t, int32(1), details.ProcessedDocuments,
"only terminal processed outcomes should be counted")
assert.Equal(t, int32(0), details.FailedDocuments,
"no failed outcomes recorded")
assert.Equal(t, int32(0), details.InvalidTypeDocuments)
assert.Equal(t, int32(33), details.ProgressPercent,
"only 1 of 3 outcomes is terminal: 33%%")
}