Jacob Mathison
a080ca59d8
Implement v2 upload batch cleanup * Implement v2 upload batch cleanup * Merge remote-tracking branch 'origin/main' into jmathison/v2-upload-batch * Address upload batch review feedback * Raise batch worker coverage * Fix batch cleanup review issues Approved-by: Jay Brown
64 lines
1.6 KiB
Go
64 lines
1.6 KiB
Go
// Listens For: Body Containing Document ID.
|
|
//
|
|
// Action:
|
|
// If the document is not already clean according to the collector standards, a clean is triggered.
|
|
// Note: Text extraction has been removed. The document pipeline ends after cleaning.
|
|
// See remove_texttract_and_mocks_plan.md for details.
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
"os"
|
|
|
|
doccleanrunner "queryorchestration/api/docCleanRunner"
|
|
"queryorchestration/internal/document/batch/foldercleanup"
|
|
"queryorchestration/internal/document/batch/outcome"
|
|
documentclean "queryorchestration/internal/document/clean"
|
|
"queryorchestration/internal/server/runner"
|
|
"queryorchestration/internal/serviceconfig/build"
|
|
"queryorchestration/internal/serviceconfig/objectstore"
|
|
|
|
_ "github.com/lib/pq"
|
|
)
|
|
|
|
// DocCleanConfig provides configuration for the document cleaning runner.
|
|
// Note: documenttext.DocTextConfig has been removed since text extraction is deprecated.
|
|
type DocCleanConfig struct {
|
|
runner.BaseConfig[doccleanrunner.Body]
|
|
objectstore.ObjectStoreConfig
|
|
}
|
|
|
|
func main() {
|
|
// Print version information before any environment checks
|
|
build.PrintVersionInfo("docCleanRunner")
|
|
|
|
ctx := context.Background()
|
|
|
|
cfg := &DocCleanConfig{}
|
|
|
|
cfg.ControllerFunc = func() runner.Controller[doccleanrunner.Body] {
|
|
clean := documentclean.New(cfg)
|
|
|
|
return doccleanrunner.New(&doccleanrunner.Services{
|
|
Clean: clean,
|
|
Outcome: outcome.New(cfg.GetDBQueries()),
|
|
Cleanup: foldercleanup.New(cfg),
|
|
})
|
|
}
|
|
|
|
server, err := runner.New(ctx, cfg)
|
|
if err != nil {
|
|
slog.Error(err.Error())
|
|
os.Exit(1)
|
|
}
|
|
|
|
err = cfg.SetStoreClient(ctx)
|
|
if err != nil {
|
|
slog.Error(err.Error())
|
|
os.Exit(1)
|
|
}
|
|
|
|
server.Listen(ctx)
|
|
}
|