Jacob Mathison
a080ca59d8
Implement v2 upload batch cleanup * Implement v2 upload batch cleanup * Merge remote-tracking branch 'origin/main' into jmathison/v2-upload-batch * Address upload batch review feedback * Raise batch worker coverage * Fix batch cleanup review issues Approved-by: Jay Brown
90 lines
2.4 KiB
Go
90 lines
2.4 KiB
Go
// **Listens For**: S3 Event notifications - limited to PUT notifications.
|
|
//
|
|
// **Action**.
|
|
//
|
|
// Checks if the uploaded document is a duplicate.
|
|
//
|
|
// Currently whether the document content is a duplicate or not is determines using the document ETag, and check if it already exists for the client.
|
|
//
|
|
// Find reference entity.
|
|
//
|
|
// If NOT a duplicate - Create a document reference entity.
|
|
//
|
|
// Create a document entry to log the successful upload of a document.
|
|
//
|
|
// Add an event to the DOC SYNC queue.
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
"os"
|
|
|
|
docinitrunner "queryorchestration/api/docInitRunner"
|
|
"queryorchestration/internal/document/batch/foldercleanup"
|
|
"queryorchestration/internal/document/batch/outcome"
|
|
documentinit "queryorchestration/internal/document/init"
|
|
"queryorchestration/internal/server/runner"
|
|
"queryorchestration/internal/serviceconfig/build"
|
|
"queryorchestration/internal/serviceconfig/objectstore"
|
|
documentsyncc "queryorchestration/internal/serviceconfig/queue/documentsync"
|
|
|
|
_ "github.com/lib/pq"
|
|
)
|
|
|
|
type DocInitConfig struct {
|
|
runner.BaseConfig[docinitrunner.Body]
|
|
documentsyncc.DocSyncConfig
|
|
objectstore.ObjectStoreConfig // Provides GetStoreClient() for S3 access to measure file size
|
|
}
|
|
|
|
func main() {
|
|
// Print version information before any environment checks
|
|
build.PrintVersionInfo("docInitRunner")
|
|
|
|
ctx := context.Background()
|
|
|
|
cfg := &DocInitConfig{}
|
|
|
|
cfg.ControllerFunc = func() runner.Controller[docinitrunner.Body] {
|
|
docinit := documentinit.New(cfg)
|
|
|
|
return docinitrunner.New(&docinitrunner.Services{
|
|
Document: docinit,
|
|
Queries: cfg.GetDBQueries(),
|
|
Outcome: outcome.New(cfg.GetDBQueries()),
|
|
Cleanup: foldercleanup.New(cfg),
|
|
})
|
|
}
|
|
|
|
server, err := runner.New(ctx, cfg)
|
|
if err != nil {
|
|
slog.Error(err.Error())
|
|
os.Exit(1)
|
|
}
|
|
|
|
// Initialize S3 client for file size measurement AFTER runner.New() parses env vars
|
|
if err := cfg.SetStoreClient(ctx); err != nil {
|
|
slog.Error("failed to initialize S3 client", "error", err)
|
|
os.Exit(1)
|
|
}
|
|
|
|
// optional add shutdown metrics
|
|
defer func() {
|
|
err = cfg.GetMetrics().RecordLifecycleEvent(1.0, "shutdown", "main")
|
|
|
|
if err != nil {
|
|
slog.Error("Failed to record shutdown metric ", "error", err)
|
|
}
|
|
}()
|
|
|
|
// start and block until server is done
|
|
server.Listen(ctx)
|
|
|
|
// optional shutdown
|
|
metrics := cfg.GetMetrics()
|
|
if err := metrics.Shutdown(ctx); err != nil {
|
|
slog.Error("Error shutting down metrics server", "error", err)
|
|
}
|
|
}
|