Merged in feature/remove-mocks (pull request #202)
Feature/remove mocks * remove mocks * cleanup db migrations
This commit is contained in:
Jay Brown
@@ -1,10 +1,9 @@
|
||||
// **Listens For**: Body Containing Document ID.
|
||||
//
|
||||
// **Action**.
|
||||
// Listens For: Body Containing Document ID.
|
||||
//
|
||||
// Action:
|
||||
// If the document is not already clean according to the collector standards, a clean is triggered.
|
||||
//
|
||||
// An event with the document id is pushed to the DOCTEXT queue.
|
||||
// Note: Text extraction has been removed. The document pipeline ends after cleaning.
|
||||
// See remove_texttract_and_mocks_plan.md for details.
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -17,14 +16,14 @@ import (
|
||||
"queryorchestration/internal/server/runner"
|
||||
"queryorchestration/internal/serviceconfig/build"
|
||||
"queryorchestration/internal/serviceconfig/objectstore"
|
||||
"queryorchestration/internal/serviceconfig/queue/documenttext"
|
||||
|
||||
_ "github.com/lib/pq"
|
||||
)
|
||||
|
||||
// DocCleanConfig provides configuration for the document cleaning runner.
|
||||
// Note: documenttext.DocTextConfig has been removed since text extraction is deprecated.
|
||||
type DocCleanConfig struct {
|
||||
runner.BaseConfig[doccleanrunner.Body]
|
||||
documenttext.DocTextConfig
|
||||
objectstore.ObjectStoreConfig
|
||||
}
|
||||
|
||||
|
||||
+34
-46
@@ -1,67 +1,55 @@
|
||||
// **Listens For**: Body Containing Document ID
|
||||
//
|
||||
// **Action**.
|
||||
//
|
||||
// If the document text is not already extracted according to the collector standards, trigger the textract job for text detection.
|
||||
//
|
||||
// This is the terminal step in the document processing pipeline.
|
||||
// Query processing has been removed - pipeline ends after text extraction.
|
||||
// DEPRECATED: This runner is deprecated and no longer has any functionality.
|
||||
// It is left in place until deployment code is updated accordingly.
|
||||
// See remove_texttract_and_mocks_plan.md for details.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
doctextrunner "queryorchestration/api/docTextRunner"
|
||||
documenttext "queryorchestration/internal/document/text"
|
||||
"queryorchestration/internal/server/runner"
|
||||
"queryorchestration/internal/serviceconfig/build"
|
||||
"queryorchestration/internal/serviceconfig/objectstore"
|
||||
"queryorchestration/internal/serviceconfig/textract"
|
||||
|
||||
_ "github.com/lib/pq"
|
||||
)
|
||||
|
||||
type DocTextConfig struct {
|
||||
runner.BaseConfig[doctextrunner.Body]
|
||||
textract.TextractConfig
|
||||
objectstore.ObjectStoreConfig
|
||||
}
|
||||
const serviceName = "docTextRunner"
|
||||
|
||||
func main() {
|
||||
// Print version information before any environment checks
|
||||
build.PrintVersionInfo("docTextRunner")
|
||||
build.PrintVersionInfo(serviceName)
|
||||
|
||||
ctx := context.Background()
|
||||
slog.Warn("DEPRECATED: " + serviceName + " is deprecated and no longer has any functionality. " +
|
||||
"This service is left in place until deployment code is updated accordingly.")
|
||||
|
||||
cfg := &DocTextConfig{}
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/health", func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
})
|
||||
|
||||
cfg.ControllerFunc = func() runner.Controller[doctextrunner.Body] {
|
||||
text := documenttext.New(cfg)
|
||||
|
||||
return doctextrunner.New(&doctextrunner.Services{
|
||||
Text: text,
|
||||
})
|
||||
server := &http.Server{
|
||||
Addr: ":8085",
|
||||
Handler: mux,
|
||||
ReadHeaderTimeout: time.Minute,
|
||||
}
|
||||
|
||||
server, err := runner.New(ctx, cfg)
|
||||
if err != nil {
|
||||
slog.Error(err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
go func() {
|
||||
slog.Info("Starting health check server on port 8085")
|
||||
if err := server.ListenAndServe(); err != http.ErrServerClosed {
|
||||
slog.Error("Health check server error", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
err = cfg.SetStoreClient(ctx)
|
||||
if err != nil {
|
||||
slog.Error(err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
sigChan := make(chan os.Signal, 1)
|
||||
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
|
||||
|
||||
err = cfg.SetTextractClient(ctx)
|
||||
if err != nil {
|
||||
slog.Error(err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
sig := <-sigChan
|
||||
slog.Info(serviceName+" received shutdown signal, exiting gracefully", "signal", sig.String())
|
||||
|
||||
server.Listen(ctx)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if err := server.Shutdown(ctx); err != nil {
|
||||
slog.Error("Error shutting down health check server", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,111 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"queryorchestration/internal/serviceconfig"
|
||||
"queryorchestration/internal/serviceconfig/aws"
|
||||
"queryorchestration/internal/serviceconfig/build"
|
||||
"queryorchestration/internal/serviceconfig/objectstore"
|
||||
"queryorchestration/internal/serviceconfig/textract"
|
||||
|
||||
documenttext "queryorchestration/internal/document/text"
|
||||
documenttypes "queryorchestration/internal/document/types"
|
||||
)
|
||||
|
||||
// Config provides the configuration for the text extractor tool.
|
||||
// Note: Query-related configuration has been removed. See remove_query_plan.md.
|
||||
type Config struct {
|
||||
serviceconfig.BaseConfig
|
||||
textract.TextractConfig
|
||||
objectstore.ObjectStoreConfig
|
||||
}
|
||||
|
||||
func main() {
|
||||
// Print version information before any environment checks
|
||||
build.PrintVersionInfo("textExtractor")
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
directory := "files/"
|
||||
|
||||
cfg := &Config{}
|
||||
profile := aws.Profile(os.Getenv("AWS_PROFILE"))
|
||||
if profile == "" {
|
||||
slog.Error("profile not available")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
err := cfg.SetTextractClientWithProfile(ctx, profile)
|
||||
if err != nil {
|
||||
slog.Error("error creating client", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
entries, err := os.ReadDir(directory)
|
||||
if err != nil {
|
||||
slog.Error("error finding directory", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
|
||||
extractFile(ctx, cfg, directory, entry)
|
||||
}
|
||||
}
|
||||
|
||||
func extractFile(ctx context.Context, cfg *Config, directory string, entry os.DirEntry) {
|
||||
filePath := filepath.Join(directory, entry.Name())
|
||||
pdfFile, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
slog.Error("error opening pdf file", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
pdf, err := documenttypes.NewPDFFromReader(pdfFile)
|
||||
if err != nil {
|
||||
slog.Error("error creating pdf file", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
textSvc := documenttext.New(cfg)
|
||||
|
||||
startTime := time.Now()
|
||||
|
||||
text, err := textSvc.GetDocumentText(ctx, pdf)
|
||||
if err != nil {
|
||||
slog.Error("error getting page count", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
outputFilename := fmt.Sprintf("./text/%s.txt", entry.Name())
|
||||
file, err := os.Create(outputFilename)
|
||||
if err != nil {
|
||||
slog.Error("error creating file", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
s, err := io.ReadAll(text)
|
||||
if err != nil {
|
||||
slog.Error("error reading text", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
_, err = file.Write(s)
|
||||
if err != nil {
|
||||
slog.Error("error writing file", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
slog.Info("extracted text", "output", outputFilename, "time", time.Since(startTime))
|
||||
}
|
||||
@@ -1,105 +0,0 @@
|
||||
//go:build aws
|
||||
|
||||
//go:generate go run -tags=aws ./textdetection.go
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
documenttext "queryorchestration/internal/document/text"
|
||||
documenttypes "queryorchestration/internal/document/types"
|
||||
"queryorchestration/internal/serviceconfig"
|
||||
"queryorchestration/internal/serviceconfig/aws"
|
||||
"queryorchestration/internal/serviceconfig/objectstore"
|
||||
"queryorchestration/internal/serviceconfig/textract"
|
||||
)
|
||||
|
||||
// Config provides the configuration for the text generator test tool.
|
||||
// Note: Query-related configuration has been removed. See remove_query_plan.md.
|
||||
type Config struct {
|
||||
serviceconfig.BaseConfig
|
||||
textract.TextractConfig
|
||||
objectstore.ObjectStoreConfig
|
||||
}
|
||||
|
||||
func main() {
|
||||
ctx := context.Background()
|
||||
|
||||
if len(os.Args) != 2 {
|
||||
fmt.Println("Error: expected one argument")
|
||||
fmt.Println("Usage:", os.Args[0], "<file_base>")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
name := os.Args[1]
|
||||
|
||||
slog.Info("arguments", "file_base", name)
|
||||
|
||||
cfg := &Config{}
|
||||
profile := aws.Profile(os.Getenv("AWS_PROFILE"))
|
||||
if profile == "" {
|
||||
slog.Error("profile not available")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
err := cfg.SetTextractClientWithProfile(ctx, profile)
|
||||
if err != nil {
|
||||
slog.Error("error creating client", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
root, err := os.OpenRoot("assets")
|
||||
if err != nil {
|
||||
slog.Error("error opening root", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
pdfFile, err := root.Open(fmt.Sprintf("original/%s.pdf", name))
|
||||
if err != nil {
|
||||
slog.Error("error opening pdf file", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
pdf, err := documenttypes.NewPDFFromReader(pdfFile)
|
||||
if err != nil {
|
||||
slog.Error("error creating pdf file", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
textSvc := documenttext.New(cfg)
|
||||
|
||||
startTime := time.Now()
|
||||
|
||||
text, err := textSvc.GetDocumentText(ctx, pdf)
|
||||
if err != nil {
|
||||
slog.Error("error getting page count", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
outputFilename := fmt.Sprintf("text/%s.txt", name)
|
||||
file, err := root.Create(outputFilename)
|
||||
if err != nil {
|
||||
slog.Error("error creating file", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
s, err := io.ReadAll(text)
|
||||
if err != nil {
|
||||
slog.Error("error reading text", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
_, err = file.Write(s)
|
||||
if err != nil {
|
||||
slog.Error("error writing file", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
slog.Info("extracted text", "output", outputFilename, "time", time.Since(startTime))
|
||||
}
|
||||
@@ -1,149 +0,0 @@
|
||||
//go:build aws
|
||||
|
||||
//go:generate go run -tags=aws ./textdetection.go
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
documenttext "queryorchestration/internal/document/text"
|
||||
documenttypes "queryorchestration/internal/document/types"
|
||||
"queryorchestration/internal/serviceconfig"
|
||||
"queryorchestration/internal/serviceconfig/aws"
|
||||
"queryorchestration/internal/serviceconfig/objectstore"
|
||||
"queryorchestration/internal/serviceconfig/textract"
|
||||
|
||||
awstextract "github.com/aws/aws-sdk-go-v2/service/textract"
|
||||
)
|
||||
|
||||
// Config provides the configuration for the text generator test tool.
|
||||
// Note: Query-related configuration has been removed. See remove_query_plan.md.
|
||||
type Config struct {
|
||||
serviceconfig.BaseConfig
|
||||
textract.TextractConfig
|
||||
objectstore.ObjectStoreConfig
|
||||
}
|
||||
|
||||
func main() {
|
||||
ctx := context.Background()
|
||||
|
||||
if len(os.Args) != 2 {
|
||||
fmt.Println("Error: expected one argument")
|
||||
fmt.Println("Usage:", os.Args[0], "<file_base>")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
name := os.Args[1]
|
||||
|
||||
slog.Info("arguments", "file_base", name)
|
||||
|
||||
cfg := &Config{}
|
||||
profile := aws.Profile(os.Getenv("AWS_PROFILE"))
|
||||
if profile == "" {
|
||||
slog.Error("profile not available")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
err := cfg.SetTextractClientWithProfile(ctx, profile)
|
||||
if err != nil {
|
||||
slog.Error("error creating client", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
root, err := os.OpenRoot("assets")
|
||||
if err != nil {
|
||||
slog.Error("error opening root", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
pdfFile, err := root.Open(fmt.Sprintf("original/%s.pdf", name))
|
||||
if err != nil {
|
||||
slog.Error("error opening pdf file", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
pdf, err := documenttypes.NewPDFFromReader(pdfFile)
|
||||
if err != nil {
|
||||
slog.Error("error creating pdf file", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
textSvc := documenttext.New(cfg)
|
||||
|
||||
count, err := pdf.GetPageCount(ctx)
|
||||
if err != nil {
|
||||
slog.Error("error getting page count", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
slog.Info("detecting text", "page_count", count)
|
||||
|
||||
startTime := time.Now()
|
||||
allResults := []map[string]*awstextract.AnalyzeDocumentOutput{}
|
||||
allResults = make([]map[string]*awstextract.AnalyzeDocumentOutput, count)
|
||||
var wg sync.WaitGroup
|
||||
for index := range count {
|
||||
i := index
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
slog.Info("getting page", "index", i)
|
||||
|
||||
baseResult, err := textSvc.GetBasePage(ctx, pdf, i)
|
||||
if err != nil {
|
||||
slog.Error("error detecting text", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
allResults[i] = map[string]*awstextract.AnalyzeDocumentOutput{
|
||||
"base": &baseResult,
|
||||
}
|
||||
|
||||
features, err := textSvc.ListRequiredFeatures(ctx, baseResult.Blocks, i)
|
||||
if err != nil {
|
||||
slog.Error("error listing features", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if len(features) > 0 {
|
||||
fullFeaturesResult, err := textSvc.GetPageWithFeatures(ctx, pdf, i, features)
|
||||
if err != nil {
|
||||
slog.Error("error getting features", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
allResults[i]["full_features"] = &fullFeaturesResult
|
||||
}
|
||||
|
||||
wg.Done()
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
jsonContent, err := json.MarshalIndent(allResults, "", "\t")
|
||||
if err != nil {
|
||||
slog.Error("error marshaling json", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
outputFilename := fmt.Sprintf("textract/%s.gen", name)
|
||||
file, err := root.Create(outputFilename)
|
||||
if err != nil {
|
||||
slog.Error("error creating file", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
_, err = file.Write(jsonContent)
|
||||
if err != nil {
|
||||
slog.Error("error writing file", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
slog.Info("extracted text", "output", outputFilename, "time", time.Since(startTime))
|
||||
}
|
||||
Reference in New Issue
Block a user