Merged in feature/textextraction (pull request #110)

Text Extraction

* bases

* go

* splitting

* structure

* movetoasync

* movetoasync

* settinguptrigger

* reorder

* storevent

* standardisepollingvalidation

* unittests

* fixlint

* fixlint

* awscfg

* generatesample

* followthrough

* tests

* clena

* store

* externalidcleanup

* clientid

* local

* baseunittests

* putobjecttests

* tests

cmd/clientSyncRunner/main.go

@@ -19,7 +19,7 @@ import (
 )
 
 type ClientSyncConfig struct {
-	runner.BaseConfig
+	runner.BaseConfig[clientsyncrunner.Body]
 	documentsync.DocSyncConfig
 }
 
@@ -28,7 +28,7 @@ func main() {
 
 	cfg := &ClientSyncConfig{}
 
-	cfg.ControllerFunc = func() runner.Controller {
+	cfg.ControllerFunc = func() runner.Controller[clientsyncrunner.Body] {
 		svc := clientsync.New(cfg)
 
 		c := clientsyncrunner.New(cfg.GetValidator(), &clientsyncrunner.Services{

cmd/docCleanRunner/main.go

@@ -16,14 +16,14 @@ import (
 	documentclean "queryorchestration/internal/document/clean"
 	"queryorchestration/internal/server/runner"
 	"queryorchestration/internal/serviceconfig/objectstore"
-	"queryorchestration/internal/serviceconfig/queue/documenttext"
+	"queryorchestration/internal/serviceconfig/queue/documenttexttrigger"
 
 	_ "github.com/lib/pq"
 )
 
 type DocCleanConfig struct {
-	runner.BaseConfig
-	documenttext.DocTextConfig
+	runner.BaseConfig[doccleanrunner.Body]
+	documenttexttrigger.DocTextTriggerConfig
 	objectstore.ObjectStoreConfig
 }
 
@@ -32,7 +32,7 @@ func main() {
 
 	cfg := &DocCleanConfig{}
 
-	cfg.ControllerFunc = func() runner.Controller {
+	cfg.ControllerFunc = func() runner.Controller[doccleanrunner.Body] {
 		clean := documentclean.New(cfg)
 
 		return doccleanrunner.New(cfg.GetValidator(), &doccleanrunner.Services{

cmd/docInitRunner/main.go

@@ -29,7 +29,7 @@ import (
 )
 
 type DocInitConfig struct {
-	runner.BaseConfig
+	runner.BaseConfig[docinitrunner.Body]
 	documentsyncc.DocSyncConfig
 }
 
@@ -38,7 +38,7 @@ func main() {
 
 	cfg := &DocInitConfig{}
 
-	cfg.ControllerFunc = func() runner.Controller {
+	cfg.ControllerFunc = func() runner.Controller[docinitrunner.Body] {
 		docinit := documentinit.New(cfg)
 
 		return docinitrunner.New(cfg.GetValidator(), &docinitrunner.Services{

cmd/docSyncRunner/main.go

@@ -25,7 +25,7 @@ import (
 )
 
 type DocSyncConfig struct {
-	runner.BaseConfig
+	runner.BaseConfig[docsyncrunner.Body]
 	documentcleanc.DocCleanConfig
 }
 
@@ -34,7 +34,7 @@ func main() {
 
 	cfg := &DocSyncConfig{}
 
-	cfg.ControllerFunc = func() runner.Controller {
+	cfg.ControllerFunc = func() runner.Controller[docsyncrunner.Body] {
 		cli := client.New(cfg)
 		doc := document.New(cfg)
 		docsync := documentsync.New(cfg, &documentsync.Services{

cmd/docTextProcessRunner/main.go

@@ -0,0 +1,66 @@
+// **Listens For**: Body Containing Textract Result Document Metadata
+//
+// **Action**.
+//
+// Process textract output.
+//
+// An event with the document id is pushed to the QUERYSYNC queue.
+package main
+
+import (
+	"context"
+	"log/slog"
+	"os"
+
+	doctextprocessrunner "queryorchestration/api/docTextProcessRunner"
+	documenttext "queryorchestration/internal/document/text"
+	"queryorchestration/internal/server/runner"
+	"queryorchestration/internal/serviceconfig/aws"
+	"queryorchestration/internal/serviceconfig/objectstore"
+	"queryorchestration/internal/serviceconfig/queue/querysync"
+	"queryorchestration/internal/serviceconfig/textract"
+
+	_ "github.com/lib/pq"
+)
+
+type DocTextProcessConfig struct {
+	runner.BaseConfig[doctextprocessrunner.Body]
+	aws.AWSConfig
+	textract.TextractConfig
+	querysync.QuerySyncConfig
+	objectstore.ObjectStoreConfig
+}
+
+func main() {
+	ctx := context.Background()
+
+	cfg := &DocTextProcessConfig{}
+
+	cfg.ControllerFunc = func() runner.Controller[doctextprocessrunner.Body] {
+		text := documenttext.New(cfg)
+
+		return doctextprocessrunner.New(cfg.GetValidator(), &doctextprocessrunner.Services{
+			Text: text,
+		})
+	}
+
+	server, err := runner.New(ctx, cfg)
+	if err != nil {
+		slog.Error(err.Error())
+		os.Exit(1)
+	}
+
+	err = cfg.SetStoreClient(ctx)
+	if err != nil {
+		slog.Error(err.Error())
+		os.Exit(1)
+	}
+
+	err = cfg.SetTextractClient(ctx)
+	if err != nil {
+		slog.Error(err.Error())
+		os.Exit(1)
+	}
+
+	server.Listen(ctx)
+}

cmd/docTextRunner/main.go

@@ -2,9 +2,9 @@
 //
 // **Action**.
 //
-// If the document text is not already extracted according to the collector standards, the text is extracted.
+// If the document text is not already extracted according to the collector standards, trigger the textract job for text detection.
 //
-// An event with the document id is pushed to the QUERYSYNC queue.
+// Otherwise, add an event to the QUERYSYNC queue.
 package main
 
 import (
@@ -15,24 +15,26 @@ import (
 	doctextrunner "queryorchestration/api/docTextRunner"
 	documenttext "queryorchestration/internal/document/text"
 	"queryorchestration/internal/server/runner"
+	"queryorchestration/internal/serviceconfig/objectstore"
 	"queryorchestration/internal/serviceconfig/queue/querysync"
 	"queryorchestration/internal/serviceconfig/textract"
 
 	_ "github.com/lib/pq"
 )
 
-type DocTextConfig struct {
-	runner.BaseConfig
-	querysync.QuerySyncConfig
+type DocTextTriggerConfig struct {
+	runner.BaseConfig[doctextrunner.Body]
 	textract.TextractConfig
+	querysync.QuerySyncConfig
+	objectstore.ObjectStoreConfig
 }
 
 func main() {
 	ctx := context.Background()
 
-	cfg := &DocTextConfig{}
+	cfg := &DocTextTriggerConfig{}
 
-	cfg.ControllerFunc = func() runner.Controller {
+	cfg.ControllerFunc = func() runner.Controller[doctextrunner.Body] {
 		text := documenttext.New(cfg)
 
 		return doctextrunner.New(cfg.GetValidator(), &doctextrunner.Services{
@@ -46,5 +48,11 @@ func main() {
 		os.Exit(1)
 	}
 
+	err = cfg.SetTextractClient(ctx)
+	if err != nil {
+		slog.Error(err.Error())
+		os.Exit(1)
+	}
+
 	server.Listen(ctx)
 }

cmd/queryRunner/main.go

@@ -24,7 +24,7 @@ import (
 )
 
 type QueryConfig struct {
-	runner.BaseConfig
+	runner.BaseConfig[queryrunner.Body]
 	queryc.QueryConfig
 }
 
@@ -33,7 +33,7 @@ func main() {
 
 	cfg := &QueryConfig{}
 
-	cfg.ControllerFunc = func() runner.Controller {
+	cfg.ControllerFunc = func() runner.Controller[queryrunner.Body] {
 		que := query.New(cfg)
 		res := result.New(cfg, &result.Services{
 			Query: que,

cmd/querySyncRunner/main.go

@@ -30,7 +30,7 @@ import (
 )
 
 type QuerySyncConfig struct {
-	runner.BaseConfig
+	runner.BaseConfig[querysyncrunner.Body]
 	queryc.QueryConfig
 }
 
@@ -39,7 +39,7 @@ func main() {
 
 	cfg := &QuerySyncConfig{}
 
-	cfg.ControllerFunc = func() runner.Controller {
+	cfg.ControllerFunc = func() runner.Controller[querysyncrunner.Body] {
 		sync := resultsync.New(cfg)
 		svc := querysync.New(cfg, &querysync.Services{
 			ResultSync: sync,

cmd/queryVersionSyncRunner/main.go

@@ -19,7 +19,7 @@ import (
 )
 
 type QueryVersionSyncConfig struct {
-	runner.BaseConfig
+	runner.BaseConfig[queryversionsyncrunner.Body]
 	clientsync.ClientSyncConfig
 }
 
@@ -28,7 +28,7 @@ func main() {
 
 	cfg := &QueryVersionSyncConfig{}
 
-	cfg.ControllerFunc = func() runner.Controller {
+	cfg.ControllerFunc = func() runner.Controller[queryversionsyncrunner.Body] {
 		svc := queryversionsync.New(cfg)
 
 		c := queryversionsyncrunner.New(cfg.GetValidator(), &queryversionsyncrunner.Services{

cmd/storeEventRunner/main.go

@@ -0,0 +1,50 @@
+// **Listens For**: S3 Event
+//
+// **Action**.
+//
+// It will identify the type of event and the location of said event, and forward the task accordingly.
+//
+// It supports the document initialisation and document text processing stages. Events may be pushed to the DOCINIT and DOCTEXTPROCESS queues.
+package main
+
+import (
+	"context"
+	"log/slog"
+	"os"
+
+	storeeventrunner "queryorchestration/api/storeEventRunner"
+	documentstore "queryorchestration/internal/document/store"
+	"queryorchestration/internal/server/runner"
+	"queryorchestration/internal/serviceconfig/queue/documentinit"
+	"queryorchestration/internal/serviceconfig/queue/documenttextprocess"
+
+	_ "github.com/lib/pq"
+)
+
+type StoreEventConfig struct {
+	runner.BaseConfig[storeeventrunner.S3EventNotification]
+	documentinit.DocInitConfig
+	documenttextprocess.DocTextProcessConfig
+}
+
+func main() {
+	ctx := context.Background()
+
+	cfg := &StoreEventConfig{}
+
+	cfg.ControllerFunc = func() runner.Controller[storeeventrunner.S3EventNotification] {
+		store := documentstore.New(cfg)
+
+		return storeeventrunner.New(cfg.GetValidator(), &storeeventrunner.Services{
+			Store: store,
+		})
+	}
+
+	server, err := runner.New(ctx, cfg)
+	if err != nil {
+		slog.Error(err.Error())
+		os.Exit(1)
+	}
+
+	server.Listen(ctx)
+}

cmd/textractGenerator_test/textdetection.go

@@ -0,0 +1,70 @@
+//go:build aws
+
+//go:generate go run -tags=aws ./textdetection.go
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"os"
+	"queryorchestration/internal/serviceconfig/aws"
+	"queryorchestration/internal/serviceconfig/textract"
+
+	awstextract "github.com/aws/aws-sdk-go-v2/service/textract"
+	"github.com/aws/aws-sdk-go-v2/service/textract/types"
+)
+
+func main() {
+	ctx := context.Background()
+
+	cfg := &textract.TextractConfig{}
+	profile := aws.Profile(os.Getenv("AWS_PROFILE"))
+	if profile == "" {
+		slog.Error("profile not available")
+		os.Exit(1)
+	}
+
+	err := cfg.SetTextractClientWithProfile(ctx, profile)
+	if err != nil {
+		slog.Error("error creating client", "error", err)
+		os.Exit(1)
+	}
+
+	base := "../../assets/sampleGeneration"
+	name := "helloWorld"
+	filename := fmt.Sprintf("%s/%s.pdf", base, name)
+	content, err := os.ReadFile(filename)
+	if err != nil {
+		slog.Error("error writing file", "error", err)
+		os.Exit(1)
+	}
+
+	slog.Info("detecting text", "filename", filename)
+
+	res, err := cfg.GetTextractClient().DetectDocumentText(ctx, &awstextract.DetectDocumentTextInput{
+		Document: &types.Document{
+			Bytes: content,
+		},
+	})
+	if err != nil {
+		slog.Error("error detecting text", "error", err)
+		os.Exit(1)
+	}
+
+	jsonContent, err := json.MarshalIndent(res, "", "\t")
+	if err != nil {
+		slog.Error("error marshaling json", "error", err)
+		os.Exit(1)
+	}
+
+	outputFilename := fmt.Sprintf("%s/generated/%s.gen", base, name)
+	err = os.WriteFile(outputFilename, jsonContent, 0600)
+	if err != nil {
+		slog.Error("error writing file", "error", err)
+		os.Exit(1)
+	}
+
+	fmt.Println("Successfully generated " + outputFilename)
+}