Merged in feature/postprocessing (pull request #114)

Feature/postprocessing

* tests

* passtest

* fixshorttests

* mosttests

* improvingbasedockerfile

* testspeeds

* testing

* host

* canparallel

* clean

* passfullsuite

* singlepagemax

* test

* findfeatures

* findstables

* tbls

* tablestoo

* tablestoo

* lateraltests

* tableloc

* cleanup

* inlinetable

* childids

* cleanup

* tests

cmd/textractGenerator_test/textdetection.go

@@ -9,17 +9,40 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
+	documenttext "queryorchestration/internal/document/text"
+	documenttypes "queryorchestration/internal/document/types"
+	"queryorchestration/internal/serviceconfig"
 	"queryorchestration/internal/serviceconfig/aws"
+	"queryorchestration/internal/serviceconfig/objectstore"
+	"queryorchestration/internal/serviceconfig/queue/querysync"
 	"queryorchestration/internal/serviceconfig/textract"
 
 	awstextract "github.com/aws/aws-sdk-go-v2/service/textract"
 	"github.com/aws/aws-sdk-go-v2/service/textract/types"
 )
 
+type Config struct {
+	serviceconfig.BaseConfig
+	textract.TextractConfig
+	querysync.QuerySyncConfig
+	objectstore.ObjectStoreConfig
+}
+
 func main() {
 	ctx := context.Background()
 
-	cfg := &textract.TextractConfig{}
+	if len(os.Args) != 3 {
+		fmt.Println("Error: expected at least two arguments")
+		fmt.Println("Usage:", os.Args[0], "<sample_name>", "<file_name>")
+		os.Exit(1)
+	}
+
+	sampleName := os.Args[1]
+	name := os.Args[2]
+
+	slog.Info("arguments", "sample_name", sampleName, "file_name", name)
+
+	cfg := &Config{}
 	profile := aws.Profile(os.Getenv("AWS_PROFILE"))
 	if profile == "" {
 		slog.Error("profile not available")
@@ -32,35 +55,105 @@ func main() {
 		os.Exit(1)
 	}
 
-	base := "../../assets/sampleGeneration"
-	name := "helloWorld"
-	filename := fmt.Sprintf("%s/%s.pdf", base, name)
-	content, err := os.ReadFile(filename)
+	root, err := os.OpenRoot("assets")
 	if err != nil {
-		slog.Error("error writing file", "error", err)
+		slog.Error("error opening root", "error", err)
 		os.Exit(1)
 	}
 
-	slog.Info("detecting text", "filename", filename)
-
-	res, err := cfg.GetTextractClient().DetectDocumentText(ctx, &awstextract.DetectDocumentTextInput{
-		Document: &types.Document{
-			Bytes: content,
-		},
-	})
+	root, err = root.OpenRoot(sampleName)
 	if err != nil {
-		slog.Error("error detecting text", "error", err)
+		slog.Error("error opening sample", "error", err)
 		os.Exit(1)
 	}
 
-	jsonContent, err := json.MarshalIndent(res, "", "\t")
+	pdfFile, err := root.Open(fmt.Sprintf("%s.pdf", name))
+	if err != nil {
+		slog.Error("error opening pdf file", "error", err)
+		os.Exit(1)
+	}
+
+	pdf, err := documenttypes.NewPDFFromReader(pdfFile)
+	if err != nil {
+		slog.Error("error creating pdf file", "error", err)
+		os.Exit(1)
+	}
+
+	textSvc := documenttext.New(cfg)
+
+	count, err := pdf.GetPageCount(ctx)
+	if err != nil {
+		slog.Error("error getting page count", "error", err)
+		os.Exit(1)
+	}
+
+	slog.Info("detecting text")
+
+	allResults := map[string][]awstextract.AnalyzeDocumentOutput{}
+	allResults["base"] = make([]awstextract.AnalyzeDocumentOutput, count)
+	allResults["table"] = []awstextract.AnalyzeDocumentOutput{}
+	for i := range count {
+		result, err := textSvc.GetBasePage(ctx, pdf, i)
+		if err != nil {
+			slog.Error("error detecting text", "error", err)
+			os.Exit(1)
+		}
+
+		for _, block := range result.Blocks {
+			feature := textSvc.GetBlockFeature(block)
+			if feature == documenttext.FeatureTable {
+				pageBytes, err := pdf.GetPageAsPNG(ctx, i)
+				if err != nil {
+					slog.Error("error getting page", "error", err)
+					os.Exit(1)
+				}
+
+				res, err := cfg.GetTextractClient().AnalyzeDocument(ctx, &awstextract.AnalyzeDocumentInput{
+					Document: &types.Document{
+						Bytes: pageBytes,
+					},
+					FeatureTypes: []types.FeatureType{
+						types.FeatureTypeTables,
+					},
+				})
+				if err != nil {
+					slog.Error("error detecting table", "error", err)
+					os.Exit(1)
+				}
+				allResults["table"] = append(allResults["table"], *res)
+			}
+		}
+
+		allResults["base"][i] = result
+	}
+
+	generatedBase := "generated"
+
+	if _, err := root.Stat(generatedBase); os.IsNotExist(err) {
+		err := root.Mkdir(generatedBase, 0755)
+		if err != nil {
+			slog.Error("error creating directory", "error", err)
+			os.Exit(1)
+		}
+
+		slog.Info("directory created", "directory", generatedBase)
+	}
+
+	jsonContent, err := json.MarshalIndent(allResults, "", "\t")
 	if err != nil {
 		slog.Error("error marshaling json", "error", err)
 		os.Exit(1)
 	}
 
-	outputFilename := fmt.Sprintf("%s/generated/%s.gen", base, name)
-	err = os.WriteFile(outputFilename, jsonContent, 0600)
+	outputFilename := fmt.Sprintf("%s/%s.gen", generatedBase, name)
+	file, err := root.Create(outputFilename)
+	if err != nil {
+		slog.Error("error creating file", "error", err)
+		os.Exit(1)
+	}
+	defer file.Close()
+
+	_, err = file.Write(jsonContent)
 	if err != nil {
 		slog.Error("error writing file", "error", err)
 		os.Exit(1)