Merged in feature/checkbox (pull request #145)

Checkbox Primary Edge Cases * hascheckboxlogic * gen * preppinggen * exploringcheckbox * removeunselected * tests * failingtest * failintests * nomockqueryapi * db * name * nocontainer * deleterow * cnc * extradocsandupdatetextracts * moretables * appndedtables * baseversion
2025-05-20 12:47:22 +00:00
parent beb221937b
commit 61d12bf8df
69 changed files with 223611 additions and 78482 deletions
@@ -2,6 +2,7 @@ package documenttext

 import (
 	"context"
+	"errors"
 	"fmt"
 	"log/slog"
 	"strings"
@@ -19,102 +20,126 @@ func (s *Service) getPageText(ctx context.Context, document documenttypes.File,
 		return "", err
 	}

-	var pageBlock *types.Block
-	for _, block := range res.Blocks {
-		if block.BlockType == types.BlockTypePage {
-			pageBlock = &block
-			break
-		}
-	}
-	if pageBlock == nil {
-		return "", fmt.Errorf("no page block found for index: %d", index)
-	}
-
-	blockMap := s.getBlockMap(res.Blocks)
-
-	lines, specialFeatures, err := s.getPageElements(ctx, document, index, *pageBlock, blockMap)
+	features, err := s.ListRequiredFeatures(ctx, res.Blocks, index)
 	if err != nil {
 		return "", err
 	}

-	return s.buildPageText(ctx, lines, blockMap, index, specialFeatures)
+	blocks := res.Blocks
+	if len(features) > 0 {
+		res, err := s.GetPageWithFeatures(ctx, document, index, features)
+		if err != nil {
+			return "", err
+		}
+
+		blocks = res.Blocks
+	}
+
+	elements, err := s.getPageElements(ctx, document, index, blocks)
+	if err != nil {
+		return "", err
+	}
+
+	return s.buildPageText(ctx, elements, index)
 }

-func (s *Service) getPageElements(ctx context.Context, document documenttypes.File, index int, pageBlock types.Block, blockMap map[uuid.UUID]types.Block) ([]uuid.UUID, specialisedFeatures, error) {
-	directChildren := s.getDirectChildren(pageBlock, blockMap)
+type PageElements struct {
+	Page           uuid.UUID
+	BlockMap       map[uuid.UUID]types.Block
+	DirectChildren []uuid.UUID
+	Tables         []uuid.UUID
+}

-	hasTable := false
-	for _, id := range directChildren {
-		if blockMap[id].BlockType == types.BlockTypeLayoutTable {
-			hasTable = true
+func (s *Service) getPageElements(ctx context.Context, document documenttypes.File, index int, blocks []types.Block) (PageElements, error) {
+	elements := PageElements{}
+
+	pageBlock, err := s.GetPageBlock(blocks)
+	if err != nil {
+		return PageElements{}, err
+	}
+
+	elements.Page = pageBlock
+	elements.BlockMap = s.GetBlockMap(blocks)
+	elements.DirectChildren = s.GetDirectChildren(pageBlock, elements.BlockMap)
+
+	for _, block := range blocks {
+		if block.BlockType != types.BlockTypeTable {
+			continue
+		}
+		id, err := uuid.Parse(*block.Id)
+		if err != nil {
+			return PageElements{}, err
+		}
+
+		elements.Tables = append(elements.Tables, id)
+	}
+
+	return elements, nil
+}
+
+func (s *Service) ListRequiredFeatures(ctx context.Context, blocks []types.Block, index int) ([]types.FeatureType, error) {
+	features := []types.FeatureType{}
+	for _, block := range blocks {
+		switch block.BlockType {
+		case types.BlockTypeLayoutTable:
+			features = append(features, types.FeatureTypeTables)
+		case types.BlockTypeLayoutKeyValue:
+			features = append(features, types.FeatureTypeForms)
 		}
 	}

-	features := []types.FeatureType{}
-	if hasTable {
-		features = append(features, types.FeatureTypeTables)
-		// features = append(features, types.FeatureTypeForms)
+	if len(features) > 0 {
+		features = append(features, types.FeatureTypeLayout)
+		features = append(features, types.FeatureTypeSignatures)
 	}

-	// Detecting Checkboxes -
-	// if has tables
-	// For a page with no tables, we look for 3 out of 4 of the following conditions
-	//     has_exhibit→ 'exhibit' is on the page
-	//     has_benefit→ one of ['benefit plan', 'benefit plan(s)', 'applicable benefit', 'benefits'] is on the page
-	//     has_plans→ one of ['plan(s):', 'plan(s)/program(s):', 'plans:', 'plan/program:', 'plan(s)', 'plans'] is on the page
-	//     has_services → one of ['services:', 'service:', 'service(s):', 'services', 'service(s)'] is on the page
-
-	if len(features) == 0 {
-		return directChildren, specialisedFeatures{}, nil
-	}
-
-	featuresOut, err := s.getSpecialisedFeatures(ctx, document, index, features)
-	if err != nil {
-		return nil, specialisedFeatures{}, err
-	}
-
-	return directChildren, featuresOut, nil
+	return features, nil
 }

 type buildParams struct {
-	numOfSignatures int
-	currentTable    int
+	tableIndex int
 }

-func (s *Service) buildPageText(ctx context.Context, lines []uuid.UUID, blockMap map[uuid.UUID]types.Block, index int, specialFeatures specialisedFeatures) (string, error) {
+func (s *Service) buildPageText(ctx context.Context, elements PageElements, index int) (string, error) {
 	var builder strings.Builder
-	params := buildParams{
-		numOfSignatures: 0,
-		currentTable:    0,
-	}
+	params := buildParams{}

-	for _, id := range lines {
-		err := s.addComponentText(ctx, blockMap[id], blockMap, specialFeatures, &params, &builder)
+	for _, id := range elements.DirectChildren {
+		err := s.addComponentText(ctx, id, elements, &params, &builder)
 		if err != nil {
 			return "", err
 		}
 	}

-	for params.currentTable < len(specialFeatures.Tables) {
-		id := specialFeatures.Tables[params.currentTable]
-		err := s.addComponentText(ctx, specialFeatures.BlockMap[id], blockMap, specialFeatures, &params, &builder)
+	for params.tableIndex < len(elements.Tables) {
+		err := s.addTable(ctx, elements.Tables[params.tableIndex], elements.BlockMap, &builder)
 		if err != nil {
 			return "", err
 		}
+
+		params.tableIndex++
 	}

-	if params.numOfSignatures > 0 {
-		builder.WriteString(fmt.Sprintf("\nThis page has %d signature.\n", params.numOfSignatures))
+	numOfSignatures := 0
+	for _, block := range elements.BlockMap {
+		if block.BlockType == types.BlockTypeSignature {
+			numOfSignatures++
+		}
+	}
+
+	if numOfSignatures > 0 {
+		builder.WriteString(fmt.Sprintf("\nThis page has %d signature.\n", numOfSignatures))
 	}

 	return builder.String(), nil
 }

-func (s *Service) addComponentText(ctx context.Context, block types.Block, blockMap map[uuid.UUID]types.Block, specialFeatures specialisedFeatures, params *buildParams, builder *strings.Builder) error {
+func (s *Service) addComponentText(ctx context.Context, id uuid.UUID, elements PageElements, params *buildParams, builder *strings.Builder) error {
+	block := elements.BlockMap[id]
 	switch block.BlockType {
-	case types.BlockTypeLayoutText, types.BlockTypeLayoutTitle, types.BlockTypeLayoutHeader, types.BlockTypeLayoutFooter, types.BlockTypeLayoutSectionHeader, types.BlockTypeLayoutPageNumber, types.BlockTypeLayoutList, types.BlockTypeLayoutFigure, types.BlockTypeLayoutKeyValue:
-		for _, id := range s.getChildIds(block) {
-			err := s.addComponentText(ctx, blockMap[id], blockMap, specialFeatures, params, builder)
+	case types.BlockTypeLayoutText, types.BlockTypeLayoutTitle, types.BlockTypeLayoutHeader, types.BlockTypeLayoutFooter, types.BlockTypeLayoutSectionHeader, types.BlockTypeLayoutPageNumber, types.BlockTypeLayoutList, types.BlockTypeLayoutFigure:
+		for _, id := range s.getRelationshipIds(block) {
+			err := s.addComponentText(ctx, id, elements, params, builder)
 			if err != nil {
 				return err
 			}
@@ -131,23 +156,24 @@ func (s *Service) addComponentText(ctx context.Context, block types.Block, block
 			slog.Error("unable to write string", "error", err)
 			return err
 		}
-	case types.BlockTypeSignature:
-		params.numOfSignatures++
-	case types.BlockTypeLayoutTable, types.BlockTypeTable:
-		if block.BlockType == types.BlockTypeLayoutTable && len(specialFeatures.Tables) <= params.currentTable {
-			for _, child := range s.getChildIds(block) {
-				err := s.addComponentText(ctx, blockMap[child], blockMap, specialFeatures, params, builder)
+	case types.BlockTypeLayoutTable:
+		if params.tableIndex < len(elements.Tables) {
+			err := s.addTable(ctx, elements.Tables[params.tableIndex], elements.BlockMap, builder)
+			if err != nil {
+				return err
+			}
+		} else {
+			for _, child := range s.getRelationshipIds(block) {
+				err := s.addComponentText(ctx, child, elements, params, builder)
 				if err != nil {
 					return err
 				}
 			}
-			params.currentTable++
-			break
 		}
-		tableId := specialFeatures.Tables[params.currentTable]
-		params.currentTable++
-		tableBlock := specialFeatures.BlockMap[tableId]
-		err := s.addTable(ctx, tableBlock, specialFeatures.BlockMap, builder)
+
+		params.tableIndex++
+	case types.BlockTypeLayoutKeyValue:
+		err := s.addKeyValue(id, elements, builder)
 		if err != nil {
 			return err
 		}
@@ -159,6 +185,13 @@ func (s *Service) addComponentText(ctx context.Context, block types.Block, block
 }

 func (s *Service) GetBasePage(ctx context.Context, doc documenttypes.File, index int) (textract.AnalyzeDocumentOutput, error) {
+	return s.GetPageWithFeatures(ctx, doc, index, []types.FeatureType{
+		types.FeatureTypeLayout,
+		types.FeatureTypeSignatures,
+	})
+}
+
+func (s *Service) GetPageWithFeatures(ctx context.Context, doc documenttypes.File, index int, features []types.FeatureType) (textract.AnalyzeDocumentOutput, error) {
 	pageBytes, err := doc.GetPageAsPNG(ctx, index)
 	if err != nil {
 		return textract.AnalyzeDocumentOutput{}, err
@@ -168,10 +201,7 @@ func (s *Service) GetBasePage(ctx context.Context, doc documenttypes.File, index
 		Document: &types.Document{
 			Bytes: pageBytes,
 		},
-		FeatureTypes: []types.FeatureType{
-			types.FeatureTypeLayout,
-			types.FeatureTypeSignatures,
-		},
+		FeatureTypes: features,
 	})
 	if err != nil {
 		return textract.AnalyzeDocumentOutput{}, err
@@ -179,3 +209,18 @@ func (s *Service) GetBasePage(ctx context.Context, doc documenttypes.File, index

 	return *res, err
 }
+
+func (s *Service) GetPageBlock(blocks []types.Block) (uuid.UUID, error) {
+	var pageBlock *types.Block
+	for _, block := range blocks {
+		if block.BlockType == types.BlockTypePage {
+			pageBlock = &block
+			break
+		}
+	}
+	if pageBlock == nil {
+		return uuid.Nil, errors.New("no page block found")
+	}
+
+	return uuid.Parse(*pageBlock.Id)
+}