jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in bugfix/inconsistent_table_sequences (pull request #523)

Browse Source 
Bugfix/inconsistent table sequences

* changes for inconsistent table sequence

* Added code comment block

* Merged main into bugfix/inconsistent_table_sequences

* Merged main into bugfix/inconsistent_table_sequences

* Merged main into bugfix/inconsistent_table_sequences

* Merged main into bugfix/inconsistent_table_sequences


Approved-by: Katon Minhas
This commit is contained in:
Alex Galarce
2025-05-08 17:18:47 +00:00
parent e51b599925
commit 147f91eaa9
1 changed files with 51 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
textract-pipeline/src/lambda/text-creation/index.py
+51 -13
View File
@@ -355,18 +355,43 @@ def concatenate_page_text_from_child_ids(
if textract_table_response is not None:
child_ids_list = []
table_text = get_table_text(textract_table_response, child_ids_list)
table_text, table_text_dataframe_dicts = get_table_text(textract_table_response, child_ids_list)
table_child_ids_list = []
table_child_ids_list = list_line_ids_from_table_blocks(
textract_table_response
)
lines_text = get_lines(textract_table_response, table_child_ids_list)
lines_text, lines_text_blocks = get_lines(textract_table_response, table_child_ids_list)
concatenated_text = f"""
{lines_text}
table_blocks = [item["table_block"] for item in table_text_dataframe_dicts]
combined_blocks = lines_text_blocks + table_blocks
{table_text}
"""
# Sort the combined list by 'Top' (vertical position) and 'Left' (horizontal position)
sorted_blocks = sorted(
combined_blocks,
key=lambda x: (
x.get("Geometry", {}).get("BoundingBox", {}).get("Top", 0),
x.get("Geometry", {}).get("BoundingBox", {}).get("Left", 0),
),
)
# All blocks merged together to create original sequence
concatenated_text = ""
for current_block in sorted_blocks:
if current_block ["BlockType"] == "LINE":
concatenated_text += (current_block["Text"] + "\n")
elif current_block ["BlockType"] == "TABLE":
for table_metadata in table_text_dataframe_dicts:
if table_metadata["table_id"] == current_block["Id"]:
dataframe_dicts_str = "\n".join(str(text) for text in table_metadata['table_text'])
concatenated_text += dataframe_dicts_str + "\n"
# concatenated_text = f"""
# {lines_text}
# {table_text}
# """
if final_number_of_signature == 0:
signature_page_word_list = ["IN WITNESS WHEREOF", "Signature"]
matched_count = strings_contained(signature_page_word_list, concatenated_text)
@@ -559,6 +584,7 @@ def get_lines(json_data, child_ids_list):
def extract_text(data):
text_lines = ""
text_lines_block = []
for block_id, block_data in data.items():
# TODO: UNDO this as the next line is commented for HN fac table troubleshoot
if block_id not in child_ids_list:
@@ -566,7 +592,8 @@ def get_lines(json_data, child_ids_list):
text_lines += (
block_data["Text"] + "\n"
) # Append text line and add newline character
return text_lines
text_lines_block.append(block_data)
return text_lines, text_lines_block
return extract_text(lines)
@@ -729,16 +756,27 @@ def get_table_text(json_data, child_ids_list):
] = merged_text
logger.info(f"Table content = {content}")
dataframe_dicts.append("-------Table Start--------")
dataframe_dicts.append(get_table_title_children_ids(table))
dataframe_dicts.append(content)
dataframe_dicts.append(get_table_footer_children_ids(table))
dataframe_dicts.append("-------Table End--------")
dataframe_dict_table_text = []
dataframe_dict_table_text.append("-------Table Start--------")
dataframe_dict_table_text.append(get_table_title_children_ids(table))
dataframe_dict_table_text.append(content)
dataframe_dict_table_text.append(get_table_footer_children_ids(table))
dataframe_dict_table_text.append("-------Table End--------")
table_metadata = {
"table_text": dataframe_dict_table_text,
"table_block": table,
"table_id": table["Id"]
}
dataframe_dicts.append(table_metadata)
dataframe_dicts_str = "\n".join(
str(dataframe_dict) for dataframe_dict in dataframe_dicts
)
return dataframe_dicts_str
return dataframe_dicts_str, dataframe_dicts
except Exception as e:
logger.error(f"Error in get_table_text: {e}")