jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/simplify-b-field-chunking (pull request #348)

Browse Source 
feature/simplify b field chunking

* Add chunk_by_exhibit function and unit tests

* Refactor preprocess

* Updated to run BU primary on entire Exhibit

* Makes BU secondary compatible with exhibit chunk mapping

* Added unit tests for get_exhibit_chunk

* Trimmed TD funcs

* Downstream changes to run_top_down

* Removed conditional funcs

* Merged main into feature/simplify-b-field-chunking

* removed conditional funcs import

* Merged main into feature/simplify-b-field-chunking

* Fixed run_bottom_up_primary arguments

* Fixed TD primary input

* Fixed grouped keyword mappings

* Remove filename tracking until postprocessing

* Removed postprocessing for client columns

* Removed col-based postprocessing

* Merged main into feature/simplify-b-field-chunking

* Fixed NoneType

* Add docstrings

* Updated unit tests for removal of 'Filename'

* Updated comments


Approved-by: Alex Galarce
This commit is contained in:
Katon Minhas
2025-01-10 17:24:44 +00:00
parent b2c4bef039
commit 3bb6abbcef
10 changed files with 278 additions and 391 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fieldExtraction/src/investment/file_processing.py
+13 -26
View File
@@ -3,7 +3,6 @@ import os
import pandas as pd
import src.investment.conditional_funcs as conditional_funcs
import src.investment.one_to_n_funcs as one_to_n_funcs
import src.investment.postprocess as postprocess
import src.investment.preprocess as preprocess
@@ -22,21 +21,18 @@ def run_ac_prompts(file_object):
print(f"Processing AC for {filename}...")
################## PREPROCESS - SMART CHUNK ##################
# Top Sheet Addition
text_dict, top_sheet_dict, exhibit_pages, num_pages, ac_chunks = (
preprocess.preprocess(contract_text, filename, fields="ac")
)
contract_text = preprocess.clean_text(contract_text)
text_dict, top_sheet_dict, num_pages = preprocess.split_text(contract_text)
ac_chunks = preprocess.one_to_one_smart_chunking(text_dict, contract_text)
print(f"AC Preprocessing Complete - {filename}")
################## RUN REGEX FUNCTIONALITY ##################
# no prompting for these fields
irs_answers = irs_hotfix(filename, text_dict, top_sheet_dict)
ac_answers_dict.update(irs_answers)
npi_answers = npi_hotfix(text_dict, top_sheet_dict)
ac_answers_dict.update(npi_answers)
################## RUN SMART CHUNKED PROMPTS ##################
ac_answers_dict, no_keyword_matches = (
smart_chunking_funcs.run_smart_chunk_prompts(
@@ -66,15 +62,9 @@ def run_ac_prompts(file_object):
os.makedirs(output_dir, exist_ok=True)
################## POSTPROCESS ##################
ac_df = pd.DataFrame([ac_answers_dict])
# print(f"Before ac_postprocess Unique values: {ac_df['CONTRACT_EFFECTIVE_DT'].unique()}")
ac_df = postprocess.ac_postprocess(ac_df, text_dict, filename, num_pages)
# print(f"After ac_postprocess Unique values: {ac_df['Contract Effective Date'].unique()}")
################## WRITE TO OUTPUT ##################
ac_df.to_csv(os.path.join(output_dir, config.AC_RESULTS_NAME), index=False)
@@ -105,36 +95,33 @@ def run_b_prompts(file_object):
print(f"Processing B for {filename}...")
################## PREPROCESS ##################
text_dict, top_sheet_dict, exhibit_pages, num_pages, ac_chunks = (
preprocess.preprocess(contract_text, filename, fields="b")
)
contract_text = preprocess.clean_text(contract_text)
text_dict, top_sheet_dict, num_pages = preprocess.split_text(contract_text)
exhibit_pages, exhibit_chunk_mapping = preprocess.one_to_n_exhibit_chunking(text_dict, filename)
text_dict = preprocess.clean_tables(text_dict, filename)
print(f"B Preprocessing Complete - {filename}")
################## RUN BOTTOM UP PROMPTS ##################
bu_results = one_to_n_funcs.run_bottom_up(
filename, text_dict
filename,
text_dict,
exhibit_chunk_mapping
) # Returns list of dictionaries
print(f"B Bottom Up Complete - {filename}")
################## RUN TOP DOWN PROMPTS ##################
combined_results = one_to_n_funcs.run_top_down(
filename, text_dict, bu_results, exhibit_pages
filename, text_dict, bu_results, exhibit_chunk_mapping
) # Returns list of dictionaries
print(f"B Top Down Complete - {filename}")
################## RUN CONDITIONAL PROMPTS ##################
conditional_results = conditional_funcs.run_conditional(
combined_results, text_dict, filename
) # List of dictionaries
print(f"B Conditional Prompts Complete - {filename}")
################## CREATE OUTPUT DIRECTORIES ##################
base_filename = os.path.splitext(filename)[0].strip()
output_dir = os.path.join(config.OUTPUT_DIRECTORY, base_filename)
os.makedirs(output_dir, exist_ok=True)
################## WRITE UNPROCESSED OUTPUT ##################
combined_df = pd.DataFrame(conditional_results)
combined_df = pd.DataFrame(combined_results)
combined_df.to_csv(
os.path.join(output_dir, config.UNPROCESSED_RESULTS_NAME), index=False
)