jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/remove-exhibit-filtering (pull request #565)

Browse Source 
strip out reimbursement exhibit identification

* strip out reimbursement exhibit identification


Approved-by: Katon Minhas
This commit is contained in:
Alex Galarce
2025-06-09 21:35:20 +00:00
parent 0d3a110e26
commit 32d431ec4e
3 changed files with 6 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fieldExtraction/src/investment/file_processing.py
+4 -4
View File
@@ -118,12 +118,12 @@ def process_file(file_object, all_dataset, run_timestamp):
contract_text = preprocess.clean_text(contract_text)
text_dict, top_sheet_dict = preprocess.split_text(contract_text)
text_dict = preprocess.clean_tables(text_dict, filename)
exhibit_dict, all_exhibit_headers, reimbursement_exhibits = preprocess.one_to_n_exhibit_chunking(text_dict, filename)
exhibit_dict, all_exhibit_headers = preprocess.one_to_n_exhibit_chunking(text_dict, filename)
print(f"{datetime_str()} Preprocessing Complete - {filename}")
################## ONE TO N ##################
if string_utils.contains_reimbursement(contract_text):
one_to_n_results, dynamic_one_to_one_fields = run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, reimbursement_exhibits, all_dataset) # Return df
one_to_n_results, dynamic_one_to_one_fields = run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, all_dataset) # Return df
one_to_n_results['FILE_NAME'] = filename
one_to_n_results = investment_postprocessing_funcs.generate_reimb_ids(one_to_n_results) # Add reimb_id
print(f"{datetime_str()} One to N Complete - {filename}")
@@ -200,11 +200,11 @@ def run_one_to_one_prompts(filename, contract_text, text_dict, top_sheet_dict, d
return one_to_one_results[0]
def run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, reimbursement_exhibits, all_dataset):
def run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, all_dataset):
################## RUN PROMPTS ##################
one_to_n_results = []
for exhibit_page in reimbursement_exhibits:
for exhibit_page in exhibit_dict.keys():
exhibit_text = exhibit_dict[exhibit_page]
exhibit_header = all_exhibit_headers[exhibit_page]