From fe028eda88ed5eb36174d2042a40528597088a50 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Thu, 23 Oct 2025 19:57:11 +0000 Subject: [PATCH] Merged in hotfix/one-to-one-missing (pull request #744) Return one_to_one results if one_to_n is empty * Return one_to_one results if one_to_n is empty Approved-by: Sha Brown Approved-by: Karan Desai --- .../src/investment/file_processing.py | 24 +++++++++---------- fieldExtraction/src/investment/main.py | 4 ++-- fieldExtraction/src/investment/row_funcs.py | 1 + 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/fieldExtraction/src/investment/file_processing.py b/fieldExtraction/src/investment/file_processing.py index 0f5b971..6da69b8 100644 --- a/fieldExtraction/src/investment/file_processing.py +++ b/fieldExtraction/src/investment/file_processing.py @@ -29,6 +29,13 @@ def process_file(file_object, constants: Constants, run_timestamp): logging_utils.set_current_file(filename) logging.info(f"{datetime_str()} Processing {filename}...") + # Set default values + dynamic_one_to_one_fields = FieldSet() + process_one_to_n = config.FIELDS in ['all', 'one_to_n'] + process_one_to_one = config.FIELDS in ['all', 'one_to_one'] + # Initialize default fallback for final results + final_results = pd.DataFrame([{"FILE_NAME": filename}]) + ################## PREPROCESS ################## contract_text = preprocess.clean_text(contract_text) text_dict, top_sheet_dict = preprocess.split_text(contract_text) @@ -37,14 +44,6 @@ def process_file(file_object, constants: Constants, run_timestamp): text_dict, constants.EXHIBIT_HEADER_MARKERS, filename ) - # Set default values - dynamic_one_to_one_fields = FieldSet() - process_one_to_n = config.FIELDS in ['all', 'one_to_n'] - process_one_to_one = config.FIELDS in ['all', 'one_to_one'] - - # Initialize default fallback for final results - final_results = pd.DataFrame([{"FILE_NAME": filename}]) - # ONE TO N PROCESSING if process_one_to_n: exhibit_dict, all_exhibit_headers = preprocess.one_to_n_exhibit_chunking( @@ -58,8 +57,9 @@ def process_file(file_object, constants: Constants, run_timestamp): one_to_n_results, dynamic_one_to_one_fields = run_one_to_n_prompts( filename, exhibit_dict, all_exhibit_headers, constants ) - one_to_n_results["FILE_NAME"] = filename - one_to_n_results = postprocessing_funcs.generate_reimb_ids(one_to_n_results) + if not one_to_n_results.empty: + one_to_n_results["FILE_NAME"] = filename + one_to_n_results = postprocessing_funcs.generate_reimb_ids(one_to_n_results) logging.info(f"{datetime_str()} One to N Complete - {filename}") else: logging.info(f"{datetime_str()} No Reimbursement Found, Skipping - {filename}") @@ -82,7 +82,7 @@ def process_file(file_object, constants: Constants, run_timestamp): logging.info(f"{datetime_str()} One to One Complete - {filename}") # Decide how to handle one_to_one results - if process_one_to_n: + if not one_to_n_results.empty: # BOTH processed - merge into one_to_n final_results = row_funcs.merge_one_to_one_into_one_to_n( one_to_n_results, one_to_one_results, constants @@ -94,7 +94,7 @@ def process_file(file_object, constants: Constants, run_timestamp): logging.info(f"{datetime_str()} Fields not configured for One to One, Skipping - {filename}") # APPLY CODES IF ONE_TO_N WAS PROCESSED - if process_one_to_n: + if not one_to_n_results.empty: results_with_code = code_funcs.code_breakout(final_results, constants) final_results = code_funcs.grouper_breakout(results_with_code) logging.info(f"{datetime_str()} Codes Complete - {filename}") diff --git a/fieldExtraction/src/investment/main.py b/fieldExtraction/src/investment/main.py index 50627b5..02a2caa 100644 --- a/fieldExtraction/src/investment/main.py +++ b/fieldExtraction/src/investment/main.py @@ -100,7 +100,7 @@ def main(testing=False, test_params={}): # Warm prompt caches before parallel processing # This ensures the cache is registered before multiple threads try to use it - logging.debug("Warming prompt caches before parallel processing...") + logging.info("Warming prompt caches before parallel processing...") try: cacheable_instructions = prompt_templates.get_cacheable_instructions(constants) for cache_key, instruction in cacheable_instructions.items(): @@ -109,7 +109,7 @@ def main(testing=False, test_params={}): cache_key=cache_key, model_id="sonnet_latest" ) - logging.debug(f"Successfully warmed {len(cacheable_instructions)} prompt caches") + logging.info(f"Successfully warmed {len(cacheable_instructions)} prompt caches") except Exception as e: logging.warning(f"Error warming caches (will proceed anyway): {str(e)}") diff --git a/fieldExtraction/src/investment/row_funcs.py b/fieldExtraction/src/investment/row_funcs.py index 2838461..e726167 100644 --- a/fieldExtraction/src/investment/row_funcs.py +++ b/fieldExtraction/src/investment/row_funcs.py @@ -66,6 +66,7 @@ def merge_one_to_one_into_one_to_n( Returns: pd.DataFrame: Updated one_to_n_results with merged values from one_to_one_results. """ + # Iterate over all key-value pairs in one_to_one_results for k, v in one_to_one_results.items(): if k not in one_to_n_results.columns: