Merged in feature/global-lesser-of (pull request #553)

Feature/global lesser of

* Merge remote-tracking branch 'origin/feature/reimb-primary-updates'

* First try at global "lesser of"

* Generate reimb_ids further upstream

* Implement global lesser of injection and extraction logic in file processing

* Update logic to change LESSER_OF_IND in-place for global affected rows; add docstrings

* Refactor global lesser of injection logic to create new rows for REIMB_IDs lacking lesser of logic; update original rows in place and improve tracking flags.

* Merge remote-tracking branch 'origin/main' into feature/global-lesser-of

* Add logging for global lesser of logic injection and update handling of original rows

* Refactor global lesser of logic: move functions to one_to_one_funcs.py and update inject_global_lesser_of_rows to use new implementations

* Move imports for METHODOLOGY_BREAKOUT_QUESTIONS and REIMB_TERM_BREAKOUT to the top of one_to_one_funcs.py for better organization and to remove prototype comments.

* isort

* Refactor import statement for generate_reimb_ids to use the full module path for clarity

* load field from JSON instead of hardcoded values

* Update logging levels in investment_field_testing notebook

* Merged main into feature/global-lesser-of

* Remove unnecessary comment from investment_postprocessing_funcs import statement for clarity

* Merge branch 'feature/global-lesser-of' of https://bitbucket.org/aarete/doczy.ai into feature/global-lesser-of

* Merge remote-tracking branch 'origin/main' into feature/global-lesser-of

* Remove debug print statement from process_file function

* fix call to Field.load_from_file

* Merge remote-tracking branch 'origin/main' into feature/global-lesser-of

* Add tracking flag for global lesser of logic when none is found

* explicitly sort by FILE_NAME, EXHIBIT_PAGE, and REIMB_LESSER_OF_ID columns

* Add new field type, "full_context_separate", to make sure that we don't run it twice.  Also formatting fix

* Fix iteration over separate fields in run_full_context_fields function


Approved-by: Katon Minhas

fieldExtraction/src/investment/file_processing.py

@@ -1,16 +1,15 @@
 import csv
 import json
-import os
-
 import logging
+import os
 
 import pandas as pd
 import src.investment.aarete_derived as aarete_derived
 import src.investment.dynamic_funcs as dynamic_funcs
+import src.investment.investment_postprocessing_funcs as investment_postprocessing_funcs
 import src.investment.one_to_n_funcs as one_to_n_funcs
 import src.investment.one_to_one_funcs as one_to_one_funcs
 import src.investment.postprocess as postprocess
-import src.investment.investment_postprocessing_funcs as investment_postprocessing_funcs
 import src.investment.preprocess as preprocess
 import src.investment.smart_chunking_funcs as smart_chunking_funcs
 import src.utils.io_utils as io_utils
@@ -47,8 +46,69 @@ def merge_one_to_one_into_one_to_n(one_to_n_results, one_to_one_results):
             one_to_n_results[k] = one_to_n_results[k].apply(
                 lambda row_val: v if string_utils.is_empty(row_val) else row_val
             )
+    
+    # Handle global lesser of injection
+    global_lesser_of = one_to_one_results.get("GLOBAL_LESSER_OF_STATEMENT")
+    if global_lesser_of and global_lesser_of not in ["N/A", "UNKNOWN"]:
+        logging.debug(f"[{one_to_one_results.get('FILE_NAME', 'UNKNOWN')}] Injecting global lesser of logic: {global_lesser_of}")
+        one_to_n_results = inject_global_lesser_of_rows(one_to_n_results, global_lesser_of, one_to_one_results.get("FILE_NAME", "UNKNOWN"))
+    else:
+        logging.debug(f"[{one_to_one_results.get('FILE_NAME', 'UNKNOWN')}] No global lesser of logic found or applicable.")
+        # Initialize the tracking flag when no global lesser of logic is found
+        one_to_n_results['GLOBAL_LESSER_OF_APPLIED'] = "N"
     return one_to_n_results
 
+def inject_global_lesser_of_rows(one_to_n_results: pd.DataFrame, global_lesser_of_stmt: str, filename: str) -> pd.DataFrame:
+    """Create new rows with global lesser of constraint for REIMB_IDs that lack lesser of logic.
+    
+    For each row where LESSER_OF_IND="N", creates a duplicate row with the global lesser
+    of statement applied via methodology breakout.  The original row is preserved with
+    LESSER_OF_IND updated to "Y" (since it's now subject to lesser of) and GLOBAL_LESSER_OF_APPLIED="N".
+    The new row has LESSER_OF_IND="Y" and GLOBAL_LESSER_OF_APPLIED="Y".  Both rows 
+    share the same REIMB_ID.
+
+    Args:
+        one_to_n_results (pd.DataFrame): DataFrame containing one-to-n results with a 'LESSER_OF_IND' column.
+        global_lesser_of_stmt (str): Global lesser of statement extracted from contract (e.g., "lesser of billed or allowable")
+        filename (str): Name of the file being processed, used for logging and debugging.
+
+    Returns:
+        pd.DataFrame: Updated one_to_n_results with new rows injected for global lesser of logic.
+    """
+    # Find rows that need global lesser of injection
+    rows_needing_injection = one_to_n_results[one_to_n_results['LESSER_OF_IND'] == "N"].copy()
+
+    if rows_needing_injection.empty:
+        logging.debug(f"No rows needing global lesser of injection for {filename}.")
+        # Still need to set tracking flag for all existing rows
+        one_to_n_results['GLOBAL_LESSER_OF_APPLIED'] = "N"
+        return one_to_n_results
+    
+    # Update original rows that lacked lesser of: they're now subject to lesser of due to
+    # global constraint
+    one_to_n_results.loc[one_to_n_results['LESSER_OF_IND'] == "N", 'LESSER_OF_IND'] = "Y"
+    # Set tracking flag for all existing rows (both original injected rows and rows that already had lesser of logic) to "N"
+    # New rows will get "Y" for this flag in create_global_lesser_of_row
+    one_to_n_results['GLOBAL_LESSER_OF_APPLIED'] = "N"
+
+    # Run methodology breakout on global statement (once)
+    global_methodology_breakout = one_to_one_funcs.run_global_lesser_of_breakout(global_lesser_of_stmt, filename)
+
+    # Create new rows for each REIMB_ID that was lacking lesser of logic
+    new_rows = []
+    for _, original_row in rows_needing_injection.iterrows():
+        new_row = one_to_one_funcs.create_global_lesser_of_row(original_row, global_methodology_breakout)
+        new_rows.append(new_row)
+    
+    # Append new rows to results
+    if new_rows:
+        new_rows_df = pd.DataFrame(new_rows)
+        one_to_n_results = pd.concat([one_to_n_results, new_rows_df], ignore_index=True)
+        logging.debug(f"Injected {len(new_rows)} global lesser of rows for {filename}.")
+    else:
+        logging.debug(f"No new rows created for global lesser of injection for {filename}.")
+    
+    return one_to_n_results
 
 def process_file(file_object, all_dataset, run_timestamp):
     filename, contract_text = file_object
@@ -65,6 +125,7 @@ def process_file(file_object, all_dataset, run_timestamp):
     if string_utils.contains_reimbursement(contract_text):
         one_to_n_results, dynamic_one_to_one_fields = run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, reimbursement_exhibits, all_dataset) # Return df
         one_to_n_results['FILE_NAME'] = filename
+        one_to_n_results = investment_postprocessing_funcs.generate_reimb_ids(one_to_n_results) # Add reimb_id
         print(f"{datetime_str()} One to N Complete - {filename}")
     else:
         one_to_n_results = pd.DataFrame([{'FILE_NAME' : filename}])