Merged in feature/global-lesser-of (pull request #553)

Feature/global lesser of

* Merge remote-tracking branch 'origin/feature/reimb-primary-updates'

* First try at global "lesser of"

* Generate reimb_ids further upstream

* Implement global lesser of injection and extraction logic in file processing

* Update logic to change LESSER_OF_IND in-place for global affected rows; add docstrings

* Refactor global lesser of injection logic to create new rows for REIMB_IDs lacking lesser of logic; update original rows in place and improve tracking flags.

* Merge remote-tracking branch 'origin/main' into feature/global-lesser-of

* Add logging for global lesser of logic injection and update handling of original rows

* Refactor global lesser of logic: move functions to one_to_one_funcs.py and update inject_global_lesser_of_rows to use new implementations

* Move imports for METHODOLOGY_BREAKOUT_QUESTIONS and REIMB_TERM_BREAKOUT to the top of one_to_one_funcs.py for better organization and to remove prototype comments.

* isort

* Refactor import statement for generate_reimb_ids to use the full module path for clarity

* load field from JSON instead of hardcoded values

* Update logging levels in investment_field_testing notebook

* Merged main into feature/global-lesser-of

* Remove unnecessary comment from investment_postprocessing_funcs import statement for clarity

* Merge branch 'feature/global-lesser-of' of https://bitbucket.org/aarete/doczy.ai into feature/global-lesser-of

* Merge remote-tracking branch 'origin/main' into feature/global-lesser-of

* Remove debug print statement from process_file function

* fix call to Field.load_from_file

* Merge remote-tracking branch 'origin/main' into feature/global-lesser-of

* Add tracking flag for global lesser of logic when none is found

* explicitly sort by FILE_NAME, EXHIBIT_PAGE, and REIMB_LESSER_OF_ID columns

* Add new field type, "full_context_separate", to make sure that we don't run it twice.  Also formatting fix

* Fix iteration over separate fields in run_full_context_fields function


Approved-by: Katon Minhas

fieldExtraction/scripts/investment_field_testing/testing.ipynb

@@ -20,6 +20,24 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "import logging\n",
+    "\n",
+    "logging.basicConfig(\n",
+    "    level=logging.DEBUG,\n",
+    "    format='%(asctime)s.%(msecs)03d - %(levelname)s - %(message)s',\n",
+    "    datefmt='%Y-%m-%d %H:%M:%S',\n",
+    "    force=True # Overwrite any previous logging configuration\n",
+    ")\n",
+    "\n",
+    "logging.getLogger('botocore').setLevel(logging.WARNING)\n",
+    "logging.getLogger('boto3').setLevel(logging.WARNING)\n",
+    "logging.getLogger('faiss').setLevel(logging.WARNING)\n",
+    "logging.getLogger('urllib3').setLevel(logging.WARNING)\n",
+    "logging.getLogger('sentence_transformers').setLevel(logging.WARNING)\n",
+    "logging.getLogger('s3transfer').setLevel(logging.WARNING)\n",
+    "logging.getLogger('chromadb').setLevel(logging.WARNING)\n",
+    "logging.getLogger('chroma').setLevel(logging.WARNING)\n",
+    "\n",
     "import os\n",
     "import pandas as pd"
    ]
@@ -45,7 +63,9 @@
     "test_params['local_input_dir'] = TEST_DATA_INPUT_DIR\n",
     "test_params['max_workers'] = 1\n",
     "test_params['input_files'] = [ # you can specify specific files to run here\n",
-    "    \"00-125434-Central MS Diagnostic, LLC-ICMProviderAgreement_78285.txt\",\n",
+    "    # \"00-125434-Central MS Diagnostic, LLC-ICMProviderAgreement_78285.txt\",\n",
+    "    \"02-0677066-Eyemasters-ICMProviderAgreement_64220_2.txt\",\n",
+    "    # \"CustomFac_AMD - AMD - UNIVERSITY MEDICAL CENTER OF EL PASO - 74-6000756 MU.txt\", # short contract for testing\n",
     "    # \"20-2401676-Community Hospital of LaGrange County, Inc dba Parkview LaGrange Hospital-ICMProviderAgreement_89356_1.txt\",\n",
     "                              ]"
    ]
@@ -67,20 +87,11 @@
    "source": [
     "df_out, df_error = investment_main(testing=True, test_params=test_params)"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_out"
-   ]
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "doczy-smart-chunking-U_35LsVn-py3.12",
+   "display_name": ".venv",
    "language": "python",
    "name": "python3"
   },

fieldExtraction/src/investment/file_processing.py

@@ -1,16 +1,15 @@
 import csv
 import json
-import os
-
 import logging
+import os
 
 import pandas as pd
 import src.investment.aarete_derived as aarete_derived
 import src.investment.dynamic_funcs as dynamic_funcs
+import src.investment.investment_postprocessing_funcs as investment_postprocessing_funcs
 import src.investment.one_to_n_funcs as one_to_n_funcs
 import src.investment.one_to_one_funcs as one_to_one_funcs
 import src.investment.postprocess as postprocess
-import src.investment.investment_postprocessing_funcs as investment_postprocessing_funcs
 import src.investment.preprocess as preprocess
 import src.investment.smart_chunking_funcs as smart_chunking_funcs
 import src.utils.io_utils as io_utils
@@ -47,8 +46,69 @@ def merge_one_to_one_into_one_to_n(one_to_n_results, one_to_one_results):
             one_to_n_results[k] = one_to_n_results[k].apply(
                 lambda row_val: v if string_utils.is_empty(row_val) else row_val
             )
+    
+    # Handle global lesser of injection
+    global_lesser_of = one_to_one_results.get("GLOBAL_LESSER_OF_STATEMENT")
+    if global_lesser_of and global_lesser_of not in ["N/A", "UNKNOWN"]:
+        logging.debug(f"[{one_to_one_results.get('FILE_NAME', 'UNKNOWN')}] Injecting global lesser of logic: {global_lesser_of}")
+        one_to_n_results = inject_global_lesser_of_rows(one_to_n_results, global_lesser_of, one_to_one_results.get("FILE_NAME", "UNKNOWN"))
+    else:
+        logging.debug(f"[{one_to_one_results.get('FILE_NAME', 'UNKNOWN')}] No global lesser of logic found or applicable.")
+        # Initialize the tracking flag when no global lesser of logic is found
+        one_to_n_results['GLOBAL_LESSER_OF_APPLIED'] = "N"
     return one_to_n_results
 
+def inject_global_lesser_of_rows(one_to_n_results: pd.DataFrame, global_lesser_of_stmt: str, filename: str) -> pd.DataFrame:
+    """Create new rows with global lesser of constraint for REIMB_IDs that lack lesser of logic.
+    
+    For each row where LESSER_OF_IND="N", creates a duplicate row with the global lesser
+    of statement applied via methodology breakout.  The original row is preserved with
+    LESSER_OF_IND updated to "Y" (since it's now subject to lesser of) and GLOBAL_LESSER_OF_APPLIED="N".
+    The new row has LESSER_OF_IND="Y" and GLOBAL_LESSER_OF_APPLIED="Y".  Both rows 
+    share the same REIMB_ID.
+
+    Args:
+        one_to_n_results (pd.DataFrame): DataFrame containing one-to-n results with a 'LESSER_OF_IND' column.
+        global_lesser_of_stmt (str): Global lesser of statement extracted from contract (e.g., "lesser of billed or allowable")
+        filename (str): Name of the file being processed, used for logging and debugging.
+
+    Returns:
+        pd.DataFrame: Updated one_to_n_results with new rows injected for global lesser of logic.
+    """
+    # Find rows that need global lesser of injection
+    rows_needing_injection = one_to_n_results[one_to_n_results['LESSER_OF_IND'] == "N"].copy()
+
+    if rows_needing_injection.empty:
+        logging.debug(f"No rows needing global lesser of injection for {filename}.")
+        # Still need to set tracking flag for all existing rows
+        one_to_n_results['GLOBAL_LESSER_OF_APPLIED'] = "N"
+        return one_to_n_results
+    
+    # Update original rows that lacked lesser of: they're now subject to lesser of due to
+    # global constraint
+    one_to_n_results.loc[one_to_n_results['LESSER_OF_IND'] == "N", 'LESSER_OF_IND'] = "Y"
+    # Set tracking flag for all existing rows (both original injected rows and rows that already had lesser of logic) to "N"
+    # New rows will get "Y" for this flag in create_global_lesser_of_row
+    one_to_n_results['GLOBAL_LESSER_OF_APPLIED'] = "N"
+
+    # Run methodology breakout on global statement (once)
+    global_methodology_breakout = one_to_one_funcs.run_global_lesser_of_breakout(global_lesser_of_stmt, filename)
+
+    # Create new rows for each REIMB_ID that was lacking lesser of logic
+    new_rows = []
+    for _, original_row in rows_needing_injection.iterrows():
+        new_row = one_to_one_funcs.create_global_lesser_of_row(original_row, global_methodology_breakout)
+        new_rows.append(new_row)
+    
+    # Append new rows to results
+    if new_rows:
+        new_rows_df = pd.DataFrame(new_rows)
+        one_to_n_results = pd.concat([one_to_n_results, new_rows_df], ignore_index=True)
+        logging.debug(f"Injected {len(new_rows)} global lesser of rows for {filename}.")
+    else:
+        logging.debug(f"No new rows created for global lesser of injection for {filename}.")
+    
+    return one_to_n_results
 
 def process_file(file_object, all_dataset, run_timestamp):
     filename, contract_text = file_object
@@ -65,6 +125,7 @@ def process_file(file_object, all_dataset, run_timestamp):
     if string_utils.contains_reimbursement(contract_text):
         one_to_n_results, dynamic_one_to_one_fields = run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, reimbursement_exhibits, all_dataset) # Return df
         one_to_n_results['FILE_NAME'] = filename
+        one_to_n_results = investment_postprocessing_funcs.generate_reimb_ids(one_to_n_results) # Add reimb_id
         print(f"{datetime_str()} One to N Complete - {filename}")
     else:
         one_to_n_results = pd.DataFrame([{'FILE_NAME' : filename}])

fieldExtraction/src/investment/one_to_one_funcs.py

@@ -1,10 +1,16 @@
-from rapidfuzz import fuzz
+import logging
 import os
+
+import pandas as pd
 import src.prompts.investment_prompts as prompts
 import src.utils.llm_utils as llm_utils
 import src.utils.string_utils as string_utils
+from rapidfuzz import fuzz
 from src import config
+from src import postprocessing_funcs as generic_postprocessing_funcs
 from src.enums.delimiters import Delimiter
+from src.investment import investment_postprocessing_funcs
+from src.investment.one_to_n_funcs import METHODOLOGY_BREAKOUT_QUESTIONS
 from src.investment.smart_chunking_funcs import (field_context, group_fields,
                                                  parse_chunk, stitch_chunks)
 from src.investment.tin_npi_funcs import (clean_provider_info,
@@ -12,10 +18,8 @@ from src.investment.tin_npi_funcs import (clean_provider_info,
                                           extract_identifiers,
                                           get_provider_info,
                                           identify_group_provider)
-from src.prompts.investment_prompts import Field, FieldSet
-from src.investment import investment_postprocessing_funcs
-from src import postprocessing_funcs as generic_postprocessing_funcs
 from src.investment.vision_funcs import get_image_array_based_answer
+from src.prompts.investment_prompts import REIMB_TERM_BREAKOUT, Field, FieldSet
 
 MAX_CONTEXT_LENGTH = 100000
 
@@ -85,6 +89,31 @@ def run_provider_info_fields(contract_text: str,
 
     return deduplicated_provider_info
 
+def extract_global_lesser_of(contract_text: str, filename:str) -> dict:
+    """Extract global "lesser of" statements that apply contract-wide.
+
+    Args:
+        contract_text (str): The full text of the contract to process.
+        filename (str): The name of the file being processed.
+
+    Returns:
+        dict: A dictionary containing the extracted "lesser of" statement.
+    """
+
+    global_lesser_of_field = Field.load_from_file(file_path=config.FIELD_JSON_PATH, field_name="GLOBAL_LESSER_OF_STATEMENT")
+    # Process using existing full context logic
+    prompt = prompts.ONE_TO_ONE_SINGLE_FIELD_TEMPLATE(
+        context=contract_text[0 : min(MAX_CONTEXT_LENGTH, len(contract_text) - 1)],
+        question=global_lesser_of_field.prompt,
+    )
+
+    logging.debug(f"Extracting global lesser of statement for {filename} with prompt: {prompt}")
+
+    response = llm_utils.invoke_claude(prompt, config.MODEL_ID_CLAUDE35_SONNET, filename)
+
+    logging.debug(f"Response for global lesser of statement extraction: {response}")
+    return {"GLOBAL_LESSER_OF_STATEMENT": string_utils.extract_text_from_delimiters(response, Delimiter.PIPE) or "N/A"}
+
 def filter_payer_from_providers(providers: list[dict], payer_name: str, threshold: int = 80) -> list[dict]:
     """Filters out providers whose names are too similar to the payer name.  Sometimes the LLM pulls the payer as one of the providers, and we want to drop those.
 
@@ -284,4 +313,79 @@ def run_full_context_fields(
         full_context_answers_dict = {}
         for field in full_context_fields.list_fields():
             full_context_answers_dict[field] = f"Error: {str(e)}"
+    
+    # Handle separate one-to-one fields that require individual processing
+    separate_fields = one_to_one_fields.filter(field_type="full_context_separate")
+
+    for field in separate_fields.fields:
+        if field.field_name == "GLOBAL_LESSER_OF_STATEMENT":
+            # Add global lesser of extraction
+            global_lesser_of_results = extract_global_lesser_of(contract_text, filename)
+            full_context_answers_dict.update(global_lesser_of_results)
+
     return full_context_answers_dict
+
+
+def create_global_lesser_of_row(original_row: pd.Series, global_methodology_breakout: dict[str, str]) -> pd.Series:
+    """Create a new row with global lesser of constraint applied to original reimbursement term.
+
+    Takes an original row that lacks lesser of logic and creates a duplicate with the global lesser of 
+    methodology breakout results applied.  Preserves the original REIMB_ID and SERVICE_TERM while updating
+    reimbursement methodology fields.
+
+    Args:
+        original_row (pd.Series): Original reimbursement row with LESSER_OF_IND="N"
+        global_methodology_breakout (dict[str, str]): Results from global lesser of methodology breakout,
+            containing fields like REIMB_FEE_RATE, REIMB_PCT_RATE, etc.
+
+    Returns:
+        pd.Series: New reimbursement row with global lesser of constraint applied.
+    """
+    new_row = original_row.copy()
+
+    # Update with global lesser of methodology breakout results.
+    for field, value in global_methodology_breakout.items():
+        new_row[field] = value
+
+    # Ensure lesser of indicator is set
+    new_row['LESSER_OF_IND'] = "Y"
+
+    # Add flag to track that this is a global lesser of row
+    new_row['GLOBAL_LESSER_OF_APPLIED'] = "Y"
+    return new_row
+
+
+def run_global_lesser_of_breakout(global_lesser_of_stmt: str, filename: str) -> dict[str, str]:
+    """Run methodology breakout on global lesser of statement to extract structured reimbursement details.
+
+    Uses the same REIMB_TERM_BREAKOUT prompt as the one used for individual service terms
+    to parse the global lesser of statement and extract fields like REIMB_FEE_RATE, REIMB_PCT_RATE, etc.
+
+    Args:
+        global_lesser_of_stmt (str): Global lesser of statement extracted from contract (e.g., "lesser of billed or allowable").
+        filename (str): Name of the file being processed, used for logging and debugging.
+    Returns:
+        dict[str, str]: Dictionary of methodology breakout fields (REIMB_PCT_RATE, 
+        REIMB_FEE_RATE, etc.) extracted from the global lesser of statement.
+        Returns an empty dict if parsing fails.
+    """
+
+    # Create a mock service term for the breakout
+    mock_service = "Global Lesser Of (all services)"
+
+    prompt = REIMB_TERM_BREAKOUT(mock_service, global_lesser_of_stmt, METHODOLOGY_BREAKOUT_QUESTIONS)
+    logging.debug(f"Running global lesser of breakout for {filename} with prompt: {prompt}")
+
+    llm_response = llm_utils.invoke_claude(prompt, config.MODEL_ID_CLAUDE35_SONNET, filename)
+
+    try:
+        breakout_results = string_utils.universal_json_load(llm_response)
+        result = breakout_results[0] if breakout_results else {}
+
+        # Add the global lesser of statement as the REIMB_TERM
+        result['REIMB_TERM'] = f"[GLOBAL LESSER OF] {global_lesser_of_stmt}"
+
+        return result
+    except:
+        logging.error(f"Failed to parse global lesser of breakout response for {filename}. Response: {llm_response}")
+        return {"REIMB_TERM": f"[GLOBAL LESSER OF] {global_lesser_of_stmt}"}

fieldExtraction/src/investment/postprocess.py

@@ -10,7 +10,17 @@ def postprocess(df):
     if df.shape[0] > 0:
         # Add postprocessing here
         df['CLIENT_NAME'] = config.CLIENT_NAME
-        
+
+        # Sort to ensure consistent ordering
+        sort_columns = ['FILE_NAME']
+        if 'EXHIBIT_PAGE' in df.columns:
+            sort_columns.append('EXHIBIT_PAGE')
+        if 'REIMB_LESSER_OF_ID' in df.columns:
+            sort_columns.append('REIMB_LESSER_OF_ID')
+
+        df = df.sort_values(by=sort_columns).reset_index(drop=True)
+
+        # Postprocess date fields
         df = investment_postprocessing_funcs.date_postprocess(df, config.FIELD_JSON_PATH)
 
         # Format rate fields
@@ -59,9 +69,6 @@ def postprocess(df):
         # Add AARETE_DERIVED_PRODUCT
         df = investment_postprocessing_funcs.add_aarete_derived_product(df)
 
-        # Add reimb_id
-        df = investment_postprocessing_funcs.generate_reimb_ids(df)
-
         # Standardize output column order - this should ALWAYS be the final postprocessing step
         df = investment_postprocessing_funcs.reorder_columns(df, COLUMN_ORDER)
         

fieldExtraction/src/prompts/investment_prompts.json

@@ -17,6 +17,12 @@
         "field_type": "methodology_breakout",
         "prompt": "If the methodology text explicitly describes reimbursement as being determined by selecting the smaller/lowest amount from two or more distinct reimbursement calculations or rate structures, write 'Y' for this field. \n\nThis includes language such as 'lesser of', 'lower of', 'minimum of', or equivalent comparative language that establishes multiple reimbursement pathways where the smallest amount is selected. \n\nWrite 'Y' only when there are multiple distinct reimbursement methodologies being compared (e.g., billed charges vs. fee schedule, different percentage calculations, brand vs. generic pricing). \n\nDo NOT write 'Y' for language that merely sets a maximum limit or cap on a single reimbursement method (such as 'not to exceed', 'up to', 'maximum of', or 'capped at'). \n\nOtherwise, write 'N'."
     },
+    {
+        "field_name": "GLOBAL_LESSER_OF_STATEMENT",
+        "relationship": "one_to_one",
+        "field_type": "full_context_separate",
+        "prompt": "Look for global reimbursement constraints that apply to all services in this contract.\nThese often appear as statements like:\n- 'Covered services will be reimbursed at the lesser of the billed amount or the allowed amount.'\n- 'All services subject to lesser of charges or X% of medicare'\n- 'Reimbursement shall not exceed the lesser of...'\n\nExtract the complete statement including any percentage or rate information, as well as any specific conditions that apply to the lesser of calculation.\nIf multiple global statements exist, separate them with a semicolon (;).\nIf no global lesser of statement is found, return 'N/A' or 'UNKNOWN'."
+    },
     {
         "field_name": "GREATER_OF_IND",
         "relationship": "one_to_n",