Merged in bugfix/reimb_dates (pull request #649)

Bugfix/reimb dates to Main

* added reimb dates split

* Merged main into bugfix/reimb_dates

* prompt changes

* added to postprocessing

* fixed key issues

* modified split function

* removed print statements

* Merge branch 'main' into bugfix/reimb_dates

* update import

* moved split date upstream

* moved split dates to one to n funcs

* Merged main into bugfix/reimb_dates


Approved-by: Katon Minhas

fieldExtraction/src/investment/file_processing.py

@@ -242,6 +242,9 @@ def run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, constants)
         one_to_n_results
     )
 
+    ################### Split REIMB_DATES ##################
+    one_to_n_results = one_to_n_funcs.split_reimb_dates(one_to_n_results, filename)
+
     ################## CONVERT TO DF ##################
     one_to_n_df = pd.DataFrame(one_to_n_results)
 

fieldExtraction/src/investment/one_to_n_funcs.py

@@ -951,3 +951,70 @@ def rate_escalator_breakout(
     return (
         string_utils.universal_json_load(llm_response) or {}
     )  # Return empty dict if no response
+
+def split_reimb_dates(one_to_n_results: list, filename: str) -> list:
+    """Extract the date range from the field REIMB_DATES and split the dates into 
+    REIMB_EFFECTIVE_DT and REIMB_TERMINATION_DT by invoking LLM and split the dates based on the context."""
+    
+    if not one_to_n_results or not isinstance(one_to_n_results, list):
+        print("No valid data to process")
+        return one_to_n_results
+    
+    # Filter records that have non-empty REIMB_DATES
+    records_to_process = []
+    for i, record in enumerate(one_to_n_results):
+        # Initialize new fields with None for all records
+        record["REIMB_EFFECTIVE_DT"] = None
+        record["REIMB_TERMINATION_DT"] = None
+        
+        # Check if this record has valid REIMB_DATES
+        if ("REIMB_DATES" in record and 
+            record["REIMB_DATES"] is not None and 
+            str(record["REIMB_DATES"]).strip() != ""):
+            records_to_process.append((i, record))
+    
+    if not records_to_process:
+        print("No valid date ranges to process")
+        # Still remove the REIMB_DATES field from all records
+        for record in one_to_n_results:
+            if "REIMB_DATES" in record:
+                record.pop("REIMB_DATES", None)
+        return one_to_n_results
+    
+    # Process each record that needs processing
+    for index, record in records_to_process:
+        date_range = str(record["REIMB_DATES"]).strip()
+        
+        try:
+            # Invoke LLM to split the date range
+            llm_output_raw = llm_utils.invoke_claude(
+                investment_prompts.SPLIT_REIMB_DATES(date_range),
+                model_id="sonnet_latest",
+                filename=filename,
+                max_tokens=200
+            )
+            
+            # Extract the effective and termination dates from the LLM output
+            llm_output_cleaned = string_utils.universal_json_load(llm_output_raw)
+            
+            # Update the specific record with parsed dates
+            if llm_output_cleaned and isinstance(llm_output_cleaned, dict):
+                if "start_date" in llm_output_cleaned:
+                    record["REIMB_EFFECTIVE_DT"] = llm_output_cleaned["start_date"]
+                if "end_date" in llm_output_cleaned:
+                    record["REIMB_TERMINATION_DT"] = llm_output_cleaned["end_date"]
+            else:
+                print(f"Invalid LLM output format for index {index}: {llm_output_cleaned}")
+                
+        except Exception as e:
+            # Broader exception handling for LLM calls and JSON parsing
+            print(f"Error processing dates for index {index}: {str(e)}")
+            # Continue processing other records even if one fails
+            continue
+    
+    # Remove REIMB_DATES field from all records after splitting
+    for record in one_to_n_results:
+        if "REIMB_DATES" in record:
+            record.pop("REIMB_DATES", None)
+    
+    return one_to_n_results

fieldExtraction/src/investment/postprocessing_funcs.py

@@ -724,7 +724,5 @@ def deduplicate_provider_columns(df: pd.DataFrame) -> pd.DataFrame:
                             deduped_list.append(item)
 
                     # Update the DataFrame
-                    df.at[index, other_col] = (
-                        " | ".join(deduped_list) if deduped_list else ""
-                    )
-    return df
+                    df.at[index, other_col] = " | ".join(deduped_list) if deduped_list else ""
+    return df

fieldExtraction/src/prompts/investment_prompts.json

@@ -75,18 +75,11 @@
         "prompt": "What is the conversion factor (a specific dollar amount) mentioned in the text? A conversion factor is a dollar amount used to determine payment for healthcare services, typically under a fee-for-service or Diagnosis-Related Group (DRG) reimbursement model. It serves as a multiplier that converts a relative value (such as a DRG weight or Relative Value Unit — RVU) into an actual payment amount. If no conversion factor is mentioned, return 'N/A'. If it is mentioned as a dollar amount, return it as a numeric value only (e.g., 1.25 for $1.25)."
     },
     {
-        "field_name": "REIMB_EFFECTIVE_DT",
+        "field_name": "REIMB_DATES",
         "relationship": "one_to_n",
         "field_type": "dynamic_reimb_info",
-        "prompt": "Identify the effective date for reimbursement rates:\n- Look for dates associated with:\n  • 'Effective' or 'effective date'\n  • 'In effect' or 'in effect on'\n  • Date ranges with 'from/through' or 'start/end'\n- Dates may appear as:\n  • Full dates (12/1/17, December 1, 2017)\n  • Month and year (December 2017)\n  • Year only (2017). Do NOT include dates written in the footer of the page.",
-        "format": "date"
-    },
-    {
-        "field_name": "REIMB_TERMINATION_DT",
-        "relationship": "one_to_n",
-        "field_type": "dynamic_reimb_info",
-        "prompt": "Identify the termination date for reimbursement rates:\n- Look for dates associated with:\n  • 'Termination' or 'termination date'\n  • 'Expires' or 'expiration date'\n  • 'Through' or 'until'\n- Dates may appear as:\n  • Full dates (12/1/17, December 1, 2017)\n  • Month and year (December 2017)\n  • Year only (2017). Do NOT include dates written in the footer of the page.",
-        "format": "date"
+        "prompt": "Identify the effective and termination date range or duration language for which reimbursement rates apply:\n- Look for dates associated with:\n  • 'Effective' or 'effective date'\n  • 'In effect' or 'in effect on'\n  • 'Termination' or 'termination date'\n  • 'Expires' or 'expiration date'\n  • Date ranges with 'from/through', 'start/end', 'through', or 'until'\n- Also capture duration/term language when mentioned in context of reimbursement rates:\n  • Examples: 'Initial Term', 'Renewal Term', 'Extension Term', 'Base Term'\n  • 'First year', 'Second year', 'Annual', 'Monthly', 'Quarterly'\n  • 'Contract period', 'Agreement term', 'Service period'\n  • Any time-based descriptors related to rate applicability\n- Capture date ranges in format: 'start_date - end_date' or single dates\n- For duration language without specific dates, capture the duration term as-is\n- Dates may appear as:\n  • Full dates (12/1/17, December 1, 2017)\n  • Month and year (December 2017)\n  • Year only (2017)\n- Do NOT include dates written in the footer of the page\n- If only one date is found, capture it as a single date\n- If multiple separate date ranges exist, capture each as a separate entry\n- If no date range, date, duration, or term is found, return 'N/A'",
+        "format": "date_range"
     },
     {
         "field_name": "FEE_SCHEDULE",

fieldExtraction/src/prompts/investment_prompts.py

@@ -1183,3 +1183,49 @@ Final answers must be provided in JSON format:
    ...
 }}
 """
+
+def SPLIT_REIMB_DATES(date_range: str) -> str:
+    """
+    Splits a date range into start and end dates, returning them in YYYY/MM/DD format.
+    Handles both explicit dates and text-based duration language.
+    
+    Args:
+        date_range (str): The date range to split, e.g., "2023/01/01 - 2023/12/31" or "Initial Term - Renewal Term"
+        
+    Returns:
+        str: A JSON string with "start_date" and "end_date" keys
+    """
+    print(f"Splitting date range: {date_range}")
+    return f"""Extract the start and end dates from the following date range context: {date_range} 
+
+Based on the date range context:
+1. If explicit dates are present (e.g., "2023/01/01 - 2023/12/31"):
+   - Identify the effective date (start_date) and termination date (end_date)
+   - Convert to YYYY/MM/DD format
+
+2. If text-based terms are present (e.g., "Initial Term", "Renewal Term", "First Year"):
+   - Look for any embedded dates within the text
+   - Extract any recognizable date patterns (MM/DD/YYYY, Month DD YYYY, etc.)
+   - Convert extracted dates to YYYY/MM/DD format
+
+3. If mixed content (e.g., "Initial Term: 01/01/2023 - 12/31/2023"):
+   - Extract the date portion and ignore the term labels
+   - Convert dates to YYYY/MM/DD format
+
+4. If only month and year are available (e.g., "January 2023", "01/2023"):
+   - Use the first day of the month (e.g., "January 2023" becomes "2023/01/01")
+   - Apply this rule for both start and end dates when applicable
+
+5. If only one date is present:
+   - Assume it is the start_date and set end_date as "N/A"
+
+6. If no extractable dates are found in the text:
+   - Return "N/A" for both start_date and end_date
+
+Return the dates in the following JSON format:
+{{
+    "start_date": "YYYY/MM/DD",
+    "end_date": "YYYY/MM/DD"
+}}
+
+Note: Use "N/A" for dates that cannot be determined or extracted from the input text."""