diff --git a/fieldExtraction/src/investment/file_processing.py b/fieldExtraction/src/investment/file_processing.py index f29613d..5d78ec8 100644 --- a/fieldExtraction/src/investment/file_processing.py +++ b/fieldExtraction/src/investment/file_processing.py @@ -242,6 +242,9 @@ def run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, constants) one_to_n_results ) + ################### Split REIMB_DATES ################## + one_to_n_results = one_to_n_funcs.split_reimb_dates(one_to_n_results, filename) + ################## CONVERT TO DF ################## one_to_n_df = pd.DataFrame(one_to_n_results) diff --git a/fieldExtraction/src/investment/one_to_n_funcs.py b/fieldExtraction/src/investment/one_to_n_funcs.py index 8d3fa4d..a85bc18 100644 --- a/fieldExtraction/src/investment/one_to_n_funcs.py +++ b/fieldExtraction/src/investment/one_to_n_funcs.py @@ -951,3 +951,70 @@ def rate_escalator_breakout( return ( string_utils.universal_json_load(llm_response) or {} ) # Return empty dict if no response + +def split_reimb_dates(one_to_n_results: list, filename: str) -> list: + """Extract the date range from the field REIMB_DATES and split the dates into + REIMB_EFFECTIVE_DT and REIMB_TERMINATION_DT by invoking LLM and split the dates based on the context.""" + + if not one_to_n_results or not isinstance(one_to_n_results, list): + print("No valid data to process") + return one_to_n_results + + # Filter records that have non-empty REIMB_DATES + records_to_process = [] + for i, record in enumerate(one_to_n_results): + # Initialize new fields with None for all records + record["REIMB_EFFECTIVE_DT"] = None + record["REIMB_TERMINATION_DT"] = None + + # Check if this record has valid REIMB_DATES + if ("REIMB_DATES" in record and + record["REIMB_DATES"] is not None and + str(record["REIMB_DATES"]).strip() != ""): + records_to_process.append((i, record)) + + if not records_to_process: + print("No valid date ranges to process") + # Still remove the REIMB_DATES field from all records + for record in one_to_n_results: + if "REIMB_DATES" in record: + record.pop("REIMB_DATES", None) + return one_to_n_results + + # Process each record that needs processing + for index, record in records_to_process: + date_range = str(record["REIMB_DATES"]).strip() + + try: + # Invoke LLM to split the date range + llm_output_raw = llm_utils.invoke_claude( + investment_prompts.SPLIT_REIMB_DATES(date_range), + model_id="sonnet_latest", + filename=filename, + max_tokens=200 + ) + + # Extract the effective and termination dates from the LLM output + llm_output_cleaned = string_utils.universal_json_load(llm_output_raw) + + # Update the specific record with parsed dates + if llm_output_cleaned and isinstance(llm_output_cleaned, dict): + if "start_date" in llm_output_cleaned: + record["REIMB_EFFECTIVE_DT"] = llm_output_cleaned["start_date"] + if "end_date" in llm_output_cleaned: + record["REIMB_TERMINATION_DT"] = llm_output_cleaned["end_date"] + else: + print(f"Invalid LLM output format for index {index}: {llm_output_cleaned}") + + except Exception as e: + # Broader exception handling for LLM calls and JSON parsing + print(f"Error processing dates for index {index}: {str(e)}") + # Continue processing other records even if one fails + continue + + # Remove REIMB_DATES field from all records after splitting + for record in one_to_n_results: + if "REIMB_DATES" in record: + record.pop("REIMB_DATES", None) + + return one_to_n_results \ No newline at end of file diff --git a/fieldExtraction/src/investment/postprocessing_funcs.py b/fieldExtraction/src/investment/postprocessing_funcs.py index 86d25b8..a85f32c 100644 --- a/fieldExtraction/src/investment/postprocessing_funcs.py +++ b/fieldExtraction/src/investment/postprocessing_funcs.py @@ -724,7 +724,5 @@ def deduplicate_provider_columns(df: pd.DataFrame) -> pd.DataFrame: deduped_list.append(item) # Update the DataFrame - df.at[index, other_col] = ( - " | ".join(deduped_list) if deduped_list else "" - ) - return df + df.at[index, other_col] = " | ".join(deduped_list) if deduped_list else "" + return df \ No newline at end of file diff --git a/fieldExtraction/src/prompts/investment_prompts.json b/fieldExtraction/src/prompts/investment_prompts.json index e10300f..18f347a 100644 --- a/fieldExtraction/src/prompts/investment_prompts.json +++ b/fieldExtraction/src/prompts/investment_prompts.json @@ -75,18 +75,11 @@ "prompt": "What is the conversion factor (a specific dollar amount) mentioned in the text? A conversion factor is a dollar amount used to determine payment for healthcare services, typically under a fee-for-service or Diagnosis-Related Group (DRG) reimbursement model. It serves as a multiplier that converts a relative value (such as a DRG weight or Relative Value Unit — RVU) into an actual payment amount. If no conversion factor is mentioned, return 'N/A'. If it is mentioned as a dollar amount, return it as a numeric value only (e.g., 1.25 for $1.25)." }, { - "field_name": "REIMB_EFFECTIVE_DT", + "field_name": "REIMB_DATES", "relationship": "one_to_n", "field_type": "dynamic_reimb_info", - "prompt": "Identify the effective date for reimbursement rates:\n- Look for dates associated with:\n • 'Effective' or 'effective date'\n • 'In effect' or 'in effect on'\n • Date ranges with 'from/through' or 'start/end'\n- Dates may appear as:\n • Full dates (12/1/17, December 1, 2017)\n • Month and year (December 2017)\n • Year only (2017). Do NOT include dates written in the footer of the page.", - "format": "date" - }, - { - "field_name": "REIMB_TERMINATION_DT", - "relationship": "one_to_n", - "field_type": "dynamic_reimb_info", - "prompt": "Identify the termination date for reimbursement rates:\n- Look for dates associated with:\n • 'Termination' or 'termination date'\n • 'Expires' or 'expiration date'\n • 'Through' or 'until'\n- Dates may appear as:\n • Full dates (12/1/17, December 1, 2017)\n • Month and year (December 2017)\n • Year only (2017). Do NOT include dates written in the footer of the page.", - "format": "date" + "prompt": "Identify the effective and termination date range or duration language for which reimbursement rates apply:\n- Look for dates associated with:\n • 'Effective' or 'effective date'\n • 'In effect' or 'in effect on'\n • 'Termination' or 'termination date'\n • 'Expires' or 'expiration date'\n • Date ranges with 'from/through', 'start/end', 'through', or 'until'\n- Also capture duration/term language when mentioned in context of reimbursement rates:\n • Examples: 'Initial Term', 'Renewal Term', 'Extension Term', 'Base Term'\n • 'First year', 'Second year', 'Annual', 'Monthly', 'Quarterly'\n • 'Contract period', 'Agreement term', 'Service period'\n • Any time-based descriptors related to rate applicability\n- Capture date ranges in format: 'start_date - end_date' or single dates\n- For duration language without specific dates, capture the duration term as-is\n- Dates may appear as:\n • Full dates (12/1/17, December 1, 2017)\n • Month and year (December 2017)\n • Year only (2017)\n- Do NOT include dates written in the footer of the page\n- If only one date is found, capture it as a single date\n- If multiple separate date ranges exist, capture each as a separate entry\n- If no date range, date, duration, or term is found, return 'N/A'", + "format": "date_range" }, { "field_name": "FEE_SCHEDULE", diff --git a/fieldExtraction/src/prompts/investment_prompts.py b/fieldExtraction/src/prompts/investment_prompts.py index ecebfdc..03add1a 100644 --- a/fieldExtraction/src/prompts/investment_prompts.py +++ b/fieldExtraction/src/prompts/investment_prompts.py @@ -1183,3 +1183,49 @@ Final answers must be provided in JSON format: ... }} """ + +def SPLIT_REIMB_DATES(date_range: str) -> str: + """ + Splits a date range into start and end dates, returning them in YYYY/MM/DD format. + Handles both explicit dates and text-based duration language. + + Args: + date_range (str): The date range to split, e.g., "2023/01/01 - 2023/12/31" or "Initial Term - Renewal Term" + + Returns: + str: A JSON string with "start_date" and "end_date" keys + """ + print(f"Splitting date range: {date_range}") + return f"""Extract the start and end dates from the following date range context: {date_range} + +Based on the date range context: +1. If explicit dates are present (e.g., "2023/01/01 - 2023/12/31"): + - Identify the effective date (start_date) and termination date (end_date) + - Convert to YYYY/MM/DD format + +2. If text-based terms are present (e.g., "Initial Term", "Renewal Term", "First Year"): + - Look for any embedded dates within the text + - Extract any recognizable date patterns (MM/DD/YYYY, Month DD YYYY, etc.) + - Convert extracted dates to YYYY/MM/DD format + +3. If mixed content (e.g., "Initial Term: 01/01/2023 - 12/31/2023"): + - Extract the date portion and ignore the term labels + - Convert dates to YYYY/MM/DD format + +4. If only month and year are available (e.g., "January 2023", "01/2023"): + - Use the first day of the month (e.g., "January 2023" becomes "2023/01/01") + - Apply this rule for both start and end dates when applicable + +5. If only one date is present: + - Assume it is the start_date and set end_date as "N/A" + +6. If no extractable dates are found in the text: + - Return "N/A" for both start_date and end_date + +Return the dates in the following JSON format: +{{ + "start_date": "YYYY/MM/DD", + "end_date": "YYYY/MM/DD" +}} + +Note: Use "N/A" for dates that cannot be determined or extracted from the input text."""