diff --git a/src/constants/investment_columns.py b/src/constants/investment_columns.py index dc4dffe..f56bf54 100644 --- a/src/constants/investment_columns.py +++ b/src/constants/investment_columns.py @@ -6,6 +6,7 @@ COLUMN_ORDER = [ "FILE_NAME", "CONTRACT_TITLE", "CONTRACT_AMENDMENT_NUM", + "FILENAME_AMENDMENT_NUM", "AARETE_DERIVED_AMENDMENT_NUM", "CLIENT_NAME", "PAYER_NAME", diff --git a/src/pipelines/shared/preprocessing/hybrid_smart_chunking_funcs.py b/src/pipelines/shared/preprocessing/hybrid_smart_chunking_funcs.py index 68dac05..df70d25 100644 --- a/src/pipelines/shared/preprocessing/hybrid_smart_chunking_funcs.py +++ b/src/pipelines/shared/preprocessing/hybrid_smart_chunking_funcs.py @@ -98,7 +98,6 @@ def process_single_field( elif isinstance(val, str): parsed[field.field_name] = val field_value = parsed[field.field_name] - if field_value == "N/A": field.field_type = "full_context" if field.field_name == "PAYER_NAME": @@ -341,6 +340,9 @@ def run_hybrid_smart_chunked_fields( gc.collect() logging.debug(f"RAG extraction done: {len(answers_dict)} fields") + # FILENAME_AMENDMENT_NUM + answers_dict = extract_amendment_num_from_filename(answers_dict, filename) + answers_dict = check_and_update_effective_date( answers_dict, text_dict, filename ) @@ -388,3 +390,42 @@ def run_hybrid_smart_chunked_fields( except Exception as e: logging.error(f"RAG pipeline failed: {e}") return {} + + +def extract_amendment_num_from_filename(answer_dict: dict, filename: str) -> dict: + """Extract amendment number from filename if not found in contract content. + + Args: + answer_dict (dict): Dictionary containing CONTRACT_AMENDMENT_NUM key. + filename (str): Filename to extract amendment number from. + + Returns: + dict: Updated answer_dict with FILENAME_AMENDMENT_NUM populated. + """ + if ( + string_utils.is_empty(answer_dict.get("CONTRACT_AMENDMENT_NUM")) + or answer_dict.get("CONTRACT_AMENDMENT_NUM") == "UNKNOWN" + ): + + prompt = prompt_templates.EXTRACT_AMENDMENT_NUM_FROM_FILENAME(filename) + llm_answer_raw = llm_utils.invoke_claude( + prompt, + "sonnet_latest", + filename, + cache=True, + instruction=prompt_templates.EXTRACT_AMENDMENT_NUM_FROM_FILENAME_INSTRUCTION(), + usage_label="EXTRACT_AMENDMENT_NUM_FROM_FILENAME", + ) + + try: + llm_answer_final = string_utils.universal_json_load(llm_answer_raw) + answer_dict["FILENAME_AMENDMENT_NUM"] = llm_answer_final.get( + "amendment_number", "N/A" + ) + except (ValueError, KeyError) as e: + logging.error( + f"Failed to parse amendment number from filename {filename}: {e}" + ) + answer_dict["FILENAME_AMENDMENT_NUM"] = "N/A" + + return answer_dict diff --git a/src/prompts/prompt_templates.py b/src/prompts/prompt_templates.py index 4ea3b42..7f088c3 100644 --- a/src/prompts/prompt_templates.py +++ b/src/prompts/prompt_templates.py @@ -1203,6 +1203,20 @@ def ONE_TO_ONE_MULTI_FIELD_INSTRUCTION() -> str: ) +def EXTRACT_AMENDMENT_NUM_FROM_FILENAME(filename) -> str: + return f"""Filename: {filename}""" + + +def EXTRACT_AMENDMENT_NUM_FROM_FILENAME_INSTRUCTION() -> str: + return """[OBJECTIVE]\n + Analyze the given filename string to extract the amendment number of the contract document if it exists. + Return only numeric values representing the amendment number. categorical values like 'A','B' etc are not amednment numbers. If no amendment number is found, return N/A. +Briefly explain your reasoning, then put your final answer in JSON dictionary format as follows: +{{ + "amendment_number": "" +}}""" + + ##################################################################################### ############################### PREPROCESSING PROMPTS ############################### #####################################################################################