From c532aec3e9ba7d06955d46c6da5520870c093f66 Mon Sep 17 00:00:00 2001 From: Mayank Aamseek Date: Fri, 6 Dec 2024 15:29:30 +0000 Subject: [PATCH] Merged in hotfix/agreement_name (pull request #321) apply agreement name hotfix only on missing * apply agreement name hotfix only on missing Approved-by: Katon Minhas --- fieldExtraction/src/cnc_hotfix.py | 42 ++++++++++++++++++------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/fieldExtraction/src/cnc_hotfix.py b/fieldExtraction/src/cnc_hotfix.py index 393b287..91c6428 100644 --- a/fieldExtraction/src/cnc_hotfix.py +++ b/fieldExtraction/src/cnc_hotfix.py @@ -642,30 +642,38 @@ def clean_agreement_name(df: pd.DataFrame, If agreement name is not found, it runs on the context selected based on keywords. """ - contract_title_chunk_dict = dict(itertools.islice(text_dict.items(), 5)) - question = prompts.AC_DICT["CONTRACT_TITLE"] + df["Agreement Name (Contract Title)_corrected"] = df['Agreement Name (Contract Title)'] + agreement_df = df[(df['Agreement Name (Contract Title)'].isna())|(df['Agreement Name (Contract Title)'].astype(str).str.len() < 5)] + agreement_df['Filename'] = agreement_df['Contract Name'] + contract_list = agreement_df['Filename'].to_list() + contract_list = list(set(contract_list)) + + # process the contract + if contract_name in contract_list: + contract_title_chunk_dict = dict(itertools.islice(text_dict.items(), 5)) + question = prompts.AC_DICT["CONTRACT_TITLE"] - context = "\n".join( - [contract_title_chunk_dict[str(page_num)] for page_num in contract_title_chunk_dict.keys()] - ) - - prompt = prompts.AC_SINGLE_FIELD_TEMPLATE(context, question) - prompt_answer_raw = claude_funcs.invoke_claude( - prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192) - - if prompt_answer_raw == "N/A": - agreement_keywords = ["AGREEMENT", "AMENDMENT", "ADDENDUM", "PROVIDER", "CONTRACT", "MEMORANDUM", "LETTER", "REQUEST", "EFFECTIVE"] - page_list = ac_smart_chunking.chunk_hierarchical( - text_dict, agreement_keywords, False - ) context = "\n".join( - [text_dict[str(page_num)] for page_num in page_list] + [contract_title_chunk_dict[str(page_num)] for page_num in contract_title_chunk_dict.keys()] ) + prompt = prompts.AC_SINGLE_FIELD_TEMPLATE(context, question) prompt_answer_raw = claude_funcs.invoke_claude( prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192) - df.loc[df['Contract Name'] == contract_name, 'Agreement Name (Contract Title)_corrected'] = prompt_answer_raw + if prompt_answer_raw == "N/A": + agreement_keywords = ["AGREEMENT", "AMENDMENT", "ADDENDUM", "PROVIDER", "CONTRACT", "MEMORANDUM", "LETTER", "REQUEST", "EFFECTIVE"] + page_list = ac_smart_chunking.chunk_hierarchical( + text_dict, agreement_keywords, False + ) + context = "\n".join( + [text_dict[str(page_num)] for page_num in page_list] + ) + prompt = prompts.AC_SINGLE_FIELD_TEMPLATE(context, question) + prompt_answer_raw = claude_funcs.invoke_claude( + prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192) + + df.loc[df['Contract Name'] == contract_name, 'Agreement Name (Contract Title)_corrected'] = prompt_answer_raw return df