From df519ea32bb66d9ea80c72339dca1cf76db57058 Mon Sep 17 00:00:00 2001 From: Mayank Aamseek Date: Thu, 14 Nov 2024 21:41:07 +0000 Subject: [PATCH] Merged in bugfix/group_chunk_json_parsing (pull request #271) term group fixed * term group fixed * Merged main into bugfix/group_chunk_json_parsing * file_processing.py edited online with Bitbucket * Merged main into bugfix/group_chunk_json_parsing Approved-by: Alex Galarce --- fieldExtraction/src/file_processing.py | 4 +++- fieldExtraction/src/keywords.py | 8 ++++++-- fieldExtraction/src/postprocessing_funcs.py | 2 +- fieldExtraction/src/prompts.py | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/fieldExtraction/src/file_processing.py b/fieldExtraction/src/file_processing.py index cb7a0c3..8c5d7f5 100644 --- a/fieldExtraction/src/file_processing.py +++ b/fieldExtraction/src/file_processing.py @@ -57,6 +57,7 @@ def run_ac_prompts(file_object): for field_group in field_groups: keyword_dict = keywords.GROUPED_KEYWORD_MAPPINGS[field_group]["keywords"] fields = keywords.GROUPED_KEYWORD_MAPPINGS[field_group]["fields"] + fields.sort() questions = {} for field in fields: questions[field] = prompts.AC_DICT[field] @@ -93,7 +94,8 @@ def run_ac_prompts(file_object): if len(fields) == 1: field_group_answer_dict = {field: field_group_answer_raw} elif len(fields) > 1: - field_group_answer_dict = json.loads(field_group_answer_raw) + field_group_answer_raw = field_group_answer_raw.replace("_DATE", "_DT") + field_group_answer_dict = ac_funcs.json_parsing_search(field_group_answer_raw, fields) # TODO: Validate field names coming out of Claude, check against fields in group, make sure they're all there, make sure there aren't any extras for field in fields: diff --git a/fieldExtraction/src/keywords.py b/fieldExtraction/src/keywords.py index de08b50..1fadd5d 100644 --- a/fieldExtraction/src/keywords.py +++ b/fieldExtraction/src/keywords.py @@ -15,12 +15,16 @@ GROUPED_KEYWORD_MAPPINGS = { "fields": [ "TERM_CLAUSE", "CONTRACT_AUTO_RENEWAL_IND", + "CONTRACT_TERMINATION_DT", "TERMINATION_UPON_NOTICE", "TERMINATION_UPON_CAUSE", ], "methodology": "or", - "keywords": ["Term", "Clause", "Renew", "upon notice", "With Cause"], - "case_sensitive": False, + "keywords": ["TERM AND TERMINATION", "Term and Termination", "Term and termination", "term and termination", "Term" + , "Renew", "renew", "Auto renew", "auto renew", "automatically renew", "evergreen", "Evergreen" + , "upon notice", "Notice", "NOTICE", "with notice", "without cause", "With Cause", "with cause" + , "Termination", "Terminate"], + "case_sensitive": True, }, # IRS group fields not smart chunked currently. # "irs_group": { diff --git a/fieldExtraction/src/postprocessing_funcs.py b/fieldExtraction/src/postprocessing_funcs.py index 3cf8e74..f4ad2e3 100644 --- a/fieldExtraction/src/postprocessing_funcs.py +++ b/fieldExtraction/src/postprocessing_funcs.py @@ -319,7 +319,7 @@ def clean_auto_renewal_ind(final_df): and "CONTRACT_TERMINATION_DT" in final_df ): final_df.loc[ - final_df["CONTRACT_AUTO_RENEWAL_IND"] == "Yes", "CONTRACT_TERMINATION_DT" + final_df["CONTRACT_AUTO_RENEWAL_IND"].isin(["Yes", "Y"]), "CONTRACT_TERMINATION_DT" ] = np.nan return final_df diff --git a/fieldExtraction/src/prompts.py b/fieldExtraction/src/prompts.py index 7f8f090..886a60a 100644 --- a/fieldExtraction/src/prompts.py +++ b/fieldExtraction/src/prompts.py @@ -405,7 +405,7 @@ AC_DICT = { "HEALTH_PLAN_STATE": "Which state is this health plan for?", "SEQUESTRATION_LANGUAGE": "Is sequestration language present in the contract? It may be after reimbursement or compensation section. It is only present in case of medicare and medicare advantage. If Yes, extract the complete paragraph.", # "SEQUESTRATION_REDUCTIONS_IND":"Are sequestration and reduction percentage present in the contract? Answer Yes if both are present otherwise answer No.", - "CONTRACT_AUTO_RENEWAL_IND": "Does the contract automatically renew? Answer ONLY with Y or N. Do NOT write N/A.", + "CONTRACT_AUTO_RENEWAL_IND": "If this contract automatically renews, answer Y. If this contract will remain in effect unless terminated, answer Y. If it will terminate after the mentioned termination date, answer N.", "AMEND_CONTRACT_NOTICE_IND": 'Is amendment clause present in contract language? The answer is usually found in section 8.7 but might not always be present there. It may be found in the section with title containing "Amendment". Answer may be found near keywords "Agreement may be amended". Answer ONLY with Y or N. Do NOT write N/A.. Here is an example of possible input text and the correct response. EXAMPLE INPUT TEXT: "may amend this Agreement by giving Provider written notice". EXAMPLE RESPONSE: "Y". If the agreement does not explicitly indicate that it can be amended by giving written notice, the answer will be "N".', "AFFILIATION_CLAUSE_IND": 'Is affiliation clause present in contract language? Answer may be found near keywords like "affiliate means a person or entity directly". Answer ONLY with Y or N. Do NOT write N/A.', # "TERM_CLAUSE": """Extract the subsection or paragraph(s) labeled Term. It may also be labeled Termination. In instances there may be 2 bodies of text that you will need to pull. For example, first paragraph may be 'MCO must follow the procedures outlines in Section...' and the second one may be 'Not later than thirty (30) days following receipt of the termination notice...'. In this case, ensure that YOU ALWAYS PULL the whole text of the two paragraphs. If there are no such sections, extract the paragraph that most closely resembles this label. If there is still no correct answer, simply return 'N/A'.""",