Merged in bugfix/group_chunk_json_parsing (pull request #271)

term group fixed

* term group fixed

* Merged main into bugfix/group_chunk_json_parsing

* file_processing.py edited online with Bitbucket
* Merged main into bugfix/group_chunk_json_parsing


Approved-by: Alex Galarce

fieldExtraction/src/file_processing.py

@@ -57,6 +57,7 @@ def run_ac_prompts(file_object):
     for field_group in field_groups:
         keyword_dict = keywords.GROUPED_KEYWORD_MAPPINGS[field_group]["keywords"]
         fields = keywords.GROUPED_KEYWORD_MAPPINGS[field_group]["fields"]
+        fields.sort()
         questions = {}
         for field in fields:
             questions[field] = prompts.AC_DICT[field]
@@ -93,7 +94,8 @@ def run_ac_prompts(file_object):
                 if len(fields) == 1:
                     field_group_answer_dict = {field: field_group_answer_raw}
                 elif len(fields) > 1:
-                    field_group_answer_dict = json.loads(field_group_answer_raw)
+                    field_group_answer_raw = field_group_answer_raw.replace("_DATE", "_DT")
+                    field_group_answer_dict = ac_funcs.json_parsing_search(field_group_answer_raw, fields)
 
                 # TODO: Validate field names coming out of Claude, check against fields in group, make sure they're all there, make sure there aren't any extras
                 for field in fields:

fieldExtraction/src/keywords.py

@@ -15,12 +15,16 @@ GROUPED_KEYWORD_MAPPINGS = {
         "fields": [
             "TERM_CLAUSE",
             "CONTRACT_AUTO_RENEWAL_IND",
+            "CONTRACT_TERMINATION_DT",
             "TERMINATION_UPON_NOTICE",
             "TERMINATION_UPON_CAUSE",
         ],
         "methodology": "or",
-        "keywords": ["Term", "Clause", "Renew", "upon notice", "With Cause"],
-        "case_sensitive": False,
+        "keywords": ["TERM AND TERMINATION", "Term and Termination", "Term and termination", "term and termination", "Term"
+                     , "Renew", "renew", "Auto renew", "auto renew", "automatically renew", "evergreen", "Evergreen"
+                     , "upon notice", "Notice", "NOTICE", "with notice", "without cause", "With Cause", "with cause"
+                     , "Termination", "Terminate"],
+        "case_sensitive": True,
     },
     # IRS group fields not smart chunked currently.
     # "irs_group": {

fieldExtraction/src/postprocessing_funcs.py

@@ -319,7 +319,7 @@ def clean_auto_renewal_ind(final_df):
         and "CONTRACT_TERMINATION_DT" in final_df
     ):
         final_df.loc[
-            final_df["CONTRACT_AUTO_RENEWAL_IND"] == "Yes", "CONTRACT_TERMINATION_DT"
+            final_df["CONTRACT_AUTO_RENEWAL_IND"].isin(["Yes", "Y"]), "CONTRACT_TERMINATION_DT"
         ] = np.nan
     return final_df
 

fieldExtraction/src/prompts.py

@@ -405,7 +405,7 @@ AC_DICT = {
     "HEALTH_PLAN_STATE": "Which state is this health plan for?",
     "SEQUESTRATION_LANGUAGE": "Is sequestration language present in the contract? It may be after reimbursement or compensation section. It is only present in case of medicare and medicare advantage. If Yes, extract the complete paragraph.",
     # "SEQUESTRATION_REDUCTIONS_IND":"Are sequestration and reduction percentage present in the contract? Answer Yes if both are present otherwise answer No.",
-    "CONTRACT_AUTO_RENEWAL_IND": "Does the contract automatically renew? Answer ONLY with Y or N. Do NOT write N/A.",
+    "CONTRACT_AUTO_RENEWAL_IND": "If this contract automatically renews, answer Y. If this contract will remain in effect unless terminated, answer Y. If it will terminate after the mentioned termination date, answer N.",
     "AMEND_CONTRACT_NOTICE_IND": 'Is amendment clause present in contract language?  The answer is usually found in section 8.7 but might not always be present there. It may be found in the section with title containing "Amendment". Answer may be found near keywords "Agreement may be amended". Answer ONLY with Y or N. Do NOT write N/A.. Here is an example of possible input text and the correct response. EXAMPLE INPUT TEXT: "may amend this Agreement by giving Provider written notice". EXAMPLE RESPONSE: "Y". If the agreement does not explicitly indicate that it can be amended by giving written notice, the answer will be "N".',
     "AFFILIATION_CLAUSE_IND": 'Is affiliation clause present in contract language? Answer may be found near keywords like "affiliate means a person or entity directly". Answer ONLY with Y or N. Do NOT write N/A.',
     # "TERM_CLAUSE": """Extract the subsection or paragraph(s) labeled Term. It may also be labeled Termination. In instances there may be 2 bodies of text that you will need to pull. For example, first paragraph may be 'MCO must follow the procedures outlines in Section...' and the second one may be 'Not later than thirty (30) days following receipt of the termination notice...'. In this case, ensure that YOU ALWAYS PULL the whole text of the two paragraphs. If there are no such sections, extract the paragraph that most closely resembles this label. If there is still no correct answer, simply return 'N/A'.""",