Merged in bugfix/DAIP2-1699-issue-tracker-fixes-one-to-one-dates-prov-info (pull request #881)

bugfix/DAIP2-1699-issue-tracker-fixes-one-to-one-dates-prov-info

* contract title fixes

* Add instruction to strip hyphens from TIN and NPI

* prompt update

* Merge branch 'bugfix/auto-renewal-term' into bugfix/DAIP2-1699-issue-tracker-fixes-one-to-one-dates-prov-info

* removed print statements

* pipeline error fixed

* signed_ind logic changed

* signatory_ind logic updated

* prompt update

* prompt update

* pipeline error fixed

* Merge branch 'DEV' into bugfix/DAIP2-1699-issue-tracker-fixes-one-to-one-dates-prov-info

* Merge branch 'DEV' into bugfix/DAIP2-1699-issue-tracker-fixes-one-to-one-dates-prov-info

* requested changes applied

* test case added

* prompt update

* contract title prompt update

* latest prompt added


Approved-by: Siddhant Medar

src/pipelines/saas/file_processing.py

@@ -230,6 +230,9 @@ def run_one_to_one_prompts(
                 )
             )
 
+    logging.debug(
+        f"Hybrid Smart Chunked Answers for {filename}: {hybrid_smart_chunked_answers_dict}"
+    )
     ################## RUN FULL CONTEXT PROMPTS ##################
     # Only run if there are fields to extract
     full_context_answers_dict = {}

src/pipelines/shared/postprocessing/postprocessing_funcs.py

@@ -998,7 +998,8 @@ def add_aarete_derived_signatory_complete_ind(df: pd.DataFrame) -> pd.DataFrame:
 
     Creates column:
         AARETE_DERIVED_SIGNATORY_COMPLETE_IND = 'Y' if
-            NUM_AVAILABLE_SIGNATORY_LINE_COUNT == NUM_SIGNED_SIGNATORY_LINE_COUNT
+            ((NUM_AVAILABLE_SIGNATORY_LINE_COUNT <= NUM_SIGNED_SIGNATORY_LINE_COUNT) and
+            (NUM_SIGNED_SIGNATORY_LINE_COUNT >= 1))
         else 'N'.
 
     Parameters
@@ -1029,10 +1030,13 @@ def add_aarete_derived_signatory_complete_ind(df: pd.DataFrame) -> pd.DataFrame:
 
         # Create indicator column
         df["AARETE_DERIVED_SIGNATORY_COMPLETE_IND"] = (
-            df["NUM_AVAILABLE_SIGNATORY_LINE_COUNT"]
-            .eq(df["NUM_SIGNED_SIGNATORY_LINE_COUNT"])
-            .map({True: "Y", False: "N"})
-        )
+            (
+                df["NUM_AVAILABLE_SIGNATORY_LINE_COUNT"].le(
+                    df["NUM_SIGNED_SIGNATORY_LINE_COUNT"]
+                )
+            )
+            & (df["NUM_SIGNED_SIGNATORY_LINE_COUNT"].ge(1))
+        ).map({True: "Y", False: "N"})
 
         return df
 

src/pipelines/shared/preprocessing/hybrid_smart_chunking_funcs.py

@@ -367,6 +367,8 @@ def prompt_hsc_single_field(
         llm_prompt = field.get_prompt_dict(
             constants
         )  # Returns dict of {field_name : prompt}
+        if field.field_name == "CONTRACT_TITLE":
+            context = text_dict["1"]
         prompt, _parser = prompt_templates.ONE_TO_ONE_SINGLE_FIELD_TEMPLATE(
             context, llm_prompt, field_name=field.field_name
         )
@@ -382,6 +384,7 @@ def prompt_hsc_single_field(
             llm_answer_raw = llm_utils.invoke_claude(
                 prompt, "sonnet_latest", filename, max_tokens=8192
             )
+        logging.debug(f"Raw LLM answer for {field.field_name}: {llm_answer_raw}")
         try:
             llm_answer_final = _parser(llm_answer_raw)  # returns dict
             field_value = llm_answer_final.get(field.field_name)

src/prompts/investment_prompts.json

@@ -342,7 +342,7 @@
         "field_name": "AUTO_RENEWAL_IND",
         "relationship": "one_to_one",
         "field_type": "smart_chunked",
-        "prompt": "This pertains to the term and termination section of the contract. If this contract automatically renews, answer Y. If this contract will remain in effect unless terminated, answer Y. Otherwise, answer N. If you cannot determine answer N.",
+        "prompt": "This pertains to the term and termination section of the contract. If this contract automatically renews (keyword \"automatically\" should be mentioned) , answer Y. Otherwise, answer N. If you cannot determine answer N.",
         "keywords": [
             "Renew",
             "renew",
@@ -360,7 +360,7 @@
         "field_name": "AUTO_RENEWAL_TERM",
         "relationship": "one_to_one",
         "field_type": "smart_chunked",
-        "prompt": "If the contract automatically renews, what is the length of the renewal term? This pertains to the term and termination section of the contract. The response should be a numerical representation of the renewal period with no punctuation. Acceptable formats are '1 year' or '12 months' (both representing the same duration). 'Year to year' and 'year' should be interpreted as '1 year'. If the contract automatically renews but the length of the renewal term is not specified, default to '1 year'. Only return the length of time for the renewal with no additional context.",
+        "prompt": "If the contract automatically renews, what is the length of the renewal term? This pertains to the term and termination section of the contract. The response should be a numerical representation of the renewal period with no punctuation. Acceptable formats are '1 year' or '12 months' (both representing the same duration). 'Year to year' and 'year' should be interpreted as '1 year'. Only return the length of time for the renewal with no additional context.",
         "keywords": [
             "Renew",
             "renew",

src/prompts/prompt_templates.py

@@ -1608,6 +1608,7 @@ Briefly explain your reasoning. After your explanation, include the heading "FIN
 - All provider objects must have these exact keys: "TIN", "NPI", "NAME"
 - ALl dict values must be strings - if there are multiple different TINs or NPIs for an identical NAME, put them in different dictionaries.
 - Use "N/A" for any missing values
+- Strip ALL non-digit characters from any found TIN or NPI (e.g., remove hyphens, spaces, parentheses).
 - A provider record is considered valid as long as either a TIN, NPI, or NAME is present. If any one or two of those fields are missing, use "N/A" for those fields.
 - Your final JSON array must begin with an opening bracket '[' and end with a closing bracket ']'
 - Make sure the JSON array is correctly formatted with commas between objects

src/tests/test_postprocess.py

@@ -10,6 +10,7 @@ from src.pipelines.shared.postprocessing.postprocessing_funcs import (
     clean_na_values,
     deduplicate_provider_columns,
     fill_claim_type_from_title,
+    add_aarete_derived_signatory_complete_ind,
     flatten_singleton_string_list,
     format_as_json_list,
     format_rate_fields_with_commas,
@@ -695,6 +696,20 @@ class TestPostprocessFunctions(unittest.TestCase):
         # Empty list should be treated as empty and filled with mode or inferred
         assert result_df.loc[0, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "M"
 
+    def test_add_aarete_derived_signatory_complete_ind(self):
+        """Test the derivation of the signatory complete indicator for fully signed, partially signed, and zero-signature scenarios."""
+        df = pd.DataFrame(
+            {
+                "NUM_AVAILABLE_SIGNATORY_LINE_COUNT": [2, 2, 0],
+                "NUM_SIGNED_SIGNATORY_LINE_COUNT": [2, 1, 0],
+            }
+        )
+        result = add_aarete_derived_signatory_complete_ind(df)
+
+        self.assertEqual(
+            result["AARETE_DERIVED_SIGNATORY_COMPLETE_IND"].tolist(), ["Y", "N", "N"]
+        )
+
     def test_clean_na_values_string_placeholders(self):
         """Test clean_na_values removes placeholder strings when they're the only value."""
         # Test with string placeholders