diff --git a/src/pipelines/saas/file_processing.py b/src/pipelines/saas/file_processing.py index c9b43cc..d1aa09e 100644 --- a/src/pipelines/saas/file_processing.py +++ b/src/pipelines/saas/file_processing.py @@ -230,6 +230,9 @@ def run_one_to_one_prompts( ) ) + logging.debug( + f"Hybrid Smart Chunked Answers for {filename}: {hybrid_smart_chunked_answers_dict}" + ) ################## RUN FULL CONTEXT PROMPTS ################## # Only run if there are fields to extract full_context_answers_dict = {} diff --git a/src/pipelines/shared/postprocessing/postprocessing_funcs.py b/src/pipelines/shared/postprocessing/postprocessing_funcs.py index 8ed7a61..b10cfc7 100644 --- a/src/pipelines/shared/postprocessing/postprocessing_funcs.py +++ b/src/pipelines/shared/postprocessing/postprocessing_funcs.py @@ -998,7 +998,8 @@ def add_aarete_derived_signatory_complete_ind(df: pd.DataFrame) -> pd.DataFrame: Creates column: AARETE_DERIVED_SIGNATORY_COMPLETE_IND = 'Y' if - NUM_AVAILABLE_SIGNATORY_LINE_COUNT == NUM_SIGNED_SIGNATORY_LINE_COUNT + ((NUM_AVAILABLE_SIGNATORY_LINE_COUNT <= NUM_SIGNED_SIGNATORY_LINE_COUNT) and + (NUM_SIGNED_SIGNATORY_LINE_COUNT >= 1)) else 'N'. Parameters @@ -1029,10 +1030,13 @@ def add_aarete_derived_signatory_complete_ind(df: pd.DataFrame) -> pd.DataFrame: # Create indicator column df["AARETE_DERIVED_SIGNATORY_COMPLETE_IND"] = ( - df["NUM_AVAILABLE_SIGNATORY_LINE_COUNT"] - .eq(df["NUM_SIGNED_SIGNATORY_LINE_COUNT"]) - .map({True: "Y", False: "N"}) - ) + ( + df["NUM_AVAILABLE_SIGNATORY_LINE_COUNT"].le( + df["NUM_SIGNED_SIGNATORY_LINE_COUNT"] + ) + ) + & (df["NUM_SIGNED_SIGNATORY_LINE_COUNT"].ge(1)) + ).map({True: "Y", False: "N"}) return df diff --git a/src/pipelines/shared/preprocessing/hybrid_smart_chunking_funcs.py b/src/pipelines/shared/preprocessing/hybrid_smart_chunking_funcs.py index 9a5bc9c..82b77e0 100644 --- a/src/pipelines/shared/preprocessing/hybrid_smart_chunking_funcs.py +++ b/src/pipelines/shared/preprocessing/hybrid_smart_chunking_funcs.py @@ -367,6 +367,8 @@ def prompt_hsc_single_field( llm_prompt = field.get_prompt_dict( constants ) # Returns dict of {field_name : prompt} + if field.field_name == "CONTRACT_TITLE": + context = text_dict["1"] prompt, _parser = prompt_templates.ONE_TO_ONE_SINGLE_FIELD_TEMPLATE( context, llm_prompt, field_name=field.field_name ) @@ -382,6 +384,7 @@ def prompt_hsc_single_field( llm_answer_raw = llm_utils.invoke_claude( prompt, "sonnet_latest", filename, max_tokens=8192 ) + logging.debug(f"Raw LLM answer for {field.field_name}: {llm_answer_raw}") try: llm_answer_final = _parser(llm_answer_raw) # returns dict field_value = llm_answer_final.get(field.field_name) diff --git a/src/prompts/investment_prompts.json b/src/prompts/investment_prompts.json index 30fdeb3..7034ad7 100644 --- a/src/prompts/investment_prompts.json +++ b/src/prompts/investment_prompts.json @@ -342,7 +342,7 @@ "field_name": "AUTO_RENEWAL_IND", "relationship": "one_to_one", "field_type": "smart_chunked", - "prompt": "This pertains to the term and termination section of the contract. If this contract automatically renews, answer Y. If this contract will remain in effect unless terminated, answer Y. Otherwise, answer N. If you cannot determine answer N.", + "prompt": "This pertains to the term and termination section of the contract. If this contract automatically renews (keyword \"automatically\" should be mentioned) , answer Y. Otherwise, answer N. If you cannot determine answer N.", "keywords": [ "Renew", "renew", @@ -360,7 +360,7 @@ "field_name": "AUTO_RENEWAL_TERM", "relationship": "one_to_one", "field_type": "smart_chunked", - "prompt": "If the contract automatically renews, what is the length of the renewal term? This pertains to the term and termination section of the contract. The response should be a numerical representation of the renewal period with no punctuation. Acceptable formats are '1 year' or '12 months' (both representing the same duration). 'Year to year' and 'year' should be interpreted as '1 year'. If the contract automatically renews but the length of the renewal term is not specified, default to '1 year'. Only return the length of time for the renewal with no additional context.", + "prompt": "If the contract automatically renews, what is the length of the renewal term? This pertains to the term and termination section of the contract. The response should be a numerical representation of the renewal period with no punctuation. Acceptable formats are '1 year' or '12 months' (both representing the same duration). 'Year to year' and 'year' should be interpreted as '1 year'. Only return the length of time for the renewal with no additional context.", "keywords": [ "Renew", "renew", diff --git a/src/prompts/prompt_templates.py b/src/prompts/prompt_templates.py index 1caaf89..2e528e0 100644 --- a/src/prompts/prompt_templates.py +++ b/src/prompts/prompt_templates.py @@ -1608,6 +1608,7 @@ Briefly explain your reasoning. After your explanation, include the heading "FIN - All provider objects must have these exact keys: "TIN", "NPI", "NAME" - ALl dict values must be strings - if there are multiple different TINs or NPIs for an identical NAME, put them in different dictionaries. - Use "N/A" for any missing values +- Strip ALL non-digit characters from any found TIN or NPI (e.g., remove hyphens, spaces, parentheses). - A provider record is considered valid as long as either a TIN, NPI, or NAME is present. If any one or two of those fields are missing, use "N/A" for those fields. - Your final JSON array must begin with an opening bracket '[' and end with a closing bracket ']' - Make sure the JSON array is correctly formatted with commas between objects diff --git a/src/tests/test_postprocess.py b/src/tests/test_postprocess.py index e5a5c3d..687c256 100644 --- a/src/tests/test_postprocess.py +++ b/src/tests/test_postprocess.py @@ -10,6 +10,7 @@ from src.pipelines.shared.postprocessing.postprocessing_funcs import ( clean_na_values, deduplicate_provider_columns, fill_claim_type_from_title, + add_aarete_derived_signatory_complete_ind, flatten_singleton_string_list, format_as_json_list, format_rate_fields_with_commas, @@ -695,6 +696,20 @@ class TestPostprocessFunctions(unittest.TestCase): # Empty list should be treated as empty and filled with mode or inferred assert result_df.loc[0, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "M" + def test_add_aarete_derived_signatory_complete_ind(self): + """Test the derivation of the signatory complete indicator for fully signed, partially signed, and zero-signature scenarios.""" + df = pd.DataFrame( + { + "NUM_AVAILABLE_SIGNATORY_LINE_COUNT": [2, 2, 0], + "NUM_SIGNED_SIGNATORY_LINE_COUNT": [2, 1, 0], + } + ) + result = add_aarete_derived_signatory_complete_ind(df) + + self.assertEqual( + result["AARETE_DERIVED_SIGNATORY_COMPLETE_IND"].tolist(), ["Y", "N", "N"] + ) + def test_clean_na_values_string_placeholders(self): """Test clean_na_values removes placeholder strings when they're the only value.""" # Test with string placeholders