Merged in bugfix/claim_type_1_to_1 (pull request #590)

Bugfix/claim type 1 to 1

* restored old prompt

* updated claim type cd prompt

* claim_type exhibit level added

* Merge branch 'bugfix/claim_type_1_to_1' of https://bitbucket.org/aarete/doczy.ai into bugfix/claim_type_1_to_1

* field type updated

* Merge remote-tracking branch 'origin/main' into bugfix/claim_type_1_to_1

* fixed location of updating field_type to full_context

* updated ancillary examples

* removed print statements

* Merge branch 'bugfix/claim_type_1_to_1' of https://bitbucket.org/aarete/doczy.ai into bugfix/claim_type_1_to_1

* prompt updated to avoid N/A

* Merge remote-tracking branch 'origin/main' into bugfix/claim_type_1_to_1

* prompt changed to handle multiple matches

* prompt updated

* Merge remote-tracking branch 'origin/main' into bugfix/claim_type_1_to_1


Approved-by: Alex Galarce

fieldExtraction/src/investment/file_processing.py

@@ -156,7 +156,7 @@ def run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, all_datase
         ################# COMBINE ANSWERS ##################
         full_answer_dict = combine_one_to_n_answers(exhibit_level_answers, reimbursement_level_answers, tin_npi_answers={})
         one_to_n_results += full_answer_dict
-       
+
     ################## Crosswalk Fields ################## 
     one_to_n_results = aarete_derived.get_crosswalk_fields(one_to_n_results)
 
@@ -169,6 +169,14 @@ def run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, all_datase
     ################## Add N/A Dynamic or Exhibit to One-to-One ##################
     dynamic_one_to_one_fields = dynamic_funcs.get_dynamic_one_to_one_fields(one_to_n_results)
 
+    # if we did not get a valid claim_type_cd response for all rows, we need to run the prompt again with full context
+    for answer_dict in one_to_n_results:
+        if "CLAIM_TYPE_CD" not in answer_dict or string_utils.is_empty(answer_dict.get("CLAIM_TYPE_CD")):
+            # If any row does not have a valid CLAIM_TYPE_CD, we need to update the field type to full_context
+            field_to_add = Field.load_from_file(file_path=config.FIELD_JSON_PATH, field_name="CLAIM_TYPE_CD")
+            field_to_add.field_type = "full_context"
+            dynamic_one_to_one_fields.add_field(field_to_add)
+
     ################## CONVERT TO DF ##################
     one_to_n_df = pd.DataFrame(one_to_n_results)
     

fieldExtraction/src/investment/one_to_n_funcs.py

@@ -35,6 +35,7 @@ def get_exhibit_level_answers(exhibit_chunk, filename):
         prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, max_tokens=8192
     )
     llm_answer_final = string_utils.universal_json_load(llm_answer_raw)
+
     return llm_answer_final
 
 

fieldExtraction/src/prompts/investment_prompts.json

@@ -168,9 +168,9 @@
     },
     {
         "field_name" : "CLAIM_TYPE_CD",
-        "relationship" : "TBD",
-        "field_type" : "TBD",
-        "prompt" : "Is the contract related to a professional, instututional, facility or ancillary provider? It may be deduced based on contract title. If the contract is with Hospital or Facility, the answer would be Institutional. If contract is directly with doctor or physician services, the answer would be Professional. If contract is with providers which only provides services like Audiology, Cardiac Monitoring, Diagnostic Imaging, Genetic Testing, Laboratory/Blood Testing, Radiology, Sleep Lab Services, Telemedicine, Allergy Services, Behavioral & Mental Health Services, answer would be Ancillary.",
+        "relationship" : "one_to_n",
+        "field_type" : "exhibit_level",
+        "prompt": "What is the claim or service Type of the contract, exhibit, or services mentioned? (Possible answers: {valid_values})\n\nDetermine the Claim Type based on the content found near compensation or service descriptions.\n\nReturn the closest match from the following: {valid_values}.\n\nGuidelines:\n- If the contract refers to services rendered by individual healthcare professionals, return 'Professional'.\n- If the document mentions services provided by facilities or institutions, return 'Institutional'.\n- If the services include Audiology, Cardiac Monitoring, Diagnostic Imaging, Genetic Testing, Durable Medical Equipment (DME), Laboratory/Blood Testing, Radiology, Sleep Lab Services, Telemedicine, Allergy Services, Behavioral & Mental Health Services, or other ancillary services, return 'Ancillary'.\n- If multiple claim types are mentioned for different services, return the one most closely associated with the compensation or service description in question. Do not return N/A in any case.",
         "valid_values" : "VALID_CLAIM_TYPE",
         "keywords" : [],
         "smart_chunking_methodology" : "TBD"
@@ -318,8 +318,8 @@
         "field_name": "AARETE_DERIVED_CLAIM_TYPE_CD",
         "base_field": "CLAIM_TYPE_CD",
         "relationship": "TBD",
-        "field_type": "TBD",
-        "valid_values":"VALID_AARETE_DERIVED_CLAIM_TYPE_CD",
+        "field_type": "crosswalk",
+        "valid_values":"TBD",
         "prompt": "Convert the claim type value to appropriate code: Return H for 'facility' or 'hospital' or 'Institutional'. Return M for 'ancillary' or 'professional'. Claim type: {CLAIM_TYPE_CD}. Return only 'H' or 'M' without any additional text.",
         "crosswalk": "crosswalk_claim_type.json"
     },