Merged in hotfix/claim-type-cd (pull request #610)

Hotfix/claim type cd

* Fix exhibit_level prompt

* e2e test passed - successfully pulling multiple Claim Types per exhibit

* Merged main into hotfix/claim-type-cd


Approved-by: Alex Galarce

fieldExtraction/crosswalk/mappings/crosswalk_claim_type.json

@@ -6,7 +6,6 @@
     "mapping": {
         "Professional": "M",
         "Ancillary":"M",
-        "Medical": "M",
         "Facility":"H",
         "Hospital":"H",
         "Institutional":"H"

fieldExtraction/src/constants/investment_values.py

@@ -11,6 +11,7 @@ VALID_PRODUCTS = list(CrosswalkBuilder().from_json(path="crosswalk/mappings/cros
 VALID_PROV_SPECIALTY = list(CrosswalkBuilder().from_json(path="crosswalk/mappings/crosswalk_provider_specialty.json").mapping.keys())
 VALID_BILL_TYPE = list(set(CrosswalkBuilder().from_json(path="crosswalk/mappings/crosswalk_bill_type.json").mapping.values()))
 VALID_PLACE_OF_SERVICE = list(CrosswalkBuilder().from_json(path="crosswalk/mappings/crosswalk_place_of_service.json").mapping.keys())
+VALID_CLAIM_TYPE = list(set(CrosswalkBuilder().from_json(path="crosswalk/mappings/crosswalk_claim_type.json").mapping.keys()))
 
 
 # Provider Type
@@ -63,12 +64,6 @@ VALID_AARETE_DERIVED_FEE_SCHEDULE_VERSION = [
     "year prior"
 ]
 
-VALID_CLAIM_TYPE = [
-    "Professional",
-    "Institutional",
-    "Ancillary"
-]
-
 # AARETE_DERIVED_REIMB_METHOD
 VALID_REIMB_TERM = [
     "Billed Charges",

fieldExtraction/src/investment/file_processing.py

@@ -169,14 +169,6 @@ def run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, all_datase
     ################## Add N/A Dynamic or Exhibit to One-to-One ##################
     dynamic_one_to_one_fields = dynamic_funcs.get_dynamic_one_to_one_fields(one_to_n_results)
 
-    # if we did not get a valid claim_type_cd response for all rows, we need to run the prompt again with full context
-    for answer_dict in one_to_n_results:
-        if "CLAIM_TYPE_CD" not in answer_dict or string_utils.is_empty(answer_dict.get("CLAIM_TYPE_CD")):
-            # If any row does not have a valid CLAIM_TYPE_CD, we need to update the field type to full_context
-            field_to_add = Field.load_from_file(file_path=config.FIELD_JSON_PATH, field_name="CLAIM_TYPE_CD")
-            field_to_add.field_type = "full_context"
-            dynamic_one_to_one_fields.add_field(field_to_add)
-
     ################## CONVERT TO DF ##################
     one_to_n_df = pd.DataFrame(one_to_n_results)
     

fieldExtraction/src/investment/one_to_n_funcs.py

@@ -30,7 +30,7 @@ def get_exhibit_level_answers(exhibit_chunk, filename):
     if not exhibit_level_fields.contains_fields():
         return {}
 
-    prompt = EXHIBIT_LEVEL(exhibit_chunk, exhibit_level_fields)
+    prompt = EXHIBIT_LEVEL(exhibit_chunk, exhibit_level_fields.print_prompt_dict())
     llm_answer_raw = llm_utils.invoke_claude(
         prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, max_tokens=8192
     )

fieldExtraction/src/prompts/investment_prompts.json

@@ -169,12 +169,21 @@
     {
         "field_name" : "CLAIM_TYPE_CD",
         "relationship" : "one_to_n",
-        "field_type" : "exhibit_level",
-        "prompt": "What is the claim or service Type of the contract, exhibit, or services mentioned? (Possible answers: {valid_values})\n\nDetermine the Claim Type based on the content found near compensation or service descriptions.\n\nReturn the closest match from the following: {valid_values}.\n\nGuidelines:\n- If the contract refers to services rendered by individual healthcare professionals, return 'Professional'.\n- If the document mentions services provided by facilities or institutions, return 'Institutional'.\n- If the services include Audiology, Cardiac Monitoring, Diagnostic Imaging, Genetic Testing, Durable Medical Equipment (DME), Laboratory/Blood Testing, Radiology, Sleep Lab Services, Telemedicine, Allergy Services, Behavioral & Mental Health Services, or other ancillary services, return 'Ancillary'.\n- If multiple claim types are mentioned for different services, return the one most closely associated with the compensation or service description in question. Do not return N/A in any case.",
+        "field_type" : "dynamic_primary",
+        "prompt": "What is the Claim Type of the contract, exhibit, or services mentioned? Choose ONLY from the following: {valid_values}. Guidelines:\n- If the contract refers to services rendered by individual healthcare providers, return 'Professional'.\n- If the document mentions services provided by hospitals, facilities or institutions, return 'Institutional'.\n- If the services include Home Health, Durable Medical Equipment (DME), Laboratory, Radiology, Diagnostic Imaging, Genetic Testing, Blood Testing, Sleep Lab Services, Telemedicine, Behavioral & Mental Health Services, or other ancillary services, return 'Ancillary'.\n- If multiple claim types are mentioned for different services, return the one most closely associated with the compensation or service description in question. If there is no clear Claim Type, explicitly written or otherwise, return N/A.",
         "valid_values" : "VALID_CLAIM_TYPE",
         "keywords" : [],
         "smart_chunking_methodology" : "TBD"
     },
+    {
+        "field_name": "AARETE_DERIVED_CLAIM_TYPE_CD",
+        "base_field": "CLAIM_TYPE_CD",
+        "relationship": "TBD",
+        "field_type": "crosswalk",
+        "valid_values":"TBD",
+        "prompt": "Convert the claim type value to appropriate code: Return H for 'facility' or 'hospital' or 'Institutional'. Return M for 'ancillary' or 'professional'. Claim type: {CLAIM_TYPE_CD}. Return only 'H' or 'M' without any additional text.",
+        "crosswalk": "crosswalk_claim_type.json"
+    },
     {
         "field_name": "CONTRACT_SID",
         "relationship": "TBD",
@@ -320,15 +329,6 @@
         "field_type": "TBD",
         "prompt": "TBD"
     },
-    {
-        "field_name": "AARETE_DERIVED_CLAIM_TYPE_CD",
-        "base_field": "CLAIM_TYPE_CD",
-        "relationship": "TBD",
-        "field_type": "crosswalk",
-        "valid_values":"TBD",
-        "prompt": "Convert the claim type value to appropriate code: Return H for 'facility' or 'hospital' or 'Institutional'. Return M for 'ancillary' or 'professional'. Claim type: {CLAIM_TYPE_CD}. Return only 'H' or 'M' without any additional text.",
-        "crosswalk": "crosswalk_claim_type.json"
-    },
     {
         "field_name": "SERVICE_AREA",
         "relationship": "TBD",