jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in bugfix/payer_name_provider_state (pull request #780)

Browse Source 
Bugfix/payer name provider state to main

* updated provider state

* updated prompt

* Merged main into bugfix/payer_name_provider_state

* updated smart chunked field types

* added post processing for provider state

* field check added

* filter by field_type instead of retrieval question

* Merged main into bugfix/payer_name_provider_state


Approved-by: Katon Minhas
This commit is contained in:
VenkataKrishna Reddy Avula
2025-11-19 19:02:45 +00:00
committed by Katon Minhas
parent f8ceb39111
commit 18d893c40a
4 changed files with 108 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fieldExtraction/src/investment/hybrid_smart_chunking_funcs.py
+6 -3
View File
@@ -175,7 +175,7 @@ def run_hybrid_smart_chunked_fields(
# Only process fields that have the retrieval_question attribute set
fields_to_process = [
field for field in one_to_one_fields.fields
if field.retrieval_question is not None
if field.field_type == "smart_chunked" and not string_utils.is_empty(field.retrieval_question)
]
if not fields_to_process:
@@ -306,8 +306,11 @@ def run_hybrid_smart_chunked_fields(
value="full_context",
)
# Normalize PROVIDER_STATE and PAYER_STATE to standardized two-letter abbreviation
for state_field in ["PROVIDER_STATE", "PAYER_STATE"]:
# Normalize PROVIDER_STATE
answers_dict = string_utils.normalize_provider_state_field(answers_dict)
# Normalize PAYER_STATE to standardized two-letter abbreviation
for state_field in ["PAYER_STATE"]:
if state_field in answers_dict:
answers_dict[state_field] = (
string_utils.normalize_state_to_abbreviation(
fieldExtraction/src/investment/one_to_one_funcs.py
+10 -14
View File
@@ -242,26 +242,22 @@ def run_full_context_fields(
full_context_fields = one_to_one_fields.filter(field_type="full_context")
# If there is only one state in the batch, don't run this, populate with the single state
if len(config.STATE) == 1:
full_context_fields.remove_field(field_name="PAYER_STATE")
full_context_answers_dict = prompt_calls.prompt_full_context(
contract_text, full_context_fields, constants, filename
)
if len(config.STATE) == 1:
full_context_answers_dict["PAYER_STATE"] = config.STATE[0]
# Normalize PROVIDER_STATE and PAYER_STATE to standardized two-letter abbreviation
for state_field in ["PROVIDER_STATE", "PAYER_STATE"]:
if state_field in full_context_answers_dict:
full_context_answers_dict[state_field] = (
string_utils.normalize_state_to_abbreviation(
full_context_answers_dict[state_field]
)
)
# Normalize PAYER_STATE to two-letter abbreviation
if "PAYER_STATE" in full_context_answers_dict:
full_context_answers_dict["PAYER_STATE"] = string_utils.normalize_state_to_abbreviation(
full_context_answers_dict["PAYER_STATE"]
)
# normalize PROVIDER_STATE to two-letter abbreviation
if "PROVIDER_STATE" in full_context_answers_dict:
full_context_answers_dict = string_utils.normalize_provider_state_field(
full_context_answers_dict
)
# Run Special Case Breakout on any breakout terms
full_context_answers_dict = one_to_one_breakout(full_context_answers_dict, filename)
fieldExtraction/src/prompts/investment_prompts.json
+19 -5
View File
@@ -199,28 +199,42 @@
{
"field_name": "CONTRACT_AMENDMENT_NUM",
"relationship": "one_to_one",
"field_type": "full_context",
"field_type": "smart_chunked",
"prompt": "Extract the amendment number if this document is an amendment to a contract. Follow these guidelines:\n1. Look for phrases like 'Amendment No. X', 'X Amendment', 'Amendment X to the Agreement', 'Xth Amendment' in the document title, preamble, or header sections.\n2. Return only the numeric value (e.g., for 'Amendment No. 3' return '3').\n3. If the amendment number is written as a word (e.g., 'First Amendment'), convert it to a number (e.g., '1').\n4. If the amendment number is an alphanumeric value, return the whole value (e.g., for '2A' return '2A').\n5. If the amendment number uses Roman numerals (e.g., 'Amendment IV'), convert it to a number (e.g., '4').\n6. If the amendment number includes decimal points (e.g., 'Amendment 2.1'), return the full numeric value (e.g., '2.1').\n7. If multiple amendment numbers appear, return the one most clearly associated with the current document.\n8. If the document is an amendment but the number cannot be determined, return 'UNKNOWN'.\n9. If the document is not an amendment, return 'N/A'.\n10. If the Amendment number contains any leading zeroes(e.g. '001'), Drop all leading zeros (e.g. '1').\n11. If the document is an amendment but the number cannot be determined, return '1'.",
"retrieval_question": "What is the amendment number of this document? Amendment No., Amendment Number, ordinal amendment, numbered amendment? Amendment to agreement, amendment to contract."
},
{
"field_name": "PROVIDER_STATE",
"relationship": "one_to_one",
"field_type": "full_context",
"prompt": "What US state is the provider based in or where does this contract apply? Look for:\n\n1. Provider address information in the contract header, preamble, or signature sections\n2. State names or abbreviations in provider contact information\n3. Language indicating the governing state or jurisdiction (e.g., 'governed by the laws of [State]')\n4. Service area or coverage area specifications\n5. Provider licensing information that mentions a specific state\n\nReturn the full state name (e.g., 'California', 'Texas', 'New York') or standard 2-letter abbreviation (e.g., 'CA', 'TX', 'NY') as it appears in the document.\n\nIf multiple states are mentioned, prioritize:\n1. The state in the provider's primary address\n2. The state mentioned in governing law clauses\n3. The state most frequently referenced in the contract\n\nIf no state can be determined, return 'N/A'.",
"field_type": "smart_chunked",
"prompt": "What US state(s) is the provider based in or where does this contract apply? Look for:\n\n1. Any provider-related address information (payment address, mailing address, service location, etc.)\n2. State names or standard 2-letter abbreviations in provider contact details\n3. Language indicating governing law or jurisdiction (e.g., 'governed by the laws of [State]')\n4. Service area or coverage area references\n5. Provider licensing information mentioning a specific state\n\nFirst extract all provider-related states found from the above scenarios in a list.\n\nThen extract the payor's state information in the same way (usually a single state).\n\nAfter extraction:\n1. Compare the provider states list with the payor state.\n2. If there is a match, return only the matching state in a single-item list.\n3. If there is no match, return all provider states in a properly formatted list with each state as a string in the list.\n\nReturn state names exactly as they appear in the document (full state name or 2-letter abbreviation).\n\nIf no state can be determined, return 'N/A'.",
"retrieval_question": "What US state is the provider based in or located in? What state does this contract apply to? What state is mentioned in the provider's address, service area, or licensing information?"
},
{
"field_name": "PROVIDER_PAYMENT_ADDRESS",
"relationship": "TBD",
"field_type": "TBD",
"prompt": "Identify the US state mentioned in the provider's PAYMENT address located in the signature section. Return the state exactly as it appears in the document (full state name or standard 2-letter abbreviation). If no state can be determined, return 'N/A'.",
"retrieval_question": "What US state is the provider's PAYMENT address located in? Which state is mentioned in the provider's payment address?"
},
{
"field_name": "PROVIDER_MAILING_ADDRESS",
"relationship": "TBD",
"field_type": "TBD",
"prompt": "Identify the US state mentioned in the provider's MAILING address within the signature section. Return the state exactly as it appears in the document (full state name or standard 2-letter abbreviation). If no state can be determined, return 'N/A'.",
"retrieval_question": "What US state is the provider's MAILING address located in? Which state is mentioned in the provider's mailing address?"
},
{
"field_name": "PAYER_STATE",
"relationship": "one_to_one",
"field_type": "full_context",
"field_type": "smart_chunked",
"prompt": "What US state is the payer based in or incorporated in? Look for:\n\n1. Payer address information in the contract header, preamble, or signature sections\n2. State names or abbreviations in payer contact information\n3. State of incorporation or principal place of business\n4. Payer headquarters location\n5. Payer licensing information that mentions a specific state\n\nReturn the full state name (e.g., 'California', 'Texas', 'New York') or standard 2-letter abbreviation (e.g., 'CA', 'TX', 'NY') as it appears in the document.\n\nIf multiple states are mentioned, prioritize:\n1. The state in the payer's primary/headquarters address\n2. The state of incorporation\n3. The state most frequently referenced for the payer organization\n\nIf no state can be determined, return 'N/A'.",
"retrieval_question": "What US state is the payer based in, incorporated in, or headquartered in? What state is mentioned in the payer's address or principal place of business?"
},
{
"field_name": "PAYER_NAME",
"relationship": "one_to_one",
"field_type": "full_context",
"field_type": "smart_chunked",
"prompt": "What is the name of the payer that is a party to the contract as stated in the Preamble?",
"retrieval_question": "What is the name of the payer, health plan, insurance company, or payor that is a party to this contract? Payer name in the preamble, parties section, or beginning of agreement."
},
fieldExtraction/src/utils/string_utils.py
+73
View File
@@ -575,3 +575,76 @@ def flatten_to_strings(items) -> list[str]:
result.append(str(items).strip())
return result
def parse_state_field_to_list(raw_value):
"""
Convert PROVIDER_STATE raw input into a clean list of state strings.
Accepts:
- list → returned as-is
- JSON list string → parsed to list
- comma-separated string → split to list
- single string → wrapped in list
- other types → converted to string → wrapped in list
Returns:
list[str]: Clean list of state strings
"""
if raw_value is None:
return []
# Case 1: Already a list
if isinstance(raw_value, list):
return raw_value
# Case 2: String input
if isinstance(raw_value, str):
stripped = raw_value.strip()
# JSON list string
if stripped.startswith("[") and stripped.endswith("]"):
try:
parsed = json.loads(stripped)
if isinstance(parsed, list):
return parsed
else:
return [parsed]
except Exception as e:
logging.debug(f"Failed JSON parse for state list: {e}. Falling back to comma split.")
# fallback: treat as comma-separated
# Comma-separated list
if "," in stripped:
return [s.strip() for s in stripped.split(",") if s.strip()]
# Single state string
return [stripped]
# Case 3: Any other type (int, dict, etc.)
try:
return [str(raw_value).strip()]
except Exception:
logging.error(f"Could not convert value to string: {raw_value!r}")
return []
def normalize_provider_state_field(answers_dict: dict, field_name: str = "PROVIDER_STATE"):
if field_name not in answers_dict or not answers_dict[field_name]:
answers_dict[field_name] = []
return answers_dict
raw_value = answers_dict[field_name]
# ---- Step 1: Parse into list ----
state_list = parse_state_field_to_list(raw_value)
# ---- Step 2: Normalize each state ----
normalized_states = []
for state in state_list:
try:
normalized_states.append(normalize_state_to_abbreviation(state))
except Exception as e:
logging.warning(f"Failed to normalize state '{state}': {e}. Keeping original.")
normalized_states.append(state)
answers_dict[field_name] = normalized_states
return answers_dict