Merged in feature/provider-state (pull request #602)

Feature/provider state

* Add prompt for extracting provider state information in investment prompts

* Add state normalization function and integrate into context field extraction

* black format string_utils.py; add tests for normalize_state_to_abbreviation function

* black format string_utils_test.py

* Add PROVIDER_STATE to investment column order

* Merged main into feature/provider-state


Approved-by: Katon Minhas

fieldExtraction/src/constants/investment_columns.py

@@ -10,6 +10,7 @@ COLUMN_ORDER = [
     "AARETE_DERIVED_AMENDMENT_NUM",
     "CLIENT_NAME",
     "PAYER_NAME",
+    "PROVIDER_STATE",
     "PROV_GROUP_TIN",
     "PROV_GROUP_NPI",
     "PROV_GROUP_NAME_FULL",

fieldExtraction/src/investment/one_to_one_funcs.py

@@ -237,6 +237,12 @@ def run_full_context_fields(
         full_context_answers_dict = {}
         for field in full_context_fields.list_fields():
             full_context_answers_dict[field] = f"Error: {str(e)}"
+
+    # Normalize PROVIDER_STATE to standardized two-letter abbreviation
+    if "PROVIDER_STATE" in full_context_answers_dict:
+        full_context_answers_dict["PROVIDER_STATE"] = string_utils.normalize_state_to_abbreviation(
+            full_context_answers_dict["PROVIDER_STATE"]
+        )
     
     # Handle separate one-to-one fields that require individual processing
     separate_fields = one_to_one_fields.filter(field_type="full_context_separate")

fieldExtraction/src/prompts/investment_prompts.json

@@ -210,6 +210,12 @@
         "field_type": "full_context",
         "prompt": "Extract the amendment number if this document is an amendment to a contract. Follow these guidelines:\n1. Look for phrases like 'Amendment No. X', 'X Amendment', 'Amendment X to the Agreement', 'Xth Amendment' in the document title, preamble, or header sections.\n2. Return only the numeric value (e.g., for 'Amendment No. 3' return '3').\n3. If the amendment number is written as a word (e.g., 'First Amendment'), convert it to a number (e.g., '1').\n4. If the amendment number is an alphanumeric value, return the whole value (e.g., for '2A' return '2A').\n5. If the amendment number uses Roman numerals (e.g., 'Amendment IV'), convert it to a number (e.g., '4').\n6. If the amendment number includes decimal points (e.g., 'Amendment 2.1'), return the full numeric value (e.g., '2.1').\n7. If multiple amendment numbers appear, return the one most clearly associated with the current document.\n8. If the document is an amendment but the number cannot be determined, return 'UNKNOWN'.\n9. If the document is not an amendment, return 'N/A'.\n10. If the Amendment number contains any leading zeroes(e.g. '001'), Drop all leading zeros (e.g. '1').\n11. If the document is an amendment but the number cannot be determined, return '1'."
     },
+    {
+    "field_name": "PROVIDER_STATE",
+    "relationship": "one_to_one",
+    "field_type": "full_context",
+    "prompt": "What US state is the provider based in or where does this contract apply? Look for:\n\n1. Provider address information in the contract header, preamble, or signature sections\n2. State names or abbreviations in provider contact information\n3. Language indicating the governing state or jurisdiction (e.g., 'governed by the laws of [State]')\n4. Service area or coverage area specifications\n5. Provider licensing information that mentions a specific state\n\nReturn the full state name (e.g., 'California', 'Texas', 'New York') or the standard 2-letter abbreviation (e.g., 'CA', 'TX', 'NY') as it appears in the document.\n\nIf multiple states are mentioned, prioritize:\n1. The state in the provider's primary address\n2. The state mentioned in governing law clauses\n3. The state most frequently referenced in the contract\n\nIf no state can be determined, return 'N/A'."
+    },
     {
         "field_name": "CLIENT_NAME",
         "relationship": "one_to_one",

fieldExtraction/src/utils/string_utils.py

@@ -10,8 +10,11 @@ import src.utils.llm_utils as llm_utils
 from src import config
 from src.enums.delimiters import Delimiter
 from src.prompts import preprocessing_prompts
-from src.regex.regex_patterns import (BACKTICK_PATTERN, PIPE_PATTERN,
-                                      TRIPLE_BACKTICK_PATTERN)
+from src.regex.regex_patterns import (
+    BACKTICK_PATTERN,
+    PIPE_PATTERN,
+    TRIPLE_BACKTICK_PATTERN,
+)
 
 
 def extract_text_from_delimiters(
@@ -71,7 +74,8 @@ def extract_text_from_delimiters(
 
     return matches[match_index]
 
-def page_key_sort(page_key: str) -> tuple[int, float|str]:
+
+def page_key_sort(page_key: str) -> tuple[int, float | str]:
     """Sorts page keys, prioritizing numeric keys first.
 
     Args:
@@ -85,14 +89,14 @@ def page_key_sort(page_key: str) -> tuple[int, float|str]:
         (0, 1.0)
         >>> page_key_sort("A")
         (1, "A")
-    
-    
+
+
     Example Usage with mixed types:
         >>> L = ["10", "2", "A", "1"]
         >>> sort(L, key=page_key_sort)
         >>> print(L)
         ['1', '2', '10', 'A']
-    
+
     """
     try:
         return (0, float(page_key))  # Numeric pages first, in numerical order
@@ -129,18 +133,18 @@ def json_parsing_search(response_text: str, field_list: list[str]) -> dict[str,
             response_text = response_text.rsplit("}", 1)[0]
     else:
         response_text = response_text.rstrip(",") + "}"
-    
+
     field_l = []
     answer_l = []
     position_dict = {}
-    
+
     for f in field_list:
         location = response_text.find('"' + f + '"')
         if location != -1:
             position_dict[location] = f
-    
+
     field_list = list(dict(sorted(position_dict.items())).values())
-    
+
     for f in field_list:
         if f in response_text:
             field_l.append(f)
@@ -253,6 +257,7 @@ def primary_string_to_dict(string_dict, filename):
             data.append(dict_)
     return data
 
+
 def universal_json_load(string_dict: str):
     """Matches json dicts and list of dicts - NOT simple lists
     Args:
@@ -262,10 +267,10 @@ def universal_json_load(string_dict: str):
     Raises:
         ValueError: If no valid JSON object is found in the input string. In addition,
         the traceback as well as the offending string are printed.
-    
+
     """
     # Try to match dictionary or list of dictionaries first
-    dict_match = re.search(r'\{.*\}|\[\s*?\{.*\}\s*?\]', string_dict, re.DOTALL)
+    dict_match = re.search(r"\{.*\}|\[\s*?\{.*\}\s*?\]", string_dict, re.DOTALL)
     if dict_match:
         matched_str = dict_match.group()
         try:
@@ -275,32 +280,36 @@ def universal_json_load(string_dict: str):
             print(f"Original string: {string_dict}")
             print(f"Traceback: {traceback.format_exc()}")
             raise ValueError(f"JSON parsing failed: {e.msg} at position {e.pos}") from e
-    
+
     # If no dict pattern found, try to match list of strings
-    list_match = re.search(r'\[(.*?)\]', string_dict, re.DOTALL)
+    list_match = re.search(r"\[(.*?)\]", string_dict, re.DOTALL)
     if list_match:
         matched_str = list_match.group()
         try:
             # Clean up the string list before parsing
             cleaned_str = re.sub(r'(?<!\\)"', '"', matched_str)  # Handle escaped quotes
-            cleaned_str = re.sub(r"'", '"', cleaned_str)  # Replace single quotes with double quotes
+            cleaned_str = re.sub(
+                r"'", '"', cleaned_str
+            )  # Replace single quotes with double quotes
             return json.loads(cleaned_str)
         except json.JSONDecodeError as e:
             print(f"Failed to parse list: {e}")
             print(f"Original string: {string_dict}")
             print(f"Traceback: {traceback.format_exc()}")
             raise ValueError(f"JSON parsing failed: {e.msg} at position {e.pos}") from e
-    
+
     return {}
 
-reimbursement_strings = [ # These are for `method='keyword'`
+
+reimbursement_strings = [  # These are for `method='keyword'`
     "%",
     "$",
     "percent",
-    "billed charges"
+    "billed charges",
 ]
 reimb_regex = r"(?<![$%])(?:\$\d+|\d+[$%])(?![$%])"
 
+
 # TODO: check should this always be checking just page 1?  Bc in file_processing and table_utils it is but in preprocessing_funcs it's checking specific pages
 def contains_reimbursement(text, page="1", method="keyword"):
     """
@@ -329,7 +338,8 @@ def contains_reimbursement(text, page="1", method="keyword"):
     else:
         raise ValueError("Invalid method. Choose 'keyword' or 'regex'.")
 
-def count_reimbursements_in_exhibit(exhibit_text: str) -> int: #JUST by regex
+
+def count_reimbursements_in_exhibit(exhibit_text: str) -> int:  # JUST by regex
     """Counts reimbursements in an exhibit text.  Reimbursements are detected by a regex
     search as defined by `reimb_regex`.
 
@@ -341,15 +351,16 @@ def count_reimbursements_in_exhibit(exhibit_text: str) -> int: #JUST by regex
     """
     return len(re.findall(reimb_regex, exhibit_text))
 
-def is_empty(value: str | list | pd.Series, pd_mask: bool=True) -> bool | pd.Series:
+
+def is_empty(value: str | list | pd.Series, pd_mask: bool = True) -> bool | pd.Series:
     """
     Checks if a value is considered empty or invalid.
 
     Args:
         value: The value to check.
         Can be a string, list, or pandas Series.
-        pd_mask (bool): Only relevant for Series inputs. 
-            If True, returns a mask for empty values in a pandas Series. 
+        pd_mask (bool): Only relevant for Series inputs.
+            If True, returns a mask for empty values in a pandas Series.
             Otherwise, the function returns True iff the entire Series is empty.
 
     Returns:
@@ -361,13 +372,15 @@ def is_empty(value: str | list | pd.Series, pd_mask: bool=True) -> bool | pd.Ser
                 - If `pd_mask` is False, returns True iff all elements are empty/invalid, or if the entire Series is empty.
     """
     empty_values = [None, "", "N/A", "NA", "null", "none", "NaN", np.nan, "nan", "None"]
-    
+
     if isinstance(value, list):  # Handle list inputs
         return not value or all(is_empty(v) for v in value)
-    
+
     # if it's a pd.Series, return the mask (True for empty values)
     if isinstance(value, pd.Series):
-        return value.isna() | (value.isin(empty_values)) if pd_mask else value.isna().all()
+        return (
+            value.isna() | (value.isin(empty_values)) if pd_mask else value.isna().all()
+        )
 
     if pd.isna(value):
         return True
@@ -378,9 +391,10 @@ def is_empty(value: str | list | pd.Series, pd_mask: bool=True) -> bool | pd.Ser
 def datetime_str():
     return datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")
 
+
 def extract_signature_page(text_dict: dict, filename: str) -> dict:
     """Extracts the signature page(s) from a contract text dictionary and returns it as a dictionary.
-    First tries to match the textract marker "this page has N signature(s)", then falls back to looking 
+    First tries to match the textract marker "this page has N signature(s)", then falls back to looking
     for the word "signature" if no matches are found.  "this page has N signature(s)" is Textract's marker for signatures
 
     Args:
@@ -394,19 +408,25 @@ def extract_signature_page(text_dict: dict, filename: str) -> dict:
     signature_pages = {}
     # This regex pattern matches the Textract marker for signatures
     # It will look for both numeric and a few textual representations of numbers (one-ten)
-    textract_pattern = re.compile(r"this page has (?:\d+|one|two|three|four|five|six|seven|eight|nine|ten) signature", re.IGNORECASE)
-        
+    textract_pattern = re.compile(
+        r"this page has (?:\d+|one|two|three|four|five|six|seven|eight|nine|ten) signature",
+        re.IGNORECASE,
+    )
+
     # First pass: Try to match the full textract pattern
     for page_num, page_text in text_dict.items():
         if textract_pattern.search(page_text):
             signature_pages[page_num] = page_text
-    
+
     # If no pages found with textract pattern, try with  keyword
     if not signature_pages:
         page_items = list(text_dict.items())
         for i, (page_num, page_text) in enumerate(page_items):
             lower_text = page_text.lower()
-            if "signature page follow" in lower_text or "signature authorization" in lower_text:
+            if (
+                "signature page follow" in lower_text
+                or "signature authorization" in lower_text
+            ):
                 # curr_page = int(float(page_num))
                 if page_num not in signature_pages:
                     signature_pages[page_num] = page_text  # current page
@@ -414,8 +434,7 @@ def extract_signature_page(text_dict: dict, filename: str) -> dict:
                     next_page_num, next_page_text = page_items[i + 1]
                     # next_page = int(float(next_page_num))
                     if next_page_num not in signature_pages:
-                        signature_pages[next_page_num] = next_page_text  # next page 
-
+                        signature_pages[next_page_num] = next_page_text  # next page
 
     return signature_pages
 
@@ -424,9 +443,9 @@ def extract_effective_date_pages(text_dict: dict, filename: str) -> dict:
     """
     Extracts the effective date page(s) from a contract text dictionary and returns it as a dictionary.
 
-    This function identifies pages that either contain the Textract marker 
-    "this page has N signature(s)" or the keyword 'effective date:'. 
-    Additionally, it ensures that the first page (page 1) is always included 
+    This function identifies pages that either contain the Textract marker
+    "this page has N signature(s)" or the keyword 'effective date:'.
+    Additionally, it ensures that the first page (page 1) is always included
     in the result, as it often contains effective date.
 
     Args:
@@ -434,7 +453,7 @@ def extract_effective_date_pages(text_dict: dict, filename: str) -> dict:
         filename (str): The name of the file being processed.
 
     Returns:
-        dict: A dictionary containing the extracted effective date page(s), 
+        dict: A dictionary containing the extracted effective date page(s),
               where keys are page numbers and values are the corresponding page text.
     """
 
@@ -442,16 +461,129 @@ def extract_effective_date_pages(text_dict: dict, filename: str) -> dict:
 
     # Regex pattern to match the Textract marker for signature pages which often has effective dates
     # This pattern accounts for both numeric and textual representations of numbers (one-ten).
-    textract_pattern = re.compile(r"this page has (?:\d+|one|two|three|four|five|six|seven|eight|nine|ten) signature", re.IGNORECASE)
-        
+    textract_pattern = re.compile(
+        r"this page has (?:\d+|one|two|three|four|five|six|seven|eight|nine|ten) signature",
+        re.IGNORECASE,
+    )
+
     # Iterate through each page in the text dictionary.
     for page_num, page_text in text_dict.items():
         # page_num = int(float(page_num))
         # Check if the page contains the Textract marker or the keyword 'effective date:'.
-        if textract_pattern.search(page_text) or 'effective date:' in page_text.lower():
+        if textract_pattern.search(page_text) or "effective date:" in page_text.lower():
             effective_date_pages[page_num] = page_text
         # Always include the first page (page 1) as it often contains effective date.
         elif page_num == 1 or page_num == "1" or page_num == "1.0":
             effective_date_pages[page_num] = page_text
 
     return effective_date_pages
+
+
+def normalize_state_to_abbreviation(state_value: str) -> str:
+    """Convert state names or abbreviations to standardized two-letter abbreviations.
+
+    Handles various formats of state names and abbreviations, including full names,
+    common abbreviations, and variations. Returns the standardized two-letter abbreviation
+    or the original value if no match is found.
+
+    Args:
+        state_value (str): State name or abbreviation from extraction
+
+    Returns:
+        str: Two-letter state abbreviation or original value if no match found
+    """
+    if not state_value or state_value in ["N/A", "UNKNOWN"]:
+        return state_value
+
+    # Dictionary mapping full state names to abbreviations
+    state_mapping = {
+        "alabama": "AL",
+        "alaska": "AK",
+        "arizona": "AZ",
+        "arkansas": "AR",
+        "california": "CA",
+        "colorado": "CO",
+        "connecticut": "CT",
+        "delaware": "DE",
+        "florida": "FL",
+        "georgia": "GA",
+        "hawaii": "HI",
+        "idaho": "ID",
+        "illinois": "IL",
+        "indiana": "IN",
+        "iowa": "IA",
+        "kansas": "KS",
+        "kentucky": "KY",
+        "louisiana": "LA",
+        "maine": "ME",
+        "maryland": "MD",
+        "massachusetts": "MA",
+        "michigan": "MI",
+        "minnesota": "MN",
+        "mississippi": "MS",
+        "missouri": "MO",
+        "montana": "MT",
+        "nebraska": "NE",
+        "nevada": "NV",
+        "new hampshire": "NH",
+        "new jersey": "NJ",
+        "new mexico": "NM",
+        "new york": "NY",
+        "north carolina": "NC",
+        "north dakota": "ND",
+        "ohio": "OH",
+        "oklahoma": "OK",
+        "oregon": "OR",
+        "pennsylvania": "PA",
+        "rhode island": "RI",
+        "south carolina": "SC",
+        "south dakota": "SD",
+        "tennessee": "TN",
+        "texas": "TX",
+        "utah": "UT",
+        "vermont": "VT",
+        "virginia": "VA",
+        "washington": "WA",
+        "west virginia": "WV",
+        "wisconsin": "WI",
+        "wyoming": "WY",
+        "district of columbia": "DC",
+    }
+
+    # Clean and normalize the input
+    cleaned_state = state_value.strip().lower()
+
+    # Check if it's already a valid 2-letter abbreviation
+    if len(cleaned_state) == 2 and cleaned_state.upper() in state_mapping.values():
+        return cleaned_state.upper()
+
+    # Check if it matches a full state name
+    if cleaned_state in state_mapping:
+        return state_mapping[cleaned_state]
+
+    # Handle common variations
+    variations = {
+        "calif": "CA",
+        "cal": "CA",
+        "fla": "FL",
+        "florida": "FL",
+        "ny": "NY",
+        "n.y.": "NY",
+        "tx": "TX",
+        "tex": "TX",
+        "penn": "PA",
+        "pa": "PA",
+        "mass": "MA",
+        "massachusetts": "MA",
+        "wash": "WA",
+        "washington state": "WA",
+        "d.c.": "DC",
+        "dc": "DC",
+        "washington dc": "DC",
+    }
+
+    if cleaned_state in variations:
+        return variations[cleaned_state]
+
+    # If no match found, return original value
+    return state_value

fieldExtraction/tests/string_utils_test.py

@@ -1,42 +1,122 @@
 import pytest
 import src.utils as utils
-from src.utils.string_utils import extract_text_from_delimiters, json_parsing_search, secondary_string_to_dict, primary_string_to_dict, contains_reimbursement, is_empty, count_reimbursements_in_exhibit, extract_signature_page, page_key_sort
+from src.utils.string_utils import (
+    extract_text_from_delimiters,
+    json_parsing_search,
+    secondary_string_to_dict,
+    primary_string_to_dict,
+    contains_reimbursement,
+    is_empty,
+    count_reimbursements_in_exhibit,
+    extract_signature_page,
+    page_key_sort,
+)
 import src.utils.llm_utils as llm_utils
 import numpy as np
 import pandas as pd
 
 from src.enums.delimiters import Delimiter
 
+
 class TestStringUtils:
-    @pytest.mark.parametrize("raw_text, delimiter, match_index, expected", [
-        ("Here is the answer |this is a pipe answer| and |more text.|", Delimiter.PIPE, 0, "this is a pipe answer"),
-        ("Here is the answer |this is a pipe answer| and |more text.|", Delimiter.PIPE, -1, "more text."),
-        ("|Here| is the answer |this is a pipe answer| and |more text.|", Delimiter.PIPE, 1, "this is a pipe answer"),
-        ("Here is the answer `this is a backtick answer` and `more text.`", Delimiter.BACKTICK, 0, "this is a backtick answer"),
-        ("Here is the answer `this is a backtick answer` and `more text.`", Delimiter.BACKTICK, -1, "more text."),
-        ("`Here` is the answer `this` is a `backtick answer` and `more text.`", Delimiter.BACKTICK, 2, "backtick answer"),
-        ("Here is the answer ```this is a triple backtick answer``` and more text.", Delimiter.TRIPLE_BACKTICK, 0, "this is a triple backtick answer"),
-        ("Here is the answer ```this``` is a ```triple``` backtick ```answer``` and more text.", Delimiter.TRIPLE_BACKTICK, -1, "answer"),
-        ("```Here``` is the answer ```this is a triple backtick answer``` and ```more text.```", Delimiter.TRIPLE_BACKTICK, 1, "this is a triple backtick answer"),
-    ])
-    def test_extract_text_from_delimiters(self, raw_text, delimiter, match_index, expected):
+    @pytest.mark.parametrize(
+        "raw_text, delimiter, match_index, expected",
+        [
+            (
+                "Here is the answer |this is a pipe answer| and |more text.|",
+                Delimiter.PIPE,
+                0,
+                "this is a pipe answer",
+            ),
+            (
+                "Here is the answer |this is a pipe answer| and |more text.|",
+                Delimiter.PIPE,
+                -1,
+                "more text.",
+            ),
+            (
+                "|Here| is the answer |this is a pipe answer| and |more text.|",
+                Delimiter.PIPE,
+                1,
+                "this is a pipe answer",
+            ),
+            (
+                "Here is the answer `this is a backtick answer` and `more text.`",
+                Delimiter.BACKTICK,
+                0,
+                "this is a backtick answer",
+            ),
+            (
+                "Here is the answer `this is a backtick answer` and `more text.`",
+                Delimiter.BACKTICK,
+                -1,
+                "more text.",
+            ),
+            (
+                "`Here` is the answer `this` is a `backtick answer` and `more text.`",
+                Delimiter.BACKTICK,
+                2,
+                "backtick answer",
+            ),
+            (
+                "Here is the answer ```this is a triple backtick answer``` and more text.",
+                Delimiter.TRIPLE_BACKTICK,
+                0,
+                "this is a triple backtick answer",
+            ),
+            (
+                "Here is the answer ```this``` is a ```triple``` backtick ```answer``` and more text.",
+                Delimiter.TRIPLE_BACKTICK,
+                -1,
+                "answer",
+            ),
+            (
+                "```Here``` is the answer ```this is a triple backtick answer``` and ```more text.```",
+                Delimiter.TRIPLE_BACKTICK,
+                1,
+                "this is a triple backtick answer",
+            ),
+        ],
+    )
+    def test_extract_text_from_delimiters(
+        self, raw_text, delimiter, match_index, expected
+    ):
         result = extract_text_from_delimiters(raw_text, delimiter, match_index)
         assert result == expected
 
-    @pytest.mark.parametrize("response_text, field_list, expected", [
-        ('{"name": "John", "age": "30", "city": "New York"}', ["name", "age", "city"], {"name": "John", "age": "30", "city": "New York"}),
-        ('{"name": "John", "city": "New York"}', ["name", "age", "city"], {"name": "John", "city": "New York"}),
-        ('', ["name", "age", "city"], {}),
-        ('{"name": "John", "age": "30", "city": "New York"}', [], {}),
-        ('{"name": "", "age": "", "city": ""}', ["name", "age", "city"], {"name": "", "age": "", "city": ""}),
-        ('{\n"name": "John",\n"age": "30",\n"city": "New York"\n}', ["name", "age", "city"], {"name": "John", "age": "30", "city": "New York"}),
-    ])
+    @pytest.mark.parametrize(
+        "response_text, field_list, expected",
+        [
+            (
+                '{"name": "John", "age": "30", "city": "New York"}',
+                ["name", "age", "city"],
+                {"name": "John", "age": "30", "city": "New York"},
+            ),
+            (
+                '{"name": "John", "city": "New York"}',
+                ["name", "age", "city"],
+                {"name": "John", "city": "New York"},
+            ),
+            ("", ["name", "age", "city"], {}),
+            ('{"name": "John", "age": "30", "city": "New York"}', [], {}),
+            (
+                '{"name": "", "age": "", "city": ""}',
+                ["name", "age", "city"],
+                {"name": "", "age": "", "city": ""},
+            ),
+            (
+                '{\n"name": "John",\n"age": "30",\n"city": "New York"\n}',
+                ["name", "age", "city"],
+                {"name": "John", "age": "30", "city": "New York"},
+            ),
+        ],
+    )
     def test_json_parsing_search(self, response_text, field_list, expected):
         assert json_parsing_search(response_text, field_list) == expected
 
     @pytest.fixture
     def mock_invoke_claude(self, mocker):
-        return mocker.patch.object(llm_utils, 'invoke_claude')
+        return mocker.patch.object(llm_utils, "invoke_claude")
 
     def test_valid_json(self, mock_invoke_claude):
         dict_string = '{"key": "value"}'
@@ -69,7 +149,7 @@ class TestStringUtils:
         mock_invoke_claude.assert_called_once()
 
     def test_no_dict_found(self, mock_invoke_claude):
-        mock_invoke_claude.return_value = '{}'
+        mock_invoke_claude.return_value = "{}"
         dict_string = "This is a plain text without any dictionary."
         filename = "test_file.txt"
         result = secondary_string_to_dict(dict_string, filename)
@@ -160,13 +240,37 @@ class TestStringUtils:
         assert result == expected_output
         assert mock_secondary_string_to_dict.call_count == 4
 
-    @pytest.mark.parametrize("text, page, expected", [
-        ({"1": "This page contains a 10% reimbursement schedule.", "2": "No relevant content here."}, "1", True),
-        ({"1": "This page has no relevant content.", "2": "Still no relevant content here."}, "1", False),
-        ({"1": "This page contains a 10% reimbursement schedule.", "2": "No relevant content here."}, "invalid_page", False),
-        ("This text contains a $100 reimbursement schedule.", "1", True),
-        ("This text has no relevant content.", "1", False),
-    ])
+    @pytest.mark.parametrize(
+        "text, page, expected",
+        [
+            (
+                {
+                    "1": "This page contains a 10% reimbursement schedule.",
+                    "2": "No relevant content here.",
+                },
+                "1",
+                True,
+            ),
+            (
+                {
+                    "1": "This page has no relevant content.",
+                    "2": "Still no relevant content here.",
+                },
+                "1",
+                False,
+            ),
+            (
+                {
+                    "1": "This page contains a 10% reimbursement schedule.",
+                    "2": "No relevant content here.",
+                },
+                "invalid_page",
+                False,
+            ),
+            ("This text contains a $100 reimbursement schedule.", "1", True),
+            ("This text has no relevant content.", "1", False),
+        ],
+    )
     def test_contains_reimbursement(self, text, page, expected):
         result = contains_reimbursement(text, page)
         assert result == expected
@@ -179,75 +283,99 @@ class TestStringUtils:
         captured = capsys.readouterr()
         assert "contains_reimbursement - Invalid data type" in captured.out
 
-    @pytest.mark.parametrize("value, expected", [
-        (None, True),
-        ("", True),
-        ("N/A", True),
-        ("null", True),
-        ("none", True),
-        ("NaN", True),
-        (np.nan, True),
-        (pd.NA, True),
-        ("This is not empty.", False),
-        (42, False),
-    ])
+    @pytest.mark.parametrize(
+        "value, expected",
+        [
+            (None, True),
+            ("", True),
+            ("N/A", True),
+            ("null", True),
+            ("none", True),
+            ("NaN", True),
+            (np.nan, True),
+            (pd.NA, True),
+            ("This is not empty.", False),
+            (42, False),
+        ],
+    )
     def test_is_empty(self, value, expected):
         result = is_empty(value)
         assert result == expected
 
-    @pytest.mark.parametrize("text_dict, filename, expected", [
-        (
-            {"1": "This page has 2 signatures.", "2": "None here.", "3": "Signature of the party (but ignored)"},
-            "test_file.txt",
-            {"1": "This page has 2 signatures."}
-        ),
-        (
-            {"1": "No relevant content here.", "2": "Nor over here."},
-            "test_file.txt",
-            {}
-        ),
-        (
-            {"1": "This page has a signature.", "2": "signature page follow.","3": "Signature of the party (but ignored)"},
-            "test_file.txt",
-            {"2": "signature page follow.","3": "Signature of the party (but ignored)"}
-        ),
-        (
-            {"1": "This page has 1 signature.", "2": "No relevant content.", "3": "Signature block here.  But ignored bc of the first textract-like page"},
-            "test_file.txt",
-            {"1": "This page has 1 signature."}
-        ),
-        (
-            {"1": "No signed names at all.", "2": "Just some random text."},
-            "test_file.txt",
-            {}
-        ),
-        (
-            {"1": "some random text", "2": "Signature authorization", "3": "Signature of the party"},
-            "test_file.txt",
-            {"2": "Signature authorization", "3": "Signature of the party"}
-        ),
-        (
-            {},
-            "test_file.txt",
-            {}
-        ),
-    ])
+    @pytest.mark.parametrize(
+        "text_dict, filename, expected",
+        [
+            (
+                {
+                    "1": "This page has 2 signatures.",
+                    "2": "None here.",
+                    "3": "Signature of the party (but ignored)",
+                },
+                "test_file.txt",
+                {"1": "This page has 2 signatures."},
+            ),
+            (
+                {"1": "No relevant content here.", "2": "Nor over here."},
+                "test_file.txt",
+                {},
+            ),
+            (
+                {
+                    "1": "This page has a signature.",
+                    "2": "signature page follow.",
+                    "3": "Signature of the party (but ignored)",
+                },
+                "test_file.txt",
+                {
+                    "2": "signature page follow.",
+                    "3": "Signature of the party (but ignored)",
+                },
+            ),
+            (
+                {
+                    "1": "This page has 1 signature.",
+                    "2": "No relevant content.",
+                    "3": "Signature block here.  But ignored bc of the first textract-like page",
+                },
+                "test_file.txt",
+                {"1": "This page has 1 signature."},
+            ),
+            (
+                {"1": "No signed names at all.", "2": "Just some random text."},
+                "test_file.txt",
+                {},
+            ),
+            (
+                {
+                    "1": "some random text",
+                    "2": "Signature authorization",
+                    "3": "Signature of the party",
+                },
+                "test_file.txt",
+                {"2": "Signature authorization", "3": "Signature of the party"},
+            ),
+            ({}, "test_file.txt", {}),
+        ],
+    )
     def test_extract_signature_page(self, text_dict, filename, expected):
         result = extract_signature_page(text_dict, filename)
         assert result == expected
 
-    @pytest.mark.parametrize("page_key, expected", [
-    ("1", (0, 1)),  # Numeric key
-    ("10", (0, 10)),  # Larger numeric key
-    ("A", (1, "A")),  # Alphabetic key
-    ("Z", (1, "Z")),  # Another alphabetic key
-    ("1A", (1, "1A")),  # Alphanumeric key
-    ("", (1, "")),  # Empty string
-    ("001", (0, 1)),  # Numeric key with leading zeros
-    ("B2", (1, "B2")),  # Alphanumeric key with letter first
-    ("-5", (0, -5)),  # Negative numeric
-    (" ", (1, " ")),  # Space character
-    ])
+    @pytest.mark.parametrize(
+        "page_key, expected",
+        [
+            ("1", (0, 1)),  # Numeric key
+            ("10", (0, 10)),  # Larger numeric key
+            ("A", (1, "A")),  # Alphabetic key
+            ("Z", (1, "Z")),  # Another alphabetic key
+            ("1A", (1, "1A")),  # Alphanumeric key
+            ("", (1, "")),  # Empty string
+            ("001", (0, 1)),  # Numeric key with leading zeros
+            ("B2", (1, "B2")),  # Alphanumeric key with letter first
+            ("-5", (0, -5)),  # Negative numeric
+            (" ", (1, " ")),  # Space character
+        ],
+    )
     def test_page_key_sort(self, page_key, expected):
         result = page_key_sort(page_key)
         assert result == expected
@@ -258,16 +386,191 @@ class TestStringUtils:
         sorted_keys = sorted(keys, key=page_key_sort)
         assert sorted_keys == expected_sorted_keys
 
+
 class TestCountReimbursementsInExhibit:
-    @pytest.mark.parametrize("exhibit_text, expected_count", [
-        ("This exhibit includes a 10% reimbursement and a $100 reimbursement.", 2),
-        ("No reimbursements mentioned here.", 0),
-        ("Reimbursement of 50% and another reimbursement of $200.", 2),
-        ("100 percent reimbursement and fifty dollars reimbursement.", 0),
-        ("", 0),
-        ("Reimbursement: 20% and $300.", 2),
-        ("Reimbursement: 20% and $300. Another 10% reimbursement.", 3),
-    ])
+    @pytest.mark.parametrize(
+        "exhibit_text, expected_count",
+        [
+            ("This exhibit includes a 10% reimbursement and a $100 reimbursement.", 2),
+            ("No reimbursements mentioned here.", 0),
+            ("Reimbursement of 50% and another reimbursement of $200.", 2),
+            ("100 percent reimbursement and fifty dollars reimbursement.", 0),
+            ("", 0),
+            ("Reimbursement: 20% and $300.", 2),
+            ("Reimbursement: 20% and $300. Another 10% reimbursement.", 3),
+        ],
+    )
     def test_count_reimbursements_in_exhibit(self, exhibit_text, expected_count):
         result = count_reimbursements_in_exhibit(exhibit_text)
         assert result == expected_count
+
+
+class TestNormalizeStateToAbbreviation:
+    @pytest.mark.parametrize(
+        "state_value, expected",
+        [
+            # Special values should be preserved
+            ("N/A", "N/A"),
+            ("UNKNOWN", "UNKNOWN"),
+            ("", ""),
+            (None, None),
+            # Valid 2-letter abbreviations should be uppercased
+            ("ca", "CA"),
+            ("CA", "CA"),
+            ("ny", "NY"),
+            ("TX", "TX"),
+            ("fl", "FL"),
+            # Full state names should be converted to abbreviations
+            ("california", "CA"),
+            ("California", "CA"),
+            ("CALIFORNIA", "CA"),
+            ("new york", "NY"),
+            ("New York", "NY"),
+            ("NEW YORK", "NY"),
+            ("texas", "TX"),
+            ("Texas", "TX"),
+            ("florida", "FL"),
+            ("Florida", "FL"),
+            # Common variations should be handled
+            ("calif", "CA"),
+            ("cal", "CA"),
+            ("fla", "FL"),
+            ("ny", "NY"),
+            ("n.y.", "NY"),
+            ("tex", "TX"),
+            ("penn", "PA"),
+            ("mass", "MA"),
+            ("wash", "WA"),
+            ("washington state", "WA"),
+            ("d.c.", "DC"),
+            ("dc", "DC"),
+            ("washington dc", "DC"),
+            ("district of columbia", "DC"),
+            # Edge cases with whitespace
+            ("  california  ", "CA"),
+            (" NY ", "NY"),
+            ("	texas	", "TX"),  # tabs
+            # Invalid/unrecognized values should be returned as-is
+            ("invalid_state", "invalid_state"),
+            ("123", "123"),
+            ("XY", "XY"),  # Not a valid state abbreviation
+            ("some random text", "some random text"),
+            # All 50 states + DC (spot check a few more)
+            ("alabama", "AL"),
+            ("alaska", "AK"),
+            ("arizona", "AZ"),
+            ("wyoming", "WY"),
+            ("hawaii", "HI"),
+            ("vermont", "VT"),
+        ],
+    )
+    def test_normalize_state_to_abbreviation(self, state_value, expected):
+        from src.utils.string_utils import normalize_state_to_abbreviation
+
+        result = normalize_state_to_abbreviation(state_value)
+        assert result == expected
+
+    def test_all_valid_abbreviations_preserved(self):
+        """Test that all valid 2-letter state abbreviations are preserved when uppercase"""
+        from src.utils.string_utils import normalize_state_to_abbreviation
+
+        valid_abbreviations = [
+            "AL",
+            "AK",
+            "AZ",
+            "AR",
+            "CA",
+            "CO",
+            "CT",
+            "DE",
+            "FL",
+            "GA",
+            "HI",
+            "ID",
+            "IL",
+            "IN",
+            "IA",
+            "KS",
+            "KY",
+            "LA",
+            "ME",
+            "MD",
+            "MA",
+            "MI",
+            "MN",
+            "MS",
+            "MO",
+            "MT",
+            "NE",
+            "NV",
+            "NH",
+            "NJ",
+            "NM",
+            "NY",
+            "NC",
+            "ND",
+            "OH",
+            "OK",
+            "OR",
+            "PA",
+            "RI",
+            "SC",
+            "SD",
+            "TN",
+            "TX",
+            "UT",
+            "VT",
+            "VA",
+            "WA",
+            "WV",
+            "WI",
+            "WY",
+            "DC",
+        ]
+
+        for abbrev in valid_abbreviations:
+            result = normalize_state_to_abbreviation(abbrev)
+            assert result == abbrev, f"Failed for abbreviation: {abbrev}"
+
+            # Also test lowercase versions
+            result_lower = normalize_state_to_abbreviation(abbrev.lower())
+            assert (
+                result_lower == abbrev
+            ), f"Failed for lowercase abbreviation: {abbrev.lower()}"
+
+    def test_case_insensitivity(self):
+        """Test that the function handles various case combinations"""
+        from src.utils.string_utils import normalize_state_to_abbreviation
+
+        test_cases = [
+            ("california", "CA"),
+            ("California", "CA"),
+            ("CALIFORNIA", "CA"),
+            ("CaLiFoRnIa", "CA"),
+            ("new york", "NY"),
+            ("New York", "NY"),
+            ("NEW YORK", "NY"),
+            ("NeW yOrK", "NY"),
+        ]
+
+        for input_state, expected in test_cases:
+            result = normalize_state_to_abbreviation(input_state)
+            assert result == expected, f"Failed for: {input_state}"
+
+    def test_edge_cases(self):
+        """Test edge cases and boundary conditions"""
+        from src.utils.string_utils import normalize_state_to_abbreviation
+
+        # Test with just whitespace
+        assert normalize_state_to_abbreviation("   ") == "   "
+
+        # Test with mixed valid/invalid content
+        assert normalize_state_to_abbreviation("california123") == "california123"
+
+        # Test very long strings
+        long_string = "this is a very long string that is not a state"
+        assert normalize_state_to_abbreviation(long_string) == long_string
+
+        # Test single characters
+        assert normalize_state_to_abbreviation("C") == "C"
+        assert normalize_state_to_abbreviation("A") == "A"