Merged in feature/active-rates (pull request #1007)

Feature/active rates

* amendment intent tag

* prompt update

* Merge remote-tracking branch 'origin/dev' into feature/active-rates

* Merge remote-tracking branch 'origin/dev' into feature/active-rates

* Merge branch 'dev' into feature/active-rates

* exhibit standardization updates

* use AARETE_DERIVED_EXHIBIT_TITLE for intent

* active rates stuff

* Merge branch 'dev' into feature/active-rates

* prompt update

* amendment intent types

* active rates logic update

* unit tests

* Merged dev into feature/active-rates

* logic updates

* Merge remote-tracking branch 'origin/dev' into feature/active-rates

* Merge remote-tracking branch 'origin/dev' into feature/active-rates

* updated instrumentation cost logs

* cost loging

* caching updates

* Merge remote-tracking branch 'origin/dev' into feature/active-rates

* Merge remote-tracking branch 'origin/dev' into feature/active-rates

* logging fix

* null check issue fixes

* caching fix


Approved-by: Praneel Panchigar
Approved-by: Siddhant Medar

src/pipelines/shared/postprocessing/active_rates.py

@@ -199,9 +199,13 @@ def add_active_rate_flags(df: pd.DataFrame) -> pd.DataFrame:
     # by service term rather than exhibit title.
 
     if "AMENDMENT_INTENT_LANGUAGE" in df.columns and "AMENDMENT_INTENT" in df.columns:
+        from src.pipelines.shared.postprocessing.postprocessing_funcs import (
+            _language_is_na,
+        )
+
         _files_no_lang = set(
             df.groupby("FILE_NAME")["AMENDMENT_INTENT_LANGUAGE"]
-            .apply(lambda s: s.isna().all())
+            .apply(lambda s: s.apply(_language_is_na).all())
             .pipe(lambda x: x[x].index)
         )
 

src/pipelines/shared/postprocessing/postprocessing_funcs.py

@@ -751,6 +751,37 @@ _VALID_INTENT_TAGS = {
     "UNCLEAR_REVIEW",
 }
 
+_NA_LANGUAGE_STRINGS = {"", "n/a", "nan"}
+
+
+def _language_is_na(val) -> bool:
+    """Return True when val carries no actionable amendment language.
+
+    Handles four storage forms that appear in the DataFrame:
+      - None / float NaN / pd.NA / pd.NaT  (any pandas/Python missing value)
+      - Python list, e.g. ["1. N/A"]  (parsed from LLM JSON response)
+      - JSON-array string, e.g. '["N/A"]'  (serialised form)
+      - Plain string, e.g. "N/A"
+    """
+    if isinstance(val, list):
+        if not val:
+            return True
+        # Strip leading enumeration ("1. ", "2. ") before checking each item.
+        return all(
+            re.sub(r"^\d+\.\s*", "", str(item)).strip().lower() in _NA_LANGUAGE_STRINGS
+            for item in val
+        )
+    # pd.isna covers None, float NaN, pd.NA, and pd.NaT for scalar values.
+    # Must come after the list branch because pd.isna on a list returns an array.
+    if pd.isna(val):
+        return True
+    s = str(val).strip()
+    # JSON-array strings like '["N/A"]' or '["1. N/A"]'
+    if s.startswith("[") and s.endswith("]"):
+        inner = re.sub(r"^\d+\.\s*", "", s[1:-1].strip().strip("\"'")).strip()
+        return inner.lower() in _NA_LANGUAGE_STRINGS
+    return s.lower() in _NA_LANGUAGE_STRINGS
+
 
 def add_amendment_intent(df: pd.DataFrame) -> pd.DataFrame:
     """
@@ -814,29 +845,35 @@ def add_amendment_intent(df: pd.DataFrame) -> pd.DataFrame:
 
         candidate_languages = df.loc[title_mask, "AMENDMENT_INTENT_LANGUAGE"]
         candidate_languages = candidate_languages[
-            ~candidate_languages.astype(str).str.strip().isin({"", "N/A", "n/a", "nan"})
+            ~candidate_languages.apply(_language_is_na)
-            & ~candidate_languages.isna()
         ]
+        _raw = candidate_languages.iloc[0] if len(candidate_languages) > 0 else ""
         amendment_language = (
-            candidate_languages.iloc[0] if len(candidate_languages) > 0 else ""
+            "\n".join(str(item) for item in _raw)
+            if isinstance(_raw, list)
+            else str(_raw)
         )
 
-        if string_utils.is_empty(str(amendment_language)) or str(
+        if string_utils.is_empty(amendment_language) or amendment_language.strip() in {
-            amendment_language
+            "N/A",
-        ).strip() in {"N/A", "n/a", "nan"}:
+            "n/a",
+            "nan",
+        }:
             intent_map[map_key] = "N/A"
             reason_map[map_key] = "no_language"
             continue
 
         try:
             prompt = prompt_templates.AMENDMENT_INTENT(
-                str(exhibit_title), str(amendment_language)
+                str(exhibit_title), amendment_language
             )
             raw = llm_utils.invoke_claude(
                 prompt,
                 "sonnet_latest",
-                filename,
+                file_name if has_file else "",
                 max_tokens=150,
+                cache=True,
+                instruction=prompt_templates.AMENDMENT_INTENT_INSTRUCTION(),
                 usage_label="AMENDMENT_INTENT",
             )
             parsed = json_utils.parse_json_dict(raw)

src/prompts/prompt_templates.py

@@ -3105,11 +3105,7 @@ def AMENDMENT_INTENT_INSTRUCTION() -> str:
 
 
 def AMENDMENT_INTENT(exhibit_title: str, amendment_language: str) -> str:
-    return (
+    return f"Amendment changes:\n{amendment_language}\n\nExhibit: {exhibit_title}"
-        f"Amendment changes:\n{amendment_language}\n\n"
-        f"Exhibit: {exhibit_title}\n\n"
-        f"{AMENDMENT_INTENT_INSTRUCTION()}"
-    )
 
 
 #####################################################################################

src/tests/test_active_rates.py

@@ -282,6 +282,18 @@ class TestAmendmentLanguageNormalisation(unittest.TestCase):
         # ["N/A"] is treated as no language → fallback OVERLAY_UPDATE applied
         self.assertEqual(result["AMENDMENT_INTENT"].iloc[0], "OVERLAY_UPDATE")
 
+    def test_placeholder_list_language_becomes_overlay_update(self):
+        # Python list ["1. N/A"] is placeholder language — not a real null, but
+        # _language_is_na should treat it as no language so fallback applies.
+        df = _df(
+            QC_RANK=["1.1"],
+            QC_FLAG=["Child"],
+            AMENDMENT_INTENT=["N/A"],
+            AMENDMENT_INTENT_LANGUAGE=[["1. N/A"]],
+        )
+        result = add_active_rate_flags(df)
+        self.assertEqual(result["AMENDMENT_INTENT"].iloc[0], "OVERLAY_UPDATE")
+
 
 # ── add_amendment_intent ──────────────────────────────────────────────────────
 
@@ -363,6 +375,23 @@ class TestAddAmendmentIntent(unittest.TestCase):
         mock_invoke.assert_not_called()
         self.assertEqual(result["AMENDMENT_INTENT"].iloc[0], "N/A")
 
+    @patch(
+        "src.pipelines.shared.postprocessing.postprocessing_funcs.llm_utils.invoke_claude"
+    )
+    def test_pd_na_language_yields_na_intent_without_llm(self, mock_invoke):
+        # pd.NA must be treated as missing — not stringified to "<NA>" and sent to LLM.
+        df = pd.DataFrame(
+            {
+                "FILE_NAME": ["f1.pdf"],
+                "AARETE_DERIVED_EXHIBIT_TITLE": ["Ex A"],
+                "AMENDMENT_INTENT_LANGUAGE": [pd.NA],
+                "QC_FLAG": ["Child"],
+            }
+        )
+        result = add_amendment_intent(df)
+        mock_invoke.assert_not_called()
+        self.assertEqual(result["AMENDMENT_INTENT"].iloc[0], "N/A")
+
     @patch(
         "src.pipelines.shared.postprocessing.postprocessing_funcs.llm_utils.invoke_claude"
     )

src/utils/instrumentation.py

@@ -91,6 +91,9 @@ USAGE_LABEL_TO_SEGMENT: dict[str, str] = {
     "prompt_provider_info": "one_to_one",
     "CHECK_PROVIDER_NAME_MATCH": "one_to_one",  # prompt_calls.py:1042
     "EXTRACT_AMENDMENT_NUM_FROM_FILENAME": "one_to_one",  # hybrid_smart_chunking_funcs.py:404
+    # AMENDMENT_INTENT_LANGUAGE: per-document extraction of verbatim amendment change list.
+    # Fires from prompt_templates.AMENDMENT_INTENT_LANGUAGE (prompt_templates.py:3020).
+    "AMENDMENT_INTENT_LANGUAGE": "one_to_one",
     # DATE_FIX / DERIVED_TERM_DATE run inside run_one_to_one_prompts, deriving
     # AARETE_DERIVED_EFFECTIVE_DT / AARETE_DERIVED_TERMINATION_DT from raw dates.
     # Classified by *when* they fire (one_to_one phase), not their postprocess-like role.
@@ -109,6 +112,15 @@ USAGE_LABEL_TO_SEGMENT: dict[str, str] = {
     # --- postprocess (runs after all per-contract extraction completes) ---
     "AARETE_DERIVED_PAYER_NAME": "postprocess",  # runner.py:290 (post-run clustering)
     "AARETE_DERIVED_PROVIDER_NAME": "postprocess",  # runner.py:300 (post-run clustering)
+    # --- active_rates (batch-level, runs after parent-child mapping) ---
+    # exhibit_title_standardization: 2-pass LLM taxonomy derivation + mapping.
+    # Fires only from active_rates._standardize_and_derive_intent (exhibit_title_standardization.py:68).
+    "exhibit_title_standardization": "active_rates",
+    # AMENDMENT_INTENT: per-(file, exhibit_title) LLM classification into
+    # FULL_REPLACEMENT / PARTIAL_REPLACEMENT / ADDITIVE / UNCLEAR_REVIEW.
+    # Fires from active_rates._standardize_and_derive_intent (postprocessing_funcs.py:829)
+    # and also from postprocess.run_postprocessing (postprocess.py:131) per-contract.
+    "AMENDMENT_INTENT": "active_rates",
 }
 
 

src/utils/llm_utils.py

@@ -632,10 +632,9 @@ def invoke_claude(
                     segment=segment,
                     model=model_id,
                     prompt_chars=len(prompt) if prompt else 0,
-                    context_chars=(
+                    context_chars=len(context_for_caching or instruction or ""),
-                        len(context_for_caching) if context_for_caching else 0
+                    has_context_cache=cache
-                    ),
+                    and bool(context_for_caching or instruction),
-                    has_context_cache=bool(context_for_caching),
                     cache_hit_inmem=True,
                     **_ctx_minus_explicit_keys(
                         dict(ctx),
@@ -659,8 +658,8 @@ def invoke_claude(
     try:
         with instr_scope(
             _usage_label=usage_label,
-            _has_context_cache=bool(context_for_caching),
+            _has_context_cache=cache and bool(context_for_caching or instruction),
-            _context_chars=len(context_for_caching) if context_for_caching else 0,
+            _context_chars=len(context_for_caching or instruction or ""),
             _prompt_chars=len(prompt) if prompt else 0,
             _cache_key=cache_key,
         ):