From 2da639e59b756c30bfaf22f4862f636a95d80829 Mon Sep 17 00:00:00 2001 From: Venkatakrishna Reddy Avula Date: Fri, 8 May 2026 15:52:32 +0000 Subject: [PATCH] Merged in feature/DAIP2-2314-DAIP2-1687-hybrid (pull request #993) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Feature/DAIP2-2314 DAIP2 1687 hybrid * remove -files from s3 prefix requirements * Resolve input paths * fix: VendorProcessor.process_file returns (df, None) tuple runner.safe_process_file unpacks the result as (cc_df, dashboard_df), so returning a single DataFrame caused every vendor/generic file to fail with "too many values to unpack (expected 2)" — Python iterates DataFrame columns during unpacking. Vendor pipelines have no dashboard variant; second slot is None and the existing `dashboard_result is not None` guard in runner.py already handles it. Co-Authored-By: Claude Opus 4.7 (1M context) * DAIP2-2314 + DAIP2-1687: pad DYNAMIC_PRIMARY + DYNAMIC_PRIMARY_ENTITY_CLASSIFICATION over 1024-token cache floor - Pad DYNAMIC_PRIMARY_INSTRUCTION with three new sections: [SCOPE BOUNDARIES], [SOURCE TEXT INTERPRETATION], [REASONING DISCIPLINE], plus a [WORKED EXAMPLES] block. Estimated tokens: 447 -> 1117 (Sonnet 4.5 1024-min, +93 margin). All additions reinforce existing rules (sibling-field separation, alias mapping, pricing-vs-LOB distinction, contrastive-clause exclusion, exhibit-header binding) — no new directives that could bias extraction. - Pad DYNAMIC_PRIMARY_ENTITY_CLASSIFICATION_INSTRUCTION with a [FINAL CHECKLIST BEFORE OUTPUT] block. Estimated tokens: 956 -> 1101 (Sonnet 4.5 1024-min, +77 margin). Reinforces the existing 4-step anti-duplication protocol and JSON shape requirements. - Register both new entries in cache_registry: DYNAMIC_PRIMARY_ENTITY_CLASSIFICATION as INSTRUCTION_PLUS_CONTEXT (caches at warm-up), DYNAMIC_PRIMARY_ENTITIES as CONTEXT (instruction is intentionally short; CONTEXT c… * black format fix * Merged dev into feature/DAIP2-2314-DAIP2-1687-hybrid * fixed raw lob values in base lob field mapping and composite entities fix * black format fix * fixed LOB Program output issues * issue fixes * remove debugging code * Updated prompts * updated additional instructions * Update Program-->LOB * LLM-based AD-Program/Product mapping to LOB even when there is a crosswalk * black format fix * Merged dev into feature/DAIP2-2314-DAIP2-1687-hybrid * added logging in prompt call tracking * added updated logging in prompt call tracking * aaded min cache token per usage label * added cache registry for dynamic primary mapping prompt calls * reolved mapping prompts ambiguities * black format fix * Phase 2 modifications added * reverted phase 2 modifications Approved-by: Katon Minhas --- documentation/PROMPT_CACHING_ANALYSIS.md | 204 +++ .../PROMPT_CACHING_SYSTEM_PROPOSAL.md | 154 ++ scripts/merge_files.py | 123 ++ src/codes/code_funcs.py | 4 +- src/config.py | 19 +- src/constants/constants.py | 43 +- src/constants/mappings/crosswalk_lob.json | 1 + .../mappings/crosswalk_product_lob.json | 96 -- src/constants/mappings/crosswalk_program.json | 269 ---- .../mappings/crosswalk_program_lob.json | 224 --- src/crosswalk/crosswalk_builder.py | 32 +- src/document_classification/main.py | 25 +- src/pipelines/runner.py | 70 +- src/pipelines/saas/file_processing.py | 36 +- src/pipelines/saas/prompts/prompt_calls.py | 251 +++- .../shared/extraction/dynamic_funcs.py | 207 +-- .../shared/extraction/one_to_n_funcs.py | 386 ++++- src/pipelines/shared/extraction/row_funcs.py | 4 - .../shared/postprocessing/aarete_derived.py | 262 ++-- .../shared/postprocessing/postprocess.py | 5 +- .../postprocessing/postprocessing_funcs.py | 11 +- .../hybrid_smart_chunking_funcs.py | 15 +- .../vendors/shared/generic_processor.py | 14 +- src/prompts/cache_registry.py | 622 ++++++++ src/prompts/fieldset.py | 7 +- src/prompts/investment_prompts.json | 25 +- src/prompts/prompt_templates.py | 1298 +++++++++++++++-- src/scripts/prompt_call_log_report.py | 254 ++++ src/tests/test_crosswalk_utils.py | 22 - src/tests/test_one_to_n.py | 6 - src/tests/test_postprocess.py | 45 +- src/utils/llm_utils.py | 275 +++- src/utils/program_product_mapping.py | 156 ++ src/utils/prompt_call_tracking.py | 354 +++++ 34 files changed, 4393 insertions(+), 1126 deletions(-) create mode 100644 documentation/PROMPT_CACHING_ANALYSIS.md create mode 100644 documentation/PROMPT_CACHING_SYSTEM_PROPOSAL.md create mode 100755 scripts/merge_files.py delete mode 100644 src/constants/mappings/crosswalk_product_lob.json delete mode 100644 src/constants/mappings/crosswalk_program.json delete mode 100644 src/constants/mappings/crosswalk_program_lob.json create mode 100644 src/prompts/cache_registry.py create mode 100644 src/scripts/prompt_call_log_report.py create mode 100644 src/utils/program_product_mapping.py create mode 100644 src/utils/prompt_call_tracking.py diff --git a/documentation/PROMPT_CACHING_ANALYSIS.md b/documentation/PROMPT_CACHING_ANALYSIS.md new file mode 100644 index 0000000..9eeb75b --- /dev/null +++ b/documentation/PROMPT_CACHING_ANALYSIS.md @@ -0,0 +1,204 @@ +# Prompt Caching Analysis Report + +**Branch:** `feature/DAIP2-2314-expand-caching-for-short-prompts` +**Date:** 2026-04-21 +**Comparison:** Feature branch vs `origin/dev` + +--- + +## Executive Summary + +This report analyzes the prompt caching implementation in the feature branch, comparing cached prompts against the dev branch and evaluating cost savings from Bedrock prompt caching. + +### Key Findings + +| Metric | Value | +|--------|-------| +| Total API Calls Analyzed | 1,251 | +| Unique Usage Labels | 35 | +| Total Cache Read Tokens | 1,472,330 | +| Total Cache Write Tokens | 87,167 | +| Overall Cost Savings | **28.1%** ($3.65 saved) | +| Prompts Actively Caching | 22 | +| Prompts Wired but Not Caching | 11 | +| Prompts Not Wired | 2 | + +--- + +## 1. Cache Effectiveness by Usage Label + +### Actively Caching (22 prompts) - High Performers + +| Usage Label | Calls | Avg Instruction Tokens | Cache Hit Rate | +|-------------|-------|------------------------|----------------| +| OUTLIER_BREAKOUT | 1 | 2,483 | 98.4% | +| SERVICE_ENRICHMENT | 30 | 1,459 | 98.3% | +| CARVEOUT_CHECK | 43 | 1,498 | 95.6% | +| METHODOLOGY_BREAKOUT | 37 | 1,927 | 95.5% | +| LESSER_OF_DISTRIBUTION | 32 | 914 | 94.5% | +| CHECK_PROVIDER_NAME_MATCH | 74 | 957 | 94.2% | +| DYNAMIC_CODE_ASSIGNMENT | 38 | 966 | 94.1% | +| CODE_EXPLICIT | 34 | 871 | 93.9% | +| GROUPER_BREAKOUT | 7 | 939 | 93.6% | +| LESSER_OF_CHECK | 6 | 900 | 92.0% | +| DYNAMIC_ASSIGNMENT | 204 | 1,578 | 89.4% | +| validate_reimbursements_for_llm | 65 | 894 | 89.2% | +| code_implicit_rag | 60 | 896 | 88.4% | +| REIMB_DATES_ASSIGNMENT | 29 | 803 | 87.2% | +| REIMBURSEMENT_PRIMARY | 38 | 1,390 | 85.0% | +| EXHIBIT_LEVEL | 33 | 1,108 | 74.0% | +| EXHIBIT_HEADER | 90 | 922 | 73.7% | +| fill_bill_type | 47 | 968 | 72.5% | +| prompt_provider_info | 24 | 966 | 60.5% | +| DYNAMIC_PRIMARY | 44 | 441 | 49.7% | + +### Wired But Not Caching (11 prompts) - Under 1024 Token Threshold + +These prompts are wired for caching but have instruction tokens below Bedrock's 1024-token minimum threshold: + +| Usage Label | Calls | Avg Instruction Tokens | Issue | +|-------------|-------|------------------------|-------| +| SPLIT_SERVICE_TERM | 10 | 1,600 | Should be caching - investigate | +| prompt_lob_relationship | 56 | 408 | **Needs padding to 1024** | +| SPLIT_REIMB_DATES | 13 | 410 | **Needs padding to 1024** | +| code_implicit_arbitration | 12 | 391 | **Needs padding to 1024** | +| AARETE_DERIVED_PAYER_NAME | 1 | 372 | **Needs padding to 1024** | +| FEE_SCHEDULE_BREAKOUT | 18 | 352 | **Needs padding to 1024** | +| code_implicit_special | 30 | 329 | **Needs padding to 1024** | +| DATE_FIX | 10 | 322 | **Needs padding to 1024** | +| SPECIAL_CASE_ASSIGNMENT | 4 | 251 | **Needs padding to 1024** | +| DERIVED_TERM_DATE | 10 | 247 | **Needs padding to 1024** | +| code_last_check | 23 | 179 | **Needs padding to 1024** | +| EXTRACT_AMENDMENT_NUM_FROM_FILENAME | 3 | 152 | **Needs padding to 1024** | +| EXHIBIT_HEADER_DEDUP | 10 | 35 | **Needs padding to 1024** | + +### Not Wired for Caching (2 prompts) + +| Usage Label | Calls | Notes | +|-------------|-------|-------| +| prompt_hsc_single_field | 109 | High volume - should investigate wiring | +| SPECIAL_CASE_BREAKOUT | 6 | Low volume | + +--- + +## 2. Prompt Template Changes (Feature Branch vs Dev) + +The feature branch includes **649 lines changed** in `src/prompts/prompt_templates.py`. Key expansions: + +### Expanded Prompts (Additional Context Added) + +| Instruction Function | Change Summary | +|---------------------|----------------| +| **EXHIBIT_LEVEL_INSTRUCTION** | +55 lines: Added detailed guidance, validation rules, examples, common pitfalls, field-specific notes, quality assurance checks | +| **DYNAMIC_PRIMARY_INSTRUCTION** | +18 lines: Added extraction guidance, ambiguity handling, quality checks | +| **LESSER_OF_DISTRIBUTION_INSTRUCTION** | Restructured (net -25 lines): Converted to cleaner decision framework format | +| **DYNAMIC_CODE_ASSIGNMENT_INSTRUCTION** | +10 lines: Added classification guidance with explicit rules | +| **CODE_IMPLICIT_INSTRUCTION** | Reformatted indentation | +| **CODE_IMPLICIT_ARBITRATION_INSTRUCTION** | +43 lines: Added clarifications, deterministic decision rules, calibration examples | +| **FILL_BILL_TYPE_INSTRUCTION** | +49 lines: Added detailed guidance, validation rules, extended examples | +| **TIN_NPI_TEMPLATE_INSTRUCTION** | +22 lines: Added provider/payer distinction clarity | + +### Analysis of Prompt Changes + +**Positive Impacts:** +- More explicit instructions reduce model ambiguity +- Quality checks and validation rules improve consistency +- Examples help calibrate model responses +- Clearer formatting improves readability + +**Potential Concerns:** +- Expanded prompts increase token count (higher cache write cost initially) +- Some prompts may have reduced due to restructuring (LESSER_OF_DISTRIBUTION) +- Need to verify output quality hasn't degraded with expanded context + +--- + +## 3. Cost Analysis + +### Bedrock Pricing Applied +- **Input tokens:** $3.00/1M tokens +- **Output tokens:** $15.00/1M tokens +- **Cache read:** $0.30/1M tokens (90% savings) +- **Cache write:** $3.75/1M tokens (25% premium) + +### Observed Results + +| Metric | Value | +|--------|-------| +| Total Input Tokens | 844,370 | +| Total Output Tokens | 401,926 | +| Total Cache Read Tokens | 1,472,330 | +| Total Cache Write Tokens | 87,167 | +| **Cost Without Caching** | $12.98 | +| **Cost With Caching** | $9.33 | +| **Savings** | $3.65 (28.1%) | + +### Per-Prompt Cost Impact (Estimated from previous testing) + +| Prompt | Cost Reduction | +|--------|---------------| +| LESSER_OF_DISTRIBUTION | 44% (Best performer) | +| EXHIBIT_HEADER | 7% | +| CHECK_PROVIDER_NAME_MATCH | 5% | +| TIN_NPI_TEMPLATE (prompt_provider_info) | 5% | +| code_implicit_rag | -12% (Negative - needs review) | + +--- + +## 4. Prompts Needing Attention + +### Priority 1: Prompts Needing Padding (Under 1024 tokens) + +These prompts are wired but not actually caching due to Bedrock's minimum threshold: + +1. **prompt_lob_relationship** (408 tokens, 56 calls) - High impact +2. **code_implicit_special** (329 tokens, 30 calls) - Medium impact +3. **code_last_check** (179 tokens, 23 calls) - Medium impact +4. **FEE_SCHEDULE_BREAKOUT** (352 tokens, 18 calls) - Medium impact +5. **SPLIT_REIMB_DATES** (410 tokens, 13 calls) +6. **code_implicit_arbitration** (391 tokens, 12 calls) +7. **SPLIT_SERVICE_TERM** (1600 tokens, 10 calls) - Should be caching, investigate +8. **DATE_FIX** (322 tokens, 10 calls) +9. **DERIVED_TERM_DATE** (247 tokens, 10 calls) +10. **EXHIBIT_HEADER_DEDUP** (35 tokens, 10 calls) + +### Priority 2: Investigate Negative Result + +- **code_implicit_rag**: Shows -12% cost (increase) - The expanded prompt may be too verbose or structured differently causing cache misses + +### Priority 3: Not Wired for Caching + +- **prompt_hsc_single_field** (109 calls) - High volume, should consider wiring + +--- + +## 5. Recommendations + +1. **Pad short prompts to 1024 tokens**: Add contextual padding to prompts under the threshold, especially high-frequency ones like `prompt_lob_relationship` and `code_implicit_special` + +2. **Investigate code_implicit_rag**: The negative cost result (-12%) suggests the expanded prompt may be causing cache fragmentation or misses + +3. **Wire prompt_hsc_single_field**: With 109 calls, this is a good candidate for caching + +4. **Monitor SPLIT_SERVICE_TERM**: At 1,600 tokens it should be caching but shows 0% cache hit rate - may be a wiring issue + +5. **Run testbed comparison**: Use `src/testbed/testbed_metrics.py` to compare output quality between cached and non-cached prompts to ensure no regression + +--- + +## 6. Files Analyzed + +- `src/test-PROMPT-CALLS_10.csv` - API call logs with cache statistics +- `src/testbed-usethis.xlsx` - Testbed comparison file (binary, not readable) +- `src/prompts/prompt_templates.py` - Prompt definitions (649 lines changed) +- `documentation/prompt_caching_tracker_v2.csv` - Caching status tracker + +--- + +## 7. Next Steps + +1. Run testbed metrics comparison script to validate output quality +2. Add padding to short prompts identified above +3. Investigate and fix code_implicit_rag negative result +4. Wire prompt_hsc_single_field for caching +5. Debug SPLIT_SERVICE_TERM caching issue diff --git a/documentation/PROMPT_CACHING_SYSTEM_PROPOSAL.md b/documentation/PROMPT_CACHING_SYSTEM_PROPOSAL.md new file mode 100644 index 0000000..e2dfbba --- /dev/null +++ b/documentation/PROMPT_CACHING_SYSTEM_PROPOSAL.md @@ -0,0 +1,154 @@ +# Prompt Caching System Proposal + +Branch: `feature/DAIP2-2314-expand-caching-for-short-prompts` + +## Goal + +Ensure every stable instruction prompt that is intended to be cached is actually cacheable, warmed on the same model used at runtime, and observable in runtime logs. + +## Recommended Caching System + +1. Introduce a central prompt cache registry. + + Each cacheable prompt should be declared once with: + - `usage_label` + - instruction function + - model family used at runtime + - minimum cache token policy + - runtime call sites + - whether the prompt also uses `context_for_caching` + + This avoids the current split where cache warming is in `prompt_templates.get_cacheable_instructions()` but runtime model choice is scattered through prompt call sites. + +2. Enforce the 1024-token minimum before runtime. + + Prompt caching should not be treated as enabled just because `cache=True` is passed. At startup, the registry should validate every cached instruction with the same tokenizer/estimate used by reporting. If a prompt is below the minimum, it should either: + - fail fast in a cache audit mode, or + - automatically append approved static cache padding/instruction context until it crosses the threshold. + + The padding must be stable and domain-relevant, not dynamic request data. + +3. Warm caches per model, not globally. + + Cache entries are model-specific. The runner currently warms all instructions with `sonnet_latest`, but `SPLIT_SERVICE_TERM` runs with `haiku_latest`. The cache warming layer should warm `(usage_label, resolved_model_id)` pairs derived from the registry. + +4. Only mark supported models as cacheable. + + `_supports_prompt_cache()` currently excludes Haiku. If Haiku prompt caching is not supported in this Bedrock setup, Haiku prompts should not be registered as cached. Either move those prompt calls to `sonnet_latest` when caching matters, or keep them uncached and report them as intentionally uncached. + +5. Keep static instruction caching separate from dynamic context caching. + + Instruction prompts can be cached predictably. `context_for_caching` is different because it changes by contract, exhibit, or page. It should be tracked as a separate cache class: + - `instruction_cache`: expected to read after warm-up + - `context_cache`: expected to create/read per repeated context + + This prevents context variability from making instruction caching look broken. + +6. Add a cache contract test. + + A local test should inspect the built request body for every registered prompt and assert: + - the instruction block has `cache_control` + - the resolved model supports caching + - the static instruction token estimate is at least 1024 + - warm-up model equals runtime model + +7. Upgrade runtime reporting. + + Add these fields to prompt-call logs: + - `resolved_model_id` + - `cache_policy`: `instruction`, `context`, `instruction+context`, `none` + - `instruction_cache_eligible` + - `context_cache_eligible` + - `cache_miss_reason` + + This makes root cause visible without manual investigation. + +## Prompt Application Report + +### Apply Instruction Caching, Already Over 1024 Tokens + +These should remain registered and warmed. They are good candidates for strict cache contract tests. + +| Prompt | Estimated Instruction Tokens | Action | +|---|---:|---| +| `OUTLIER_BREAKOUT` | 2483 | Keep cached | +| `METHODOLOGY_BREAKOUT` | 1927 | Keep cached | +| `SPLIT_SERVICE_TERM` | 1600 | Cache only if runtime model supports caching; currently runs on Haiku | +| `DYNAMIC_ASSIGNMENT` | 1578 | Keep cached | +| `CARVEOUT_CHECK` | 1498 | Keep cached | +| `SERVICE_ENRICHMENT` | 1459 | Keep cached | +| `REIMBURSEMENT_PRIMARY` | 1390 | Keep cached | +| `EXHIBIT_LEVEL` | 1108 | Keep cached | + +### Apply Instruction Caching After Padding/Expansion + +These are wired or partially wired but below the 1024-token minimum. They should be padded or expanded with stable domain guidance before being considered cached. + +| Prompt | Estimated Instruction Tokens | Tokens Needed | Action | +|---|---:|---:|---| +| `fill_bill_type` | 968 | 56 | Add small static guidance | +| `DYNAMIC_CODE_ASSIGNMENT` | 966 | 58 | Add small static guidance | +| `CHECK_PROVIDER_NAME_MATCH` | 957 | 67 | Add small static guidance | +| `GROUPER_BREAKOUT` | 939 | 85 | Add small static guidance | +| `EXHIBIT_HEADER` | 922 | 102 | Add static header examples/rules | +| `LESSER_OF_DISTRIBUTION` | 914 | 110 | Add static decision examples | +| `LESSER_OF_CHECK` | 900 | 124 | Add static classification examples | +| `CODE_EXPLICIT` | 871 | 153 | Add static code-system rules | +| `REIMB_DATES_ASSIGNMENT` | 803 | 221 | Add date extraction examples | +| `DYNAMIC_PRIMARY` | 441 | 583 | Expand instruction substantially | +| `SPLIT_REIMB_DATES` | 410 | 614 | Expand instruction substantially | +| `prompt_lob_relationship` | 408 | 616 | Move field prompt into instruction or add static LOB rules | +| `code_implicit_arbitration` | 391 | 633 | Expand arbitration rules | +| `AARETE_DERIVED_PAYER_NAME` | 372 | 652 | Expand payer naming rules | +| `FEE_SCHEDULE_BREAKOUT` | 352 | 672 | Expand fee schedule rules | +| `code_implicit_special` | 329 | 695 | Expand special code rules | +| `DATE_FIX` | 322 | 702 | Expand date normalization rules | +| `SPECIAL_CASE_ASSIGNMENT` | 251 | 773 | Expand special case rules | +| `DERIVED_TERM_DATE` | 247 | 777 | Expand term date rules | +| `code_last_check` | 179 | 845 | Expand or leave uncached if low value | +| `EXTRACT_AMENDMENT_NUM_FROM_FILENAME` | 152 | 872 | Expand or leave uncached if low value | +| `EXHIBIT_HEADER_DEDUP` | 35 | 989 | Do not cache unless redesigned | + +### Wire for Caching or Mark Intentionally Uncached + +These showed runtime volume but no instruction function match in the audit. They need explicit registry decisions. + +| Prompt | Runtime Finding | Action | +|---|---|---| +| `prompt_hsc_single_field` | 109 calls, `cache_enabled=False`, large dynamic prompt | Split stable HSC instructions from dynamic chunk and cache the instruction | +| `SPECIAL_CASE_BREAKOUT` | 6 calls, no instruction cache | Wire only if repeated volume justifies expansion | +| `code_implicit_rag` | Cached in API, audit alias missing | Add registry alias to the actual instruction function | +| `prompt_provider_info` | Cached in API, audit alias missing | Add registry alias to `TIN_NPI_TEMPLATE_INSTRUCTION` or rename usage label | +| `validate_reimbursements_for_llm` | Cached in API, audit alias missing | Add registry alias to `VALIDATE_REIMBURSEMENTS_INSTRUCTION` | + +## Immediate Fixes + +1. Change `SPLIT_SERVICE_TERM` to use a cache-supported model or mark it intentionally uncached. Current behavior is misleading: `cache=True` is passed, but no Bedrock cache tokens are recorded. + +2. Add a registry alias map for usage labels that do not match instruction names: + - `code_implicit_rag` -> `CODE_IMPLICIT` + - `prompt_provider_info` -> `TIN_NPI_TEMPLATE` + - `validate_reimbursements_for_llm` -> `VALIDATE_REIMBURSEMENTS` + - `prompt_lob_relationship` -> its real instruction function + +3. Expand the near-threshold instructions first because they need the least work and have current evidence of API cache reads: + - `fill_bill_type` + - `DYNAMIC_CODE_ASSIGNMENT` + - `CHECK_PROVIDER_NAME_MATCH` + - `GROUPER_BREAKOUT` + - `EXHIBIT_HEADER` + - `LESSER_OF_DISTRIBUTION` + - `LESSER_OF_CHECK` + - `CODE_EXPLICIT` + +4. Redesign `prompt_hsc_single_field` into `(instruction, prompt)` form. It is high-volume and currently sends all prompt text dynamically. + +## Expected Behavior After Fix + +For instruction-only prompts, after warm-up the API should report cache reads on normal calls. The cached percentage will still not be 100% of total billed input because the dynamic user prompt remains uncached. The correct target is: + +- `cache_creation_tokens` only during warm-up or first uncached use +- `cache_read_tokens > 0` on every eligible runtime call +- `cache_miss_reason` empty for registered instruction cache prompts + +For context-cached prompts, 100% cache reads should not be expected unless the same context block repeats. The report should evaluate those separately. diff --git a/scripts/merge_files.py b/scripts/merge_files.py new file mode 100755 index 0000000..21b1651 --- /dev/null +++ b/scripts/merge_files.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +""" +Script to merge multiple text files into a single file. + +Usage: + python merge_files.py ... + python merge_files.py --dir [--pattern ] + +This script concatenates the contents of the input files into the output file, +separating each file's content with a newline. +For CSV files, it properly merges by keeping only the header from the first file. +""" + +import sys +import os +import glob +import csv + +def merge_files(file_list, output_file): + """ + Merge a list of files into a single output file. + + Args: + file_list (list): List of file paths to merge. + output_file (str): Path to the output file. + """ + if not file_list: + return + + # Check if files are CSV based on extension + is_csv = any(fname.lower().endswith('.csv') for fname in file_list) + + if is_csv: + merge_csv_files(file_list, output_file) + else: + merge_text_files(file_list, output_file) + +def merge_text_files(file_list, output_file): + """ + Merge text files by concatenating their contents. + """ + with open(output_file, 'w', encoding='utf-8') as outfile: + for fname in file_list: + if not os.path.isfile(fname): + print(f"Warning: {fname} is not a file or does not exist. Skipping.") + continue + try: + with open(fname, 'r', encoding='utf-8') as infile: + content = infile.read() + outfile.write(content) + outfile.write('\n') # Add a newline separator between files + except Exception as e: + print(f"Error reading {fname}: {e}") + +def merge_csv_files(file_list, output_file): + """ + Merge CSV files by keeping header from first file and appending data from others. + """ + first_file = True + + with open(output_file, 'w', newline='', encoding='utf-8') as outfile: + writer = None + + for fname in file_list: + if not os.path.isfile(fname): + print(f"Warning: {fname} is not a file or does not exist. Skipping.") + continue + + try: + with open(fname, 'r', encoding='utf-8') as infile: + reader = csv.reader(infile) + + for row_num, row in enumerate(reader): + if first_file or row_num > 0: # Skip header for subsequent files + if writer is None: + writer = csv.writer(outfile) + writer.writerow(row) + + first_file = False + + except Exception as e: + print(f"Error reading {fname}: {e}") + +def get_files_from_dir(directory, pattern='*'): + """ + Get all files from a directory matching a pattern. + + Args: + directory (str): Directory path. + pattern (str): Glob pattern for files. + + Returns: + list: List of file paths. + """ + if not os.path.isdir(directory): + print(f"Error: {directory} is not a directory.") + return [] + return glob.glob(os.path.join(directory, pattern)) + +if __name__ == "__main__": + if len(sys.argv) < 3: + print("Usage: python merge_files.py ...") + print(" or: python merge_files.py --dir [--pattern ]") + sys.exit(1) + + output_file = sys.argv[1] + + if sys.argv[2] == '--dir': + if len(sys.argv) < 4: + print("Usage: python merge_files.py --dir [--pattern ]") + sys.exit(1) + directory = sys.argv[3] + pattern = sys.argv[5] if len(sys.argv) > 5 and sys.argv[4] == '--pattern' else '*' + input_files = get_files_from_dir(directory, pattern) + else: + input_files = sys.argv[2:] + + if not input_files: + print("No input files found.") + sys.exit(1) + + merge_files(input_files, output_file) + print(f"Merged {len(input_files)} files into {output_file}") diff --git a/src/codes/code_funcs.py b/src/codes/code_funcs.py index 25e04f5..c622d31 100644 --- a/src/codes/code_funcs.py +++ b/src/codes/code_funcs.py @@ -732,8 +732,8 @@ def fill_grouper_cd_desc(answer_dict: dict, constants: Constants) -> dict: for code in grouper_cd: try: - code_without_severity = int( - code.split("-")[0] + code_without_severity = str( + int(code.split("-")[0]) ) # Remove severity level if present except Exception as e: logging.warning( diff --git a/src/config.py b/src/config.py index 11563b2..1398e20 100644 --- a/src/config.py +++ b/src/config.py @@ -178,8 +178,16 @@ CLIENT = [s.strip() for s in get_arg_value("client", "None").split("-") if s] LOCAL_PATH = get_arg_value( "input_dir", "data/test" ) # Valid: any valid path that contains txt files -S3_PREFIX = get_arg_value("s3_prefix", "") +S3_PREFIX = get_arg_value("s3_prefix", "") + "/txt/" S3_BUCKET = get_arg_value("s3_bucket", "") + +# program_product_lob_mapping.csv is expected in the same S3 prefix as the txt files. +PROGRAM_PRODUCT_MAPPING_CSV_KEY = ( + get_arg_value("s3_prefix", "") + "/program_product_lob_mapping.csv" + if S3_PREFIX + else None +) + OUTPUT_DIRECTORY = get_arg_value( "output_dir", "output_individual" ) # Valid: any valid directory @@ -338,6 +346,11 @@ def resolve_model_id(model_identifier: str) -> str: ENABLE_ANSWER_TRACKING = False ANSWER_TRACKING_DICT = {} +# Runtime prompt-call tracking (one row per invoke_claude call) +ENABLE_PROMPT_CALL_LOGGING = ( + get_arg_value("enable_prompt_call_logging", "True") == "True" +) + ######################################## POSTPROCESSING SETTINGS ######################################## FUZZY_MATCH_THRESHOLD = 0.8 FILE_NAME_COLUMN = "Contract Name" @@ -437,9 +450,9 @@ RUN_BASE_PREFIX = get_arg_value( if RUN_BASE_PREFIX: _run_base = RUN_BASE_PREFIX.rstrip("/") + "/" if not S3_PREFIX: - S3_PREFIX = f"{_run_base}txt-files/" + S3_PREFIX = f"{_run_base}txt/" if not PDF_S3_PREFIX: - PDF_S3_PREFIX = f"{_run_base}pdf-files/" + PDF_S3_PREFIX = f"{_run_base}pdf/" if not PDF_S3_BUCKET: PDF_S3_BUCKET = S3_BUCKET diff --git a/src/constants/constants.py b/src/constants/constants.py index 7afa328..3a15105 100644 --- a/src/constants/constants.py +++ b/src/constants/constants.py @@ -1,10 +1,17 @@ +from __future__ import annotations + from pathlib import Path +from typing import TYPE_CHECKING + from sentence_transformers import SentenceTransformer from src.crosswalk.crosswalk_builder import CrosswalkBuilder from src.crosswalk.list_builder import ListBuilder from src.constants.embedding import CodeEmbedding +if TYPE_CHECKING: + from src.utils.program_product_mapping import ProgramProductMapping + # Absolute path to src/ directory _SRC_DIR = Path(__file__).parent.parent.resolve() @@ -17,7 +24,7 @@ def _path(relative_path: str) -> str: class Constants: """Class to manage all constants and mappings for a Doczy.ai execution""" - def __init__(self): + def __init__(self, program_product_mapping: "ProgramProductMapping | None" = None): #################################### Dynamic and Exhibit-Level #################################### @@ -27,20 +34,30 @@ class Constants: ) self.VALID_LOBS = list(self.CROSSWALK_LOB.mapping.keys()) - self.CROSSWALK_PROGRAM = CrosswalkBuilder().from_json( - path=_path("constants/mappings/crosswalk_program.json") - ) - self.VALID_PROGRAMS = list(self.CROSSWALK_PROGRAM.mapping.keys()) - self.CROSSWALK_NETWORK = CrosswalkBuilder().from_json( path=_path("constants/mappings/crosswalk_network.json") ) self.VALID_NETWORKS = list(self.CROSSWALK_NETWORK.mapping.keys()) - self.CROSSWALK_PRODUCT = CrosswalkBuilder().from_json( - path=_path("constants/mappings/crosswalk_product_lob.json") + # AARETE_DERIVED_PROGRAM → AARETE_DERIVED_LOB and AARETE_DERIVED_PRODUCT → AARETE_DERIVED_LOB: + # Built from program_product_lob_mapping.csv loaded at startup. Empty when no mapping. + mapping = program_product_mapping + self.CROSSWALK_PROGRAM_LOB = CrosswalkBuilder().from_dict( + mapping=mapping.program_to_lob if mapping else {}, + from_col="AARETE_DERIVED_PROGRAM", + to_col="AARETE_DERIVED_LOB", + ) + self.CROSSWALK_PRODUCT_LOB = CrosswalkBuilder().from_dict( + mapping=mapping.product_to_lob if mapping else {}, + from_col="AARETE_DERIVED_PRODUCT", + to_col="AARETE_DERIVED_LOB", + ) + self.VALID_AARETE_DERIVED_PROGRAMS = ( + mapping.valid_aarete_derived_programs if mapping else [] + ) + self.VALID_AARETE_DERIVED_PRODUCTS = ( + mapping.valid_aarete_derived_products if mapping else [] ) - self.VALID_PRODUCTS = list(self.CROSSWALK_PRODUCT.mapping.keys()) # Dynamic Codes self.CROSSWALK_PROV_SPECIALTY_CD = CrosswalkBuilder().from_json( @@ -269,7 +286,6 @@ class Constants: return [ v.upper() for v in self.VALID_LOBS - + self.VALID_PRODUCTS + list( set( CrosswalkBuilder() @@ -278,13 +294,6 @@ class Constants: ) ) + self.VALID_CLAIM_TYPE - + list( - set( - CrosswalkBuilder() - .from_json(path=_path("constants/mappings/crosswalk_program.json")) - .mapping.values() - ) - ) + ["INPATIENT", "OUTPATIENT", "IN-PATIENT", "OUT-PATIENT", "IP", "OP"] + [ "COVERED SERVICES", diff --git a/src/constants/mappings/crosswalk_lob.json b/src/constants/mappings/crosswalk_lob.json index 8fadd75..02ecfaf 100644 --- a/src/constants/mappings/crosswalk_lob.json +++ b/src/constants/mappings/crosswalk_lob.json @@ -6,6 +6,7 @@ "mapping": { "Medicaid" : "Medicaid", "Medicare" : "Medicare", + "Medicare Advantage (MA)" : "Medicare", "Duals" : "Duals", "Medicare-Medicaid (MM)" : "Duals", "Commercial" : "Commercial", diff --git a/src/constants/mappings/crosswalk_product_lob.json b/src/constants/mappings/crosswalk_product_lob.json deleted file mode 100644 index 091f0be..0000000 --- a/src/constants/mappings/crosswalk_product_lob.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "metadata": { - "from_col": "AARETE_DERIVED_PRODUCT", - "to_col": "AARETE_DERIVED_LOB" - }, - "mapping": { - }, - "client_mapping" : { - "Healthfirst" : { - "Medicare Select" : "Medicare" - }, - "Centene" : { - - }, - "Aetna" : { - - }, - "CHC" : { - - }, - "Fidelis" : { - - }, - "Parkland" : { - "HEALTHfirst" : "Medicaid", - "KIDSfirst" : "Medicaid" - }, - "Molina" :{ - "Molina Medicare Options" : "Medicare", - "Molina Medicare Options Plus" : "Duals", - "Molina Health Benefit Exchange" : "Marketplace" - }, - "Humana" : { - - }, - "Moda" : { - "Synergy" : "Commercial", - "Connexus" : "Commercial", - "Beacon" : "Commercial", - "Community Care Network" : "Commercial", - "OHSU" : "Commercial", - "PEBB" : "Commercial", - "OEBB" : "Commercial", - "Affinity" : "Marketplace", - "Moda Select" : "Marketplace" - }, - "Healthnet" : { - "Indemnity" : "Commercial", - "Ambetter" : "Commercial", - "Healthy Families Benefit Program" : "Medicaid", - "Salud con Health Net" : "" - }, - "Trillium" : { - "Universal" : "Medicaid" - }, - "Sentara" : { - - }, - "CareSource" : { - - }, - "BCBS" : { - "adultBasic" : "Medicaid", - "Affordablue" : "Marketplace", - "Keystone Health Plan West" : "Commercial", - "PremierBlue" : "Commercial", - "Choice Blue" : "Commercial", - "Special Care Program" : "Marketplace", - "Classic Blue" : "Commercial", - "Preferred Blue" : "Commercial", - "Direct Blue" : "Commercial", - "Select Blue" : "Commerical", - "Community Blue" : "Commercial", - "Connect Blue" : "Commercial", - "My Blue Access" : "Marketplace", - "Short Term Blue" : "Commercial", - "Advance Blue" : "Commercial", - "Simply Blue" : "Commercial", - "Essential" : "Medicaid|Commercial", - "Essential Plan - Aliessa" : "Medicaid", - "Essential Plan - QHP" : "Commercial", - "Exchange" : "Marketplace", - "First Priority Health" : "Commercial", - "Highmark Choice Company" : "Commercial", - "Protection Plus" : "Commercial", - "Super Blue" : "Commercial", - "Super Blue Plus" : "Commercial", - "Super Blue Select" : "Commercial", - "Quality Blue" : "Commercial|Medicare", - "Blue High Performance Network (BlueHPN)" : "Commercial", - "Together Blue" : "Marketplace", - "Traditional" : "Commercial", - "True Performance Program" : "Commercial|Medicare" - } - } -} \ No newline at end of file diff --git a/src/constants/mappings/crosswalk_program.json b/src/constants/mappings/crosswalk_program.json deleted file mode 100644 index c6f9c1d..0000000 --- a/src/constants/mappings/crosswalk_program.json +++ /dev/null @@ -1,269 +0,0 @@ -{ - "metadata": { - "from_col": "PROGRAM", - "to_col": "AARETE_DERIVED_PROGRAM" - }, - "mapping": { - "Children's Health Insurance Program (CHIP)" : "CHIP", - "Children's Health Insurance Program Perinate (CHIP-P)" : "CHIPP", - "Healthy Kids" : "CHIP", - "Medicaid Managed Care" : "MMC", - "Affordable Care Act (ACA)" : "ACA", - "Aged, Blind, and Disabled (ABD)" : "ABD", - "Alternative Benefit Plan (ABP)" : "ABP", - "Community First Choice (CFC)" : "CFC", - "Foster Care (FC)" : "FC", - "Home and Community Based Services Waiver (HCBS)" : "HCBS", - "Intellectual and Developmental Disability (IDD)" : "IDD", - "Institution for Medical Disease (IMD)" : "IMD", - "Managed Long Term Care (MLTC)" : "MLTC", - "Self-Directed Personal Assistant Services (PAS)" : "PAS", - "Psych Under 21 (PU21)" : "PU21", - "Supplemental Nutrition Assistance Program (SNAP)" : "SNAP", - "Temporary Assistance Needy Families (TANF)" : "TANF", - "Long-Term Services and Supports (LTSS)" : "LTSS", - "Special Needs Plan (SNP)" : "SNP", - "Medicare Advantage Special Needs Plan (MA-SNP)" : "MASNP", - "Chronic Condition Special Needs Plan (CSNP)" : "CSNP", - "Dual Eligible Special Needs Plan (DSNP)" : "DSNP", - "Fully Integrated Dual Eligible Special Needs Plan (FIDE-SNP)" : "FIDE SNP", - "Highly Integrated Dual Eligible Special Needs Plan (HIDE-SNP)" : "HIDE SNP", - "Institutional Special Needs Plan (I-SNP)" : "ISNP", - "Medicare-Medicaid Alignment Initiative (MMAI)" : "MMAI", - "Medicare-Medicaid Coordinated Plan (MMCP)" : "MMCP", - "Medicare-Medicaid Plan (MMP)" : "MMP", - "Program of All-Inclusive Care for the Elderly (PACE)" : "PACE", - "Advantage" : "MA", - "Medicare Advantage" : "MA", - "Medicare Part C" : "MA", - "Supplemental" : "Supp", - "Medigap" : "Supp", - "Self-Funded" : "SF", - "Administrative Services Only (ASO)" : "SF", - "Third Party Administrator (TPA)" : "SF", - "Fully-Funded" : "FF", - "Platinum" : "PLAT", - "Gold" : "GOLD", - "Silver" : "SLVR", - "Bronze" : "BRNZ", - "Connector" : "CONN", - "Catastrophic" : "CATA" - }, - "state_mapping" : { - "AL" : { - "ALL Kids" : "CHIP" - }, - "AK" : { - "Denali KidCare" : "CHIP", - "DenaliCare" : "DENALI" - }, - "AZ" : { - "Arizona Health Care Cost Containment System (AHCCCS)" : "AHCCCS", - "KidsCare" : "CHIP" - }, - "AR" : { - "ARKids First" : "CHIP" - }, - "CA" : { - "Medi-Cal" : "MEDI_CAL", - "Covered California" : "COVEREDCAL", - "Medi-Cal Access Program (MCAP)" : "CHIP", - "Healthy Families Benefit Program" : "CHIP" - }, - "CO" : { - "Health First Colorado" : "HFCO", - "Connect for Health Colorado" : "CFHCO", - "Child Health Plan Plus (CHP+)" : "CHIP" - }, - "CT" : { - "Husky Health" : "HUSKY", - "Access Health CT" : "AHCT", - "HUSKY A" : "CHIP", - "HUSKY B" : "CHIP", - "HUSKY C" : "CHIP", - "HUSKY D" : "CHIP" - }, - "DE" : { - "Diamond State Health Plan" : "DSHP", - "Choose Health Delaware" : "CHDE", - "Delaware Healthy Children Program (DHCP)" : "CHIP" - }, - "DC" : { - "DC Health Link" : "DCHL", - "DC Healthy Families" : "CHIP" - }, - "FL" : { - "Florida Statewide Medicaid Managed Care Program (SMMC)" : "SMMC", - "Florida KidCare" : "CHIP" - }, - "GA" : { - "Georgia Access" : "GAACCESS", - "PeachCare for Kids" : "CHIP" - }, - "HI" : { - "Med-Quest" : "MQUEST", - "QUEST Integration" : "CHIP" - }, - "ID" : { - "Your Health Idaho" : "YHID", - "Idaho Medicaid for CHIP" : "CHIP" - }, - "IL" : { - "HealthChoice Illinois" : "HCIL", - "Get Covered Illinois" : "GCIL", - "All Kids" : "CHIP" - }, - "IN" : { - "Hoosier Healthwise" : "CHIP" - }, - "IA" : { - "Iowa Health Link" : "IAHL", - "Healthy and Well Kids in Iowa (HAWK-I)" : "CHIP" - }, - "KS" : { - "KanCare" : "KANCARE" - }, - "KY" : { - "Kynect" : "KYNECT", - "Kentucky Children's Health Insurance Program (KCHIP)" : "CHIP" - }, - "LA" : { - "Healthy Louisiana" : "HEALTHYLA", - "Louisiana State Insurance Exchange" : "LASIE", - "LaCHIP" : "CHIP" - }, - "ME" : { - "MaineCare" : "MECARE", - "Cover ME" : "COVERME", - "Cub Care" : "CHIP" - }, - "MD" : { - "Maryland HealthChoice" : "MDHC", - "Maryland Children's Health Program (MCHP)" : "CHIP" - }, - "MA" : { - "MassHealth" : "MASSHEALTH", - "MassHealth CHIP" : "CHIP" - }, - "MI" : { - "Healthy Michigan Plan" : "HMIP", - "MI Child" : "CHIP" - }, - "MN" : { - "Minnesota Medical Assistance" : "MNMA", - "MNSure" : "MNSURE", - "MinnesotaCare" : "CHIP" - }, - "MS" : { - "MississippiCAN" : "MSCAN", - "Mississippi CHIP" : "CHIP" - }, - "MO" : { - "MO HealthNet" : "MOHN", - "MO HealthNet for Kids" : "CHIP" - }, - "MT" : { - "Healthy Montana Kids (HMK)" : "CHIP" - }, - "NE" : { - "Heritage Health" : "HHNE", - "Kids Connection" : "CHIP" - }, - "NV" : { - "Nevada Health Link" : "NVHL", - "Nevada Check Up" : "CHIP" - }, - "NH" : { - "NH Health Families" : "NHHF", - "New Hampshire Healthy Families" : "CHIP" - }, - "NJ" : { - "FamilyCare" : "NJFC", - "Get Covered NJ" : "GCNJ" - }, - "NM" : { - "Centennial Care" : "CCNM", - "WellNM" : "WELLNM", - "New MexiKids" : "CHIP" - }, - "NY" : { - "NY State of Health" : "NYSOH", - "Health and Recovery Plan (HARP)" : "HARP", - "Child Health Plus (CHP)" : "CHPLUS", - "Essential Plan" : "EPNY", - "New York State Health Insurance Exchange Qualified Health Plan" : "QHP", - "Basic Health Plan" : "BHP" - }, - "NC" : { - "NC Health Choice" : "CHIP" - }, - "ND" : { - "Healthy Steps" : "CHIP" - }, - "OH" : { - "Healthy Start" : "CHIP", - "MyCare Ohio" : "MYCAREOH" - }, - "OK" : { - "SoonerCare" : "SOONER", - "SoonerSelect" : "SOONER" - }, - "OR" : { - "Oregon Health Plan" : "ORHP", - "OregonHealthCare" : "ORHC" - }, - "PA" : { - "Pennsylvania Medical Assistance" : "PAMA", - "Pennie" : "PENNIE", - "Pennsylvania CHIP" : "CHIP" - }, - "RI" : { - "HealthSource RI" : "HSRI", - "Rite Care" : "CHIP" - }, - "SC" : { - "Partners for Healthy Children" : "CHIP" - }, - "SD" : { - "South Dakota Medicaid for CHIP" : "CHIP" - }, - "TN" : { - "TennCare" : "TENNCARE", - "CoverKids" : "CHIP" - }, - "TX" : { - "STAR" : "STAR", - "STAR Health" : "STAR", - "STAR Kids" : "STAR Kids", - "STAR+PLUS" : "STAR+PLUS" - }, - "UT" : { - "Healthy U" : "HEALTHYU", - "Healthy Kids Plus" : "CHIP" - }, - "VT" : { - "Green Mountain Care" : "GMCARE", - "Dr. Dynasaur" : "CHIP" - }, - "VA" : { - "Cardinal Care" : "CCVA", - "Cover Virginia" : "COVERVA", - "FAMIS" : "CHIP" - }, - "WA" : { - "Apple Health" : "AHWA", - "Washington Healthplanfinder" : "WAHPF", - "Apple Health for Kids" : "CHIP" - }, - "WV" : { - "WVCHIP" : "CHIP" - }, - "WI" : { - "BadgerCare Plus" : "BADGER" - }, - "WY" : { - "EqualityCare" : "EQCAREWY", - "Kid Care CHIP" : "CHIP" - } - } -} \ No newline at end of file diff --git a/src/constants/mappings/crosswalk_program_lob.json b/src/constants/mappings/crosswalk_program_lob.json deleted file mode 100644 index 0ec9edb..0000000 --- a/src/constants/mappings/crosswalk_program_lob.json +++ /dev/null @@ -1,224 +0,0 @@ -{ - "metadata": { - "from_col": "AARETE_DERIVED_PROGRAM", - "to_col": "AARETE_DERIVED_LOB" - }, - "mapping": { - "CHIP" : "Medicaid", - "CHIPP" : "Medicaid", - "MMC" : "Medicaid", - "ACA" : "Marketplace", - "ABD" : "Medicaid", - "ABP" : "Medicaid", - "CFC" : "Medicaid", - "FC" : "Medicaid", - "HCBS" : "Medicaid", - "HHS" : "Medicaid", - "HOSPICE" : "Medicaid", - "IDD" : "Medicaid", - "IMD" : "Medicaid", - "MLTC" : "Medicaid", - "PAS" : "Medicaid", - "PU21" : "Medicaid", - "SNAP" : "Medicaid", - "TANF" : "Medicaid", - "LTSS" : "Medicaid", - "BHP" : "Medicaid", - "SNP" : "Duals", - "CSNP" : "Duals", - "DSNP" : "Duals", - "FIDE SNP" : "Duals", - "HIDE SNP" : "Duals", - "ISNP" : "Duals", - "MMAI" : "Duals", - "MMCP" : "Duals", - "MMP" : "Duals", - "PACE" : "Duals", - "MA" : "Medicare", - "MASNP" : "Medicare", - "Supp" : "Medicare", - "SF" : "Commercial", - "FF" : "Commercial", - "PLAT" : "Marketplace", - "GOLD" : "Marketplace", - "SLVR" : "Marketplace", - "BRNZ" : "Marketplace", - "CONN" : "Marketplace", - "CATA" : "Marketplace", - "QHP" : "Marketplace" - }, - "state_mapping" : { - "AL" : { - - }, - "AK" : { - "DENALI" : "Medicaid" - }, - "AZ" : { - "AHCCCS" : "Medicaid" - }, - "AR" : { - - }, - "CA" : { - "MEDI_CAL" : "Medicaid", - "COVEREDCAL" : "Marketplace" - }, - "CO" : { - "HFCO" : "Medicaid", - "CFHCO" : "Marketplace" - }, - "CT" : { - "HUSKY" : "Medicaid", - "AHCT" : "Marketplace" - }, - "DE" : { - "DSHP" : "Medicaid", - "CHDE" : "Marketplace" - }, - "DC" : { - "DCHL" : "Marketplace" - }, - "FL" : { - "SMMC" : "Medicaid" - }, - "GA" : { - "GAACCESS" : "Marketplace" - }, - "HI" : { - "MQUEST" : "Medicaid" - }, - "ID" : { - "YHID" : "Marketplace" - }, - "IL" : { - "HCIL" : "Marketplace", - "GCIL" : "Marketplace" - }, - "IN" : { - - }, - "IA" : { - "IAHL" : "Medicaid" - }, - "KS" : { - "KANCARE" : "Medicaid" - }, - "KY" : { - "KYNECT" : "Marketplace" - }, - "LA" : { - "HEALTHYLA" : "Medicaid", - "LASIE" : "Marketplace" - }, - "ME" : { - "MECARE" : "Medicaid", - "COVERME" : "Marketplace" - }, - "MD" : { - "MDHC" : "Medicaid" - }, - "MA" : { - "MASSHEALTH" : "Medicaid" - }, - "MI" : { - "HMIP" : "Medicaid" - }, - "MN" : { - "MNMA" : "Medicaid", - "MNSURE" : "Marketplace" - }, - "MS" : { - "MSCAN" : "Medicaid" - }, - "MO" : { - "MOHN" : "Medicaid" - }, - "MT" : { - - }, - "NE" : { - "HHNE" : "Medicaid" - }, - "NV" : { - "NVHL" : "Marketplace" - }, - "NH" : { - "NHHF" : "Marketplace" - }, - "NJ" : { - "NJFC" : "Medicaid", - "GCNJ" : "Marketplace" - }, - "NM" : { - "CCNM" : "Medicaid", - "WELLNM" : "Marketplace" - }, - "NY" : { - "NYSOH" : "Marketplace", - "HARP" : "Medicaid", - "CHPLUS" : "Medicaid", - "EPNY" : "Medicaid" - }, - "NC" : { - - }, - "ND" : { - - }, - "OH" : { - "MYCAREOH" : "Duals" - }, - "OK" : { - "SOONER" : "Medicaid" - }, - "OR" : { - "ORHP" : "Medicaid", - "ORHC" : "Marketplace" - }, - "PA" : { - "PENNIE" : "Marketplace", - "PAMA" : "Medicaid" - }, - "RI" : { - "HSRI" : "Marketplace" - }, - "SC" : { - - }, - "SD" : { - - }, - "TN" : { - "TENNCARE" : "Medicaid" - }, - "TX" : { - "STAR" : "Medicaid", - "STAR+PLUS" : "Medicaid", - "STAR Kids" : "Medicaid" - }, - "UT" : { - "HEALTHYU" : "Medicaid" - }, - "VT" : { - "GMCARE" : "Medicaid" - }, - "VA" : { - "CCVA" : "Medicaid", - "COVERVA" : "Marketplace" - }, - "WA" : { - "AHWA" : "Medicaid", - "WAHPF" : "Marketplace" - }, - "WV" : { - - }, - "WI" : { - "BADGER" : "Medicaid" - }, - "WY" : { - "EQCAREWY" : "Medicaid" - } - } -} \ No newline at end of file diff --git a/src/crosswalk/crosswalk_builder.py b/src/crosswalk/crosswalk_builder.py index dd1c87b..7560845 100644 --- a/src/crosswalk/crosswalk_builder.py +++ b/src/crosswalk/crosswalk_builder.py @@ -2,7 +2,6 @@ import csv import json import pandas as pd -from src.config import CLIENT, STATE class CrosswalkBuilder: @@ -108,36 +107,7 @@ class CrosswalkBuilder: self.metadata = data.get("metadata", {}) - mapping = data.get("mapping", {}) - - if "state_mapping" in data: - for state in STATE: - mapping.update(data.get("state_mapping", {}).get(state, {})) - if "client_mapping" in data: - client_mapping = data.get("client_mapping", {}) - clients = [ - str(c).strip() - for c in CLIENT - if c is not None and str(c).strip().lower() != "none" - ] - - if clients: - for client in clients: - mapping.update(client_mapping.get(client, {})) - else: - # No explicit client provided: merge all client mappings with deduplication. - # Keep first value for duplicate keys. - merged_client_mapping: dict[str, str] = {} - for client_map in client_mapping.values(): - if not isinstance(client_map, dict): - continue - - for key, value in client_map.items(): - merged_client_mapping.setdefault(key, value) - - mapping.update(merged_client_mapping) - - self.mapping = mapping + self.mapping = data.get("mapping", {}) return self diff --git a/src/document_classification/main.py b/src/document_classification/main.py index 1cee58b..0199837 100644 --- a/src/document_classification/main.py +++ b/src/document_classification/main.py @@ -22,6 +22,7 @@ from src.pipelines.shared.preprocessing.preprocessing_funcs import ( ) from src.prompts.fieldset import Field from src.utils import io_utils, llm_utils, duplicate_detection +import src.utils.program_product_mapping as ppm_utils from src import config # Load prompts from JSON using Field class (with thread-safe caching) @@ -59,17 +60,14 @@ _MAPPINGS_DIR = os.path.join(os.path.dirname(__file__), "..", "constants", "mapp _LOB_JSON_FILES = [ "crosswalk_lob.json", - "crosswalk_program_lob.json", - "crosswalk_product_lob.json", ] def _load_lob_keywords_from_mappings(): """Load LOB keywords and their normalized LOB values from mapping JSON files. - Reads crosswalk_lob.json, crosswalk_program_lob.json, and crosswalk_product_lob.json - and extracts all keyword -> normalized_LOB pairs from their mapping, state_mapping, - and client_mapping sections. + Reads crosswalk_lob.json and extracts all keyword -> normalized_LOB pairs from their + mapping, state_mapping, and client_mapping sections. Returns: dict: Mapping of keyword (str) -> normalized LOB category (str). @@ -986,6 +984,21 @@ def generate_pre_doczy_report(input_dict, run_timestamp): f"{master_dup_count} master filename duplicates" ) + # ── Load program/product mapping for PROGRAM and PRODUCT summary fields ── + programs_list: list = [] + products_list: list = [] + if config.S3_BUCKET and config.PROGRAM_PRODUCT_MAPPING_CSV_KEY: + _mapping = ppm_utils.load_from_s3( + config.S3_BUCKET, config.PROGRAM_PRODUCT_MAPPING_CSV_KEY + ) + if _mapping is not None: + programs_list = _mapping.valid_aarete_derived_programs + products_list = _mapping.valid_aarete_derived_products + df["PROGRAM"] = [programs_list] * len(df) + df["PRODUCT"] = [products_list] * len(df) + print("Program/Product mapping loaded for summary fields") + print(f"Sample PROGRAM values: {programs_list}") + print(f"Sample PRODUCT values: {products_list}") col_order = [ "FILE_NAME", "STATUS", @@ -1000,6 +1013,8 @@ def generate_pre_doczy_report(input_dict, run_timestamp): "LOB_PAGES", "LOB_KEYWORDS_MATCHED", "LOB_DERIVED", + "PROGRAM", + "PRODUCT", ] df = df[[c for c in col_order if c in df.columns]] diff --git a/src/pipelines/runner.py b/src/pipelines/runner.py index 548fa13..aea8426 100644 --- a/src/pipelines/runner.py +++ b/src/pipelines/runner.py @@ -28,6 +28,7 @@ random.seed(42) warnings.filterwarnings("ignore", message=".*protected namespace.*") import src.prompts.prompt_templates as prompt_templates +import src.prompts.cache_registry as cache_registry import src.utils.instrumentation as instrumentation import src.utils.io_utils as io_utils import src.utils.llm_utils as llm_utils @@ -35,6 +36,7 @@ import src.utils.logging_utils as logging_utils import src.utils.usage_tracking as usage_tracking import src.utils.timing_utils as timing_utils import src.utils.duplicate_detection as duplicate_detection +import src.utils.program_product_mapping as ppm_utils from src.pipelines.runtime_context import set_active_client from src.pipelines.shared.extraction import one_to_one_funcs from src.pipelines.shared.postprocessing import postprocessing_funcs @@ -167,17 +169,33 @@ def main(client: str = "saas", testing=False, test_params={}): # Windows paths cannot contain ':'; use '-' in time component run_timestamp = datetime.now().strftime(f"run_%Y%m%d_%H-%M_{config.BATCH_ID}") + config.PROMPT_CALL_RUN_TIMESTAMP = run_timestamp instrumentation.configure_for_run(config.BATCH_ID, run_timestamp) # Track overall pipeline time pipeline_start = time.time() + # Load program_product_lob_mapping.csv once (if available) and inject into Constants + program_product_mapping = None + if config.S3_BUCKET and config.PROGRAM_PRODUCT_MAPPING_CSV_KEY: + with timing_utils.timed_block("load_program_product_mapping", log_level="INFO"): + program_product_mapping = ppm_utils.load_from_s3( + bucket=config.S3_BUCKET, + key=config.PROGRAM_PRODUCT_MAPPING_CSV_KEY, + ) + if program_product_mapping is None: + logging.warning( + f"program_product_lob_mapping.csv not found at " + f"s3://{config.S3_BUCKET}/{config.PROGRAM_PRODUCT_MAPPING_CSV_KEY}; " + "crosswalk/derived mappings will be empty." + ) + # Read Constants with timing_utils.timed_block("read_constants", log_level="INFO"): - constants = Constants() + constants = Constants(program_product_mapping=program_product_mapping) - # Read and process input + # Read and process input — when mapping is loaded, txt files live in the txt_files/ subfolder with timing_utils.timed_block("read_input", log_level="INFO"): if not testing: input_dict = io_utils.read_input() @@ -217,19 +235,53 @@ def main(client: str = "saas", testing=False, test_params={}): return # ========== COMMON: Warm prompt caches ========== + # Drive warming from `cache_registry`, which knows the (usage_label, + # runtime_model) pairing. Cache entries are model-specific in Bedrock, + # so warming `SPLIT_SERVICE_TERM` on `sonnet_latest` while it runs on + # `haiku_latest` was previously paying create-tokens for a cache the + # runtime call could never read. The registry filters to instruction- + # cached entries on cache-supporting models only. logging.info("Warming prompt caches before parallel processing...") with timing_utils.timed_block("warm_prompt_caches", log_level="INFO"): + warmed = 0 + skipped = 0 try: - cacheable_instructions = prompt_templates.get_cacheable_instructions() - for cache_key, instruction in cacheable_instructions.items(): - llm_utils.warm_prompt_cache( - instruction=instruction, - cache_key=cache_key, - model_id="sonnet_latest", + for ( + usage_label, + resolved_model_id, + instruction_text, + ) in cache_registry.iter_warming_targets(): + ok = llm_utils.warm_prompt_cache( + instruction=instruction_text, + cache_key=usage_label, + model_id=resolved_model_id, ) + if ok: + warmed += 1 + else: + skipped += 1 logging.info( - f"Successfully warmed {len(cacheable_instructions)} prompt caches" + "Warmed %d prompt caches (skipped %d already-warm or " + "unsupported pairs).", + warmed, + skipped, ) + + # Surface registry entries whose instruction is below the + # 1024-token Bedrock minimum so the gap is visible in pipeline + # logs, not just at runtime. + below = cache_registry.audit_below_threshold() + if below: + logging.warning( + "Cache audit: %d instruction prompts are below the " + "%d-token minimum: %s", + len(below), + cache_registry.MIN_CACHE_TOKENS, + ", ".join( + f"{label}({tokens}<{minimum})" + for label, tokens, minimum in below + ), + ) except Exception as e: logging.warning(f"Error warming caches (will proceed anyway): {str(e)}") diff --git a/src/pipelines/saas/file_processing.py b/src/pipelines/saas/file_processing.py index 79c0afa..f82d12e 100644 --- a/src/pipelines/saas/file_processing.py +++ b/src/pipelines/saas/file_processing.py @@ -156,7 +156,7 @@ def process_file(file_object, constants: Constants, run_timestamp): "merge_one_to_one_into_one_to_n", context=filename ): if not one_to_n_results.empty: - # BOTH processed - merge into one_to_n + # BOTH processed - merge into one_to_n results for unified output final_results = row_funcs.merge_one_to_one_into_one_to_n( one_to_n_results, one_to_one_results, constants ) @@ -184,6 +184,20 @@ def process_file(file_object, constants: Constants, run_timestamp): final_results = code_funcs.grouper_breakout(results_with_code) logging.debug(f"{datetime_str()} Codes Complete - {filename}") + # Derive AARETE_DERIVED_PROGRAM / AARETE_DERIVED_PRODUCT from PROGRAM / PRODUCT + if not final_results.empty: + with timing_utils.timed_block( + "add_aarete_derived_program_product", context=filename + ): + final_results = one_to_n_funcs.add_aarete_derived_program_product( + final_results, constants, filename + ) + + # Derive AARETE_DERIVED_LOB from AARETE_DERIVED_PROGRAM / AARETE_DERIVED_PRODUCT + if not final_results.empty: + with timing_utils.timed_block("fill_na_mapping", context=filename): + final_results = aarete_derived.fill_na_mapping(final_results, constants) + # POSTPROCESS with ( instrumentation_context.instr_scope(segment="postprocess"), @@ -335,6 +349,23 @@ def run_one_to_one_prompts( one_to_one_results, contract_text ) + ################## Split Dynamic Primary Entities ################## + # If the DYNAMIC_PRIMARY_ENTITIES smart_chunked fallback was triggered (all four + # dynamic primary fields were empty across every one_to_n row), the HSC step above + # will have populated DYNAMIC_PRIMARY_ENTITIES in one_to_one_results. + # Classify it into LOB/PROGRAM/PRODUCT/NETWORK before crosswalk runs so that + # AARETE_DERIVED_LOB and AARETE_DERIVED_NETWORK are derived correctly. + if not string_utils.is_empty(one_to_one_results.get("DYNAMIC_PRIMARY_ENTITIES")): + split_results = one_to_n_funcs.split_dynamic_primary_entities( + [one_to_one_results], filename + ) + one_to_one_results = split_results[0] + + ################## Derive AARETE_DERIVED_LOB/NETWORK ################## + one_to_n_funcs.map_lob_network_to_aarete_derived( + [one_to_one_results], constants, filename + ) + ################## Crosswalk Fields ################## # All field format normalization is handled at prompt_calls level via field-aware parsers # Derived fields (AARETE_DERIVED_*, NUM_*_SIGNATORY_*) are created as strings directly @@ -343,9 +374,6 @@ def run_one_to_one_prompts( [one_to_one_results], constants ) - ################## Fill NA Mapping ################## - one_to_one_results = aarete_derived.fill_na_mapping(one_to_one_results) - return one_to_one_results[0] diff --git a/src/pipelines/saas/prompts/prompt_calls.py b/src/pipelines/saas/prompts/prompt_calls.py index 13187e0..0e2879b 100644 --- a/src/pipelines/saas/prompts/prompt_calls.py +++ b/src/pipelines/saas/prompts/prompt_calls.py @@ -154,6 +154,116 @@ def prompt_dynamic_primary( return llm_answer_final +def prompt_dynamic_primary_entities( + exhibit_text: str, + combined_field_prompt: str, + filename: str, +) -> list[Any]: + """Extract a combined exhibit-level dynamic primary entity list.""" + context_text, prompt, _parser = prompt_templates.DYNAMIC_PRIMARY_ENTITIES( + exhibit_text, + ) + + llm_answer_raw = llm_utils.invoke_claude( + prompt, + "sonnet_latest", + filename, + cache=True, + instruction=prompt_templates.DYNAMIC_PRIMARY_ENTITIES_INSTRUCTION( + combined_field_prompt=combined_field_prompt + ), + context_for_caching=context_text, + usage_label="DYNAMIC_PRIMARY_ENTITIES", + ) + return _parser(llm_answer_raw) + + +def prompt_classify_dynamic_primary_entities( + dynamic_primary_entities: list[str], + filename: str, +) -> dict[str, Any]: + """Classify extracted dynamic primary entities into LOB/PROGRAM/PRODUCT/NETWORK.""" + context_text, prompt, _parser = ( + prompt_templates.DYNAMIC_PRIMARY_ENTITY_CLASSIFICATION( + dynamic_primary_entities, + ) + ) + + llm_answer_raw = llm_utils.invoke_claude( + prompt, + "sonnet_latest", + filename, + cache=True, + instruction=prompt_templates.DYNAMIC_PRIMARY_ENTITY_CLASSIFICATION_INSTRUCTION(), + context_for_caching=context_text, + usage_label="DYNAMIC_PRIMARY_ENTITY_CLASSIFICATION", + ) + + return _parser(llm_answer_raw) + + +def prompt_map_lob_to_valid( + lob_values: list[str], + valid_lobs: list[str], + filename: str, +) -> list[str]: + """Map raw classified LOB values to valid LOB values using LLM. + + Args: + lob_values: List of raw LOB strings from the classification step. + valid_lobs: Allowed output values (from VALID_LOBS constant). + filename: Filename for LLM invocation and logging. + + Returns: + List of matched valid LOB values. Returns an empty list if no matches. + """ + context_text, prompt, _parser = prompt_templates.MAP_LOB_TO_VALID( + lob_values, valid_lobs + ) + llm_answer_raw = llm_utils.invoke_claude( + prompt, + "sonnet_latest", + filename, + cache=True, + instruction=prompt_templates.MAP_LOB_TO_VALID_INSTRUCTION(), + context_for_caching=context_text, + usage_label="MAP_LOB_TO_VALID", + ) + result = _parser(llm_answer_raw) + return result if isinstance(result, list) else [] + + +def prompt_map_network_to_valid( + network_values: list[str], + valid_networks: list[str], + filename: str, +) -> list[str]: + """Map raw classified NETWORK values to valid NETWORK values using LLM. + + Args: + network_values: List of raw NETWORK strings from the classification step. + valid_networks: Allowed output values (from VALID_NETWORKS constant). + filename: Filename for LLM invocation and logging. + + Returns: + List of matched valid NETWORK values. Returns an empty list if no matches. + """ + context_text, prompt, _parser = prompt_templates.MAP_NETWORK_TO_VALID( + network_values, valid_networks + ) + llm_answer_raw = llm_utils.invoke_claude( + prompt, + "sonnet_latest", + filename, + cache=True, + instruction=prompt_templates.MAP_NETWORK_TO_VALID_INSTRUCTION(), + context_for_caching=context_text, + usage_label="MAP_NETWORK_TO_VALID", + ) + result = _parser(llm_answer_raw) + return result if isinstance(result, list) else [] + + def prompt_reimbursement_primary( page_text: str, filename: str, @@ -190,6 +300,7 @@ def prompt_reimbursement_primary( def prompt_dynamic_code_assignment(service_term: str, filename: str): + """Extract dynamic code fields associated with a service term.""" prompt, _parser = prompt_templates.DYNAMIC_CODE_ASSIGNMENT(service_term) logging.debug(f"Dynamic Code Assignment Prompt for {filename}: {prompt}") llm_response = llm_utils.invoke_claude( @@ -204,7 +315,7 @@ def prompt_dynamic_code_assignment(service_term: str, filename: str): try: dynamic_code_assignment_answers = _parser(llm_response) - except ValueError as e: + except ValueError: dynamic_code_assignment_answers = {} return dynamic_code_assignment_answers @@ -234,6 +345,12 @@ def prompt_methodology_breakout( logging.debug(f"LLM Response for {filename}: {llm_response}") try: methodology_breakout_answers = _parser(llm_response) + # Normalize output - methodology_breakout returns list[dict] with methodology fields + # Use FieldSet to get field names from single source of truth + methodology_breakout_fields = FieldSet( + config.FIELD_JSON_PATH, field_type="methodology_breakout" + ) + methodology_field_names = methodology_breakout_fields.list_fields() # Handle both list and single dict cases if isinstance(methodology_breakout_answers, dict): methodology_breakout_answers = [methodology_breakout_answers] @@ -412,6 +529,72 @@ def prompt_lob_relationship( return normalized if normalized else "Exclusive" +def prompt_derive_aarete_program( + program_values: list[str], + valid_aarete_derived_programs: list[str], + filename: str, +) -> list[str]: + """Map PROGRAM values to AARETE_DERIVED_PROGRAM valid values using LLM. + + Args: + program_values: List of raw PROGRAM strings extracted from the contract. + valid_aarete_derived_programs: Allowed output values (from VALID_AARETE_DERIVED_PROGRAMS). + filename: Filename for LLM invocation and logging. + + Returns: + List of matched AARETE_DERIVED_PROGRAM values from the valid values list. + Returns an empty list if no matches are found. + """ + context_text, prompt, _parser = prompt_templates.AARETE_DERIVED_PROGRAM( + program_values, valid_aarete_derived_programs + ) + llm_answer_raw = llm_utils.invoke_claude( + prompt, + "sonnet_latest", + filename, + cache=True, + instruction=prompt_templates.AARETE_DERIVED_PROGRAM_INSTRUCTION(), + context_for_caching=context_text, + usage_label="AARETE_DERIVED_PROGRAM", + ) + result = _parser(llm_answer_raw) + return result if isinstance(result, list) else [] + + +def prompt_derive_aarete_product( + product_values: list[str], + valid_aarete_derived_products: list[str], + filename: str, +) -> list[str]: + """Map PRODUCT values to AARETE_DERIVED_PRODUCT valid values using LLM. + + Args: + product_values: List of raw PRODUCT strings extracted from the contract. + valid_aarete_derived_products: Allowed output values (from VALID_AARETE_DERIVED_PRODUCTS). + filename: Filename for LLM invocation and logging. + + Returns: + List of matched AARETE_DERIVED_PRODUCT values from the valid values list. + Returns an empty list if no matches are found. + """ + context_text, prompt, _parser = prompt_templates.AARETE_DERIVED_PRODUCT( + product_values, valid_aarete_derived_products + ) + llm_answer_raw = llm_utils.invoke_claude( + prompt, + "sonnet_latest", + filename, + cache=True, + instruction=prompt_templates.AARETE_DERIVED_PRODUCT_INSTRUCTION(), + context_for_caching=context_text, + usage_label="AARETE_DERIVED_PRODUCT", + ) + + result = _parser(llm_answer_raw) + + return result if isinstance(result, list) else [] + + def prompt_special_case_assignment( exhibit_text: str, reimbursement_dict: dict, @@ -741,7 +924,7 @@ def prompt_dynamic_assignment( field_name = dynamic_field.field_name field_prompt = dynamic_field.get_prompt(constants) - # Use specialized prompt for REIMB_DATES assignment with context caching + # Use specialized prompts for selected dynamic assignment fields with context caching if field_name == "REIMB_DATES": context_text, prompt, _parser = prompt_templates.REIMB_DATES_ASSIGNMENT( service_term, @@ -752,6 +935,18 @@ def prompt_dynamic_assignment( ) instruction = prompt_templates.REIMB_DATES_ASSIGNMENT_INSTRUCTION() usage_label = "REIMB_DATES_ASSIGNMENT" + elif field_name == "DYNAMIC_PRIMARY_ENTITIES": + context_text, prompt, _parser = ( + prompt_templates.DYNAMIC_PRIMARY_ENTITIES_ASSIGNMENT( + service_term, + reimb_term, + field_prompt, + exhibit_text_simplified, + page_num, + ) + ) + instruction = prompt_templates.DYNAMIC_PRIMARY_ENTITIES_ASSIGNMENT_INSTRUCTION() + usage_label = "DYNAMIC_PRIMARY_ENTITIES_ASSIGNMENT" else: # Use generic DYNAMIC_ASSIGNMENT for other fields with context caching context_text, prompt, _parser = prompt_templates.DYNAMIC_ASSIGNMENT( @@ -1307,9 +1502,15 @@ def prompt_split_service_term(service_term: str, filename: str) -> list[str]: """ prompt, _parser = prompt_templates.SPLIT_SERVICE_TERM(service_term) + # Runs on Sonnet (not Haiku) because Haiku 4.5 requires 4096 tokens + # for cache_control to be honored; the SPLIT_SERVICE_TERM instruction + # is ~1600 tokens, so it would cache on Sonnet (1024-token minimum) + # but never on Haiku. Service-term inputs repeat heavily across + # contracts, so cache reads on Sonnet beat uncached Haiku on cost + # even though Haiku's per-token rate is lower. llm_answer_raw = llm_utils.invoke_claude( prompt, - model_id="haiku_latest", + model_id="sonnet_latest", filename=filename, cache=True, instruction=prompt_templates.SPLIT_SERVICE_TERM_INSTRUCTION(), @@ -1328,3 +1529,47 @@ def prompt_split_service_term(service_term: str, filename: str) -> list[str]: logging.error(f"Error parsing LLM response for SPLIT_SERVICE_TERM: {str(e)}") logging.error(f"Raw response: {llm_answer_raw}") return [] + + +def prompt_program_to_lob( + program_values: list, + valid_lobs: list, + filename: str, + payer_name: str = "", + payer_state: str = "", +) -> list: + """Use LLM to derive LOB values from program values (raw PROGRAM + AARETE_DERIVED_PROGRAM) against the valid LOB list.""" + context, parser = prompt_templates.PROGRAM_TO_LOB( + program_values, valid_lobs, payer_name=payer_name, payer_state=payer_state + ) + llm_answer_raw = llm_utils.invoke_claude( + context, + "sonnet_latest", + filename, + cache=True, + instruction=prompt_templates.PROGRAM_TO_LOB_INSTRUCTION(), + usage_label="PROGRAM_TO_LOB", + ) + return parser(llm_answer_raw) + + +def prompt_product_to_lob( + product_values: list, + valid_lobs: list, + filename: str, + payer_name: str = "", + payer_state: str = "", +) -> list: + """Use LLM to derive LOB values from product values (raw PRODUCT + AARETE_DERIVED_PRODUCT) against the valid LOB list.""" + context, parser = prompt_templates.PRODUCT_TO_LOB( + product_values, valid_lobs, payer_name=payer_name, payer_state=payer_state + ) + llm_answer_raw = llm_utils.invoke_claude( + context, + "sonnet_latest", + filename, + cache=True, + instruction=prompt_templates.PRODUCT_TO_LOB_INSTRUCTION(), + usage_label="PRODUCT_TO_LOB", + ) + return parser(llm_answer_raw) diff --git a/src/pipelines/shared/extraction/dynamic_funcs.py b/src/pipelines/shared/extraction/dynamic_funcs.py index 46c1a21..650b59b 100644 --- a/src/pipelines/shared/extraction/dynamic_funcs.py +++ b/src/pipelines/shared/extraction/dynamic_funcs.py @@ -20,6 +20,16 @@ if TYPE_CHECKING: from src.pipelines.shared.extraction.page_funcs import Page +def _build_dynamic_primary_combined_prompt( + dynamic_primary_fields: FieldSet, constants: Constants +) -> str: + """Build a single combined prompt from the dynamic primary fields in the FieldSet.""" + parts = [] + for field in dynamic_primary_fields.fields: + parts.append(f"{field.field_name}: {field.get_prompt(constants)}") + return "\n\n".join(parts) + + def dynamic_primary( exhibit_text: str, exhibit_level_answers: dict, @@ -47,45 +57,36 @@ def dynamic_primary( dynamic_reimbursement_fields = FieldSet() exhibit_level_answer_dict = {} - for field in list(dynamic_primary_fields.fields): - exhibit_text_answer = prompt_calls.prompt_dynamic_primary( - exhibit_text, - field, - constants, - filename, - prompt_templates.DYNAMIC_PRIMARY, - ) - # If there is at least ONE Non-N/A answers in the Exhibit - if not string_utils.is_empty(exhibit_text_answer): - # Special handling for LOB: If both Medicare and Medicaid are detected, add Duals - if field.field_name == "LOB": - values_lower = [val.lower() for val in exhibit_text_answer] + combined_prompt = _build_dynamic_primary_combined_prompt( + dynamic_primary_fields, constants + ) + dynamic_primary_entities = prompt_calls.prompt_dynamic_primary_entities( + exhibit_text, + combined_prompt, + filename, + ) - # Check if both Medicare and Medicaid are present (case-insensitive) - has_medicare = any("medicare" in val for val in values_lower) - has_medicaid = any("medicaid" in val for val in values_lower) + exhibit_level_answer_dict["DYNAMIC_PRIMARY_ENTITIES"] = dynamic_primary_entities - # If both are present and Duals is not already in the list, add it - if has_medicare and has_medicaid: - if "duals" not in values_lower: - if exhibit_text_answer: - exhibit_text_answer.append("Duals") - else: - exhibit_text_answer = ["Duals"] - logging.debug( - f"Added 'Duals' to LOB valid values because both Medicare and Medicaid were detected" - ) + dynamic_primary_entities_field = Field.from_values( + field_name="DYNAMIC_PRIMARY_ENTITIES", + relationship="one_to_n", + field_type="dynamic_primary", + prompt=( + "Candidate dynamic primary entities extracted at exhibit level. " + "Assign only the subset that applies to the current reimbursement term: {valid_values}" + ), + valid_values=dynamic_primary_entities, + ) - field.update_valid_values(exhibit_text_answer + ["N/A"]) - dynamic_reimbursement_fields.add_field(field) - dynamic_primary_fields.remove_field(field.field_name) - # If the field is not found, add to Exhibit-Level answers (the answer will be N/A or similar) - else: - exhibit_level_answer_dict[field.field_name] = exhibit_text_answer + # dynamic_primary_entities_field.update_valid_values(dynamic_primary_entities) + dynamic_reimbursement_fields.add_field(dynamic_primary_entities_field) + + for field_name in dynamic_primary_fields.list_fields().copy(): + dynamic_primary_fields.remove_field(field_name) # Update exhibit_level_answers.update(exhibit_level_answer_dict) - return exhibit_level_answers, dynamic_reimbursement_fields @@ -258,38 +259,14 @@ def add_one_to_one_field( field_to_add.field_type = "smart_chunked" field_to_add.relationship = "one_to_one" - # Special Handling for Program, Product, and Network: - # Skip when full LOB (all rows have LOB). When partial LOB, pass PROGRAM/PRODUCT; - # merge fills only empty cells, so 1:N values are preserved. - - if field_to_add.field_name in ["PROGRAM", "PRODUCT", "NETWORK"]: - lob_values = [] - for answer_dict in answer_dicts: - lob_value = answer_dict.get("LOB", "") - if isinstance(lob_value, list): - for item in lob_value: - if not string_utils.is_empty(item): - lob_values.append(str(item)) - elif not string_utils.is_empty(lob_value): - lob_values.append(lob_value) - has_lob = len(set(lob_values)) > 0 - lob_empty_count = sum( - 1 for d in answer_dicts if string_utils.is_empty(d.get("LOB")) - ) - partial_lob = has_lob and lob_empty_count > 0 - # Skip only when full LOB; allow PROGRAM, PRODUCT when partial - if has_lob and not partial_lob: - return one_to_one_fields - if field_to_add.field_name == "CLAIM_TYPE_CD": field_to_add.prompt = ( field_to_add.prompt + prompt_templates.HSC_CLAIM_TYPES_ADDITIONAL_INSTRUCTION() ) - # Special instructions for dynamic primary fields when passed to 1:1 - # Add specific template-language guidance, then append general full context instructions - if field_to_add.field_name in ["LOB", "PRODUCT", "PROGRAM", "NETWORK"]: + # Special instructions for dynamic primary entities field when passed to 1:1 + if field_to_add.field_name == "DYNAMIC_PRIMARY_ENTITIES": field_to_add.prompt = ( field_to_add.prompt + prompt_templates.HSC_DYNAMIC_PRIMARY_INSTRUCTION() ) @@ -311,49 +288,83 @@ def get_dynamic_one_to_one_fields( answer_dicts: list[dict[str, str]], constants: Constants ): """ - Returns a FieldSet object of dynamic fields that have are empty for at least one dict in answer_dicts + Returns a FieldSet object of dynamic fields that have are empty for at least one dict in answer_dicts. + + Special case: if ALL dynamic_primary fields are empty across every row (i.e. for each + exhibit, all dynamic primary fields were empty), adds a DYNAMIC_PRIMARY_ENTITIES + smart_chunked field instead of individual LOB/PROGRAM/PRODUCT/NETWORK fields so that + the full entity extraction + split path runs in one_to_one. All other dynamic fields + (e.g. CLAIM_TYPE_CD) still go through the normal loop. """ # These fields get passed to 1:1 if ALL of the answers are empty + if not answer_dicts: + return FieldSet() + all_empty_fields = FieldSet( config.FIELD_JSON_PATH, relationship="one_to_n", base_field=True ) one_to_one_fields = FieldSet() # Empty FieldSet to populate if criteria are met - # Check if any LOB value has been found - if so, skip PROGRAM, PRODUCT, NETWORK - # Only search for PROGRAM, PRODUCT, NETWORK when NO LOB has been found - # NOTE: Check raw LOB field, not AARETE_DERIVED_LOB, because AARETE_DERIVED_LOB - # is populated later via crosswalk and may be derived from PROGRAM/PRODUCT - # LOB can be a string or a list (from JSON format) - # Extract all LOB values, handling both string and list formats - # All values must be hashable (strings) to create a set - lob_values = [] - for answer_dict in answer_dicts: - lob_value = answer_dict.get("LOB", "") - if isinstance(lob_value, list): - # If it's a list, extract each item and ensure it's a string - for item in lob_value: - if not string_utils.is_empty(item): - # Convert to string to ensure hashability (handles edge cases) - lob_values.append(str(item)) - elif not string_utils.is_empty(lob_value): - # If it's a string, add it directly (already hashable) - lob_values.append(lob_value) - unique_lobs = set(lob_values) - has_lob = len(unique_lobs) > 0 + # Derive the dynamic_primary field names directly from the dynamic_primary FieldSet + # (not from all_empty_fields which only contains crosswalk/derived entries). + dynamic_primary_base_fields = [ + field.field_name + for field in FieldSet( + config.FIELD_JSON_PATH, + relationship="one_to_n", + field_type="dynamic_primary", + ).fields + ] - # Partial LOB: some rows have LOB, some don't (e.g. stripped headers) - # When partial, pass LOB+PROGRAM+PRODUCT to 1:1; merge fills only empty cells - lob_empty_count = sum( - 1 for d in answer_dicts if string_utils.is_empty(d.get("LOB")) + # Trigger DYNAMIC_PRIMARY_ENTITIES path only when ALL dynamic_primary fields are empty + # across every row — meaning for each exhibit, all dynamic primary fields were empty. + use_dynamic_primary_entities_path = bool(dynamic_primary_base_fields) and all( + string_utils.is_empty(answer_dict.get(field_name)) + for answer_dict in answer_dicts + for field_name in dynamic_primary_base_fields ) - partial_lob = has_lob and lob_empty_count > 0 + if use_dynamic_primary_entities_path: + dynamic_primary_fields = FieldSet( + config.FIELD_JSON_PATH, + relationship="one_to_n", + field_type="dynamic_primary", + ) + combined_prompt = _build_dynamic_primary_combined_prompt( + dynamic_primary_fields, constants + ) + dynamic_primary_entities_field = Field.from_values( + field_name="DYNAMIC_PRIMARY_ENTITIES", + relationship="one_to_one", + field_type="smart_chunked", + prompt=prompt_templates.DYNAMIC_PRIMARY_ENTITIES_INSTRUCTION( + combined_prompt + ), + retrieval_question="Extract all Lines of Business, Networks, Programs, and Products this contract applies to.", + ) + one_to_one_fields = add_one_to_one_field( + one_to_one_fields, dynamic_primary_entities_field, answer_dicts, constants + ) + logging.debug( + "All dynamic primary fields empty across contract — using DYNAMIC_PRIMARY_ENTITIES smart_chunked path for 1:1" + ) # Handle ALL empty fields for field in all_empty_fields.fields: field_name = field.field_name if "REIMB" in field_name: # Don't do this for the REIMB_ fields continue + if field_name in [ + "LOB", + "PROGRAM", + "PRODUCT", + "NETWORK", + "AARETE_DERIVED_LOB", + "AARETE_DERIVED_PROGRAM", + "AARETE_DERIVED_PRODUCT", + "AARETE_DERIVED_NETWORK", + ]: + continue empty_count = sum( 1 @@ -363,24 +374,15 @@ def get_dynamic_one_to_one_fields( total_count = len(answer_dicts) base = field.base_field if field.base_field else field_name - # LOB: pass when empty in ANY row (partial detection due to stripped headers) - # PROGRAM, PRODUCT: also pass when partial_lob and empty in any row (LOB relationship) - # Others: pass only when empty in ALL rows - should_pass_to_1to1 = ( - empty_count == total_count - or (base == "LOB" and empty_count > 0) - or (partial_lob and base in ["PROGRAM", "PRODUCT"] and empty_count > 0) - ) + should_pass_to_1to1 = empty_count == total_count if should_pass_to_1to1: - # Skip PROGRAM, PRODUCT, NETWORK when full LOB (all rows have LOB) - # Do NOT skip when partial_lob; pass PROGRAM, PRODUCT for consistency - if ( - has_lob - and not partial_lob - and base in ["PROGRAM", "PRODUCT", "NETWORK"] - ): - continue + logging.debug( + f"Field '{field_name}' is empty in all {total_count} rows — adding to one-to-one FieldSet" + ) + # Dynamic primary fields (LOB, PROGRAM, PRODUCT, NETWORK) and their + # derived counterparts (AARETE_DERIVED_*) are NEVER extracted individually — + # they always go through the DYNAMIC_PRIMARY_ENTITIES path. field_to_add = Field.load_from_file( file_path=config.FIELD_JSON_PATH, @@ -389,5 +391,4 @@ def get_dynamic_one_to_one_fields( one_to_one_fields = add_one_to_one_field( one_to_one_fields, field_to_add, answer_dicts, constants ) - return one_to_one_fields diff --git a/src/pipelines/shared/extraction/one_to_n_funcs.py b/src/pipelines/shared/extraction/one_to_n_funcs.py index bf5d82d..d038241 100644 --- a/src/pipelines/shared/extraction/one_to_n_funcs.py +++ b/src/pipelines/shared/extraction/one_to_n_funcs.py @@ -873,6 +873,372 @@ def get_lob_relationship(answer_dicts, exhibit_text, filename): return updated_dicts +def _normalize_dynamic_primary_entities(raw_value) -> list[str]: + """Normalize dynamic primary entities to a clean, distinct list of strings.""" + if string_utils.is_empty(raw_value): + return [] + + if isinstance(raw_value, list): + values = string_utils.flatten_to_strings(raw_value) + else: + values = string_utils.extract_nested_values(raw_value) + + cleaned = [] + for value in values: + text = str(value).strip() + if string_utils.is_empty(text): + continue + if text.upper() in {"N/A", "UNKNOWN", "NONE", "NULL"}: + continue + if text not in cleaned: + cleaned.append(text) + return cleaned + + +def _map_lob_network_to_aarete_derived( + answer_dict: dict, + valid_lobs: list[str], + valid_networks: list[str], + constants: Constants, + filename: str, +) -> None: + """Derive AARETE_DERIVED_LOB and AARETE_DERIVED_NETWORK from base LOB/NETWORK fields. + + Two-step process — the intermediate step is never stored in the output: + 1. LLM maps raw base-field text (exact contract language) to valid crosswalk keys. + 2. Crosswalk maps those valid keys to AARETE_DERIVED values, which are stored + in answer_dict["AARETE_DERIVED_LOB"] / answer_dict["AARETE_DERIVED_NETWORK"]. + + Downstream get_crosswalk_fields will not overwrite pre-populated AARETE_DERIVED fields. + Modifies answer_dict in-place. + """ + # Derive AARETE_DERIVED_LOB + if valid_lobs: + raw_lobs = _normalize_dynamic_primary_entities(answer_dict.get("LOB")) + if raw_lobs: + try: + lob_keys = prompt_calls.prompt_map_lob_to_valid( + lob_values=raw_lobs, + valid_lobs=valid_lobs, + filename=filename, + ) + if lob_keys: + derived_lobs = [] + for key in lob_keys: + val = constants.CROSSWALK_LOB.mapping.get(key) + if val is None: + continue + if isinstance(val, list): + derived_lobs.extend(v for v in val if v and v != "N/A") + elif val and val != "N/A": + derived_lobs.append(val) + if derived_lobs: + answer_dict["AARETE_DERIVED_LOB"] = list( + dict.fromkeys(derived_lobs) + ) + except Exception: + logging.exception( + "Failed to derive AARETE_DERIVED_LOB from LOB; skipping" + ) + + # Derive AARETE_DERIVED_NETWORK + if valid_networks: + raw_networks = _normalize_dynamic_primary_entities(answer_dict.get("NETWORK")) + if raw_networks: + try: + network_keys = prompt_calls.prompt_map_network_to_valid( + network_values=raw_networks, + valid_networks=valid_networks, + filename=filename, + ) + if network_keys: + derived_networks = [] + for key in network_keys: + val = constants.CROSSWALK_NETWORK.mapping.get(key) + if val is None: + continue + if isinstance(val, list): + derived_networks.extend(v for v in val if v and v != "N/A") + elif val and val != "N/A": + derived_networks.append(val) + if derived_networks: + answer_dict["AARETE_DERIVED_NETWORK"] = list( + dict.fromkeys(derived_networks) + ) + except Exception: + logging.exception( + "Failed to derive AARETE_DERIVED_NETWORK from NETWORK; skipping" + ) + + +def map_lob_network_to_aarete_derived( + all_exhibit_rows: list[dict], constants: Constants, filename: str +) -> list[dict]: + """Derive AARETE_DERIVED_LOB/NETWORK for every row from their base LOB/NETWORK fields. + + Runs the two-step intermediate mapping (exact contract text → crosswalk keys → AARETE_DERIVED + values) for each row. Called explicitly just before get_crosswalk_fields so that + split_dynamic_primary_entities remains focused on classification only. + """ + valid_lobs = constants.get_constant("VALID_LOBS") + valid_networks = constants.get_constant("VALID_NETWORKS") + for answer_dict in all_exhibit_rows: + _map_lob_network_to_aarete_derived( + answer_dict, valid_lobs, valid_networks, constants, filename + ) + return all_exhibit_rows + + +def split_dynamic_primary_entities( + all_exhibit_rows: list[dict], filename: str +) -> list[dict]: + """Split DYNAMIC_PRIMARY_ENTITIES into LOB/PROGRAM/PRODUCT/NETWORK and remove temp field.""" + if not all_exhibit_rows: + return all_exhibit_rows + + for answer_dict in all_exhibit_rows: + raw_entities = answer_dict.get("DYNAMIC_PRIMARY_ENTITIES") + entities = _normalize_dynamic_primary_entities(raw_entities) + + if not entities: + answer_dict.pop("DYNAMIC_PRIMARY_ENTITIES", None) + _deduplicate_primary_fields(answer_dict, filename) + continue + + # Step 1: Classify entities semantically — no valid-value constraints at this stage + try: + classified = prompt_calls.prompt_classify_dynamic_primary_entities( + dynamic_primary_entities=entities, + filename=filename, + ) + except Exception: + logging.exception( + "Failed to classify DYNAMIC_PRIMARY_ENTITIES; skipping row" + ) + answer_dict.pop("DYNAMIC_PRIMARY_ENTITIES", None) + continue + + # Step 2: Build collision-guard sets from raw classified LOB/NETWORK values to + # filter PROGRAM/PRODUCT values that duplicate a confirmed LOB or NETWORK entity. + mapped_lobs_lower = { + v.lower() + for v in _normalize_dynamic_primary_entities(classified.get("LOB")) + } + mapped_networks_lower = { + v.lower() + for v in _normalize_dynamic_primary_entities(classified.get("NETWORK")) + } + + # Step 3: Merge classified values into answer_dict fields + for field_name in ["LOB", "NETWORK", "PROGRAM", "PRODUCT"]: + new_values = _normalize_dynamic_primary_entities(classified.get(field_name)) + if not new_values: + continue + + # Guard PROGRAM/PRODUCT against confirmed LOB or NETWORK values + if field_name in ("PROGRAM", "PRODUCT"): + new_values = [ + value + for value in new_values + if value.lower() not in mapped_lobs_lower + and value.lower() not in mapped_networks_lower + ] + + if not new_values: + continue + + existing_values = _normalize_dynamic_primary_entities( + answer_dict.get(field_name) + ) + merged = list(dict.fromkeys(existing_values + new_values)) + answer_dict[field_name] = merged + + answer_dict.pop("DYNAMIC_PRIMARY_ENTITIES", None) + + # Step 4: Cross-field deduplication — enforce LOB > NETWORK > PROGRAM > PRODUCT priority. + # A value that appears in multiple fields (e.g. from the raw extraction putting the same + # entity into both PROGRAM and PRODUCT) is retained only in the highest-priority field. + _deduplicate_primary_fields(answer_dict, filename) + + return all_exhibit_rows + + +def _deduplicate_primary_fields(answer_dict: dict, filename: str = "") -> None: + """Enforce LOB > NETWORK > PROGRAM > PRODUCT priority across primary classification fields. + + If a value appears in multiple fields, it is retained only in the highest-priority field + and removed from all lower-priority ones. This handles cases where the raw one-to-n + extraction puts the same entity into both PROGRAM and PRODUCT (bypassing the + classification step entirely). + + Modifies answer_dict in-place. + """ + + def _vals(field: str) -> list[str]: + return _normalize_dynamic_primary_entities(answer_dict.get(field)) + + lob_lower = {v.lower() for v in _vals("LOB")} + network_lower = {v.lower() for v in _vals("NETWORK")} + removals: dict[str, list[str]] = {} + + # Remove from NETWORK anything already claimed by LOB + network_vals = _vals("NETWORK") + if network_vals: + filtered = [v for v in network_vals if v.lower() not in lob_lower] + if len(filtered) != len(network_vals): + removals["NETWORK"] = [v for v in network_vals if v.lower() in lob_lower] + answer_dict["NETWORK"] = filtered + network_lower = {v.lower() for v in filtered} + + # Remove from PROGRAM anything already claimed by LOB or NETWORK + program_vals = _vals("PROGRAM") + if program_vals: + filtered = [ + v + for v in program_vals + if v.lower() not in lob_lower and v.lower() not in network_lower + ] + if len(filtered) != len(program_vals): + removals["PROGRAM"] = [ + v + for v in program_vals + if v.lower() in lob_lower or v.lower() in network_lower + ] + answer_dict["PROGRAM"] = filtered + program_lower = {v.lower() for v in filtered} + else: + program_lower = set() + + # Remove from PRODUCT anything already claimed by LOB, NETWORK, or PROGRAM + product_vals = _vals("PRODUCT") + if product_vals: + filtered = [ + v + for v in product_vals + if v.lower() not in lob_lower + and v.lower() not in network_lower + and v.lower() not in program_lower + ] + if len(filtered) != len(product_vals): + removals["PRODUCT"] = [ + v + for v in product_vals + if v.lower() in lob_lower + or v.lower() in network_lower + or v.lower() in program_lower + ] + answer_dict["PRODUCT"] = filtered + + +def _normalize_to_uppercase_list(value) -> list[str]: + """Normalize a PROGRAM or PRODUCT value to a list of trimmed, uppercase strings.""" + if string_utils.is_empty(value): + return [] + if isinstance(value, list): + return [str(item).strip().upper() for item in value if str(item).strip()] + text = str(value).strip() + return [text.upper()] if text else [] + + +def add_aarete_derived_program_product( + df: pd.DataFrame, constants: Constants, filename: str +) -> pd.DataFrame: + """Derive AARETE_DERIVED_PROGRAM and AARETE_DERIVED_PRODUCT from PROGRAM and PRODUCT. + + For each field declared as ``field_type="derived_dynamic_primary"`` in the FieldSet: + + - If the corresponding valid values list (resolved from Constants) is **non-empty**, + the LLM is used to map each row's base field values (PROGRAM / PRODUCT) to + standardized derived values constrained to the valid values list. + - If the valid values list is **empty or absent**, the function falls back to simple + trim-and-titlecase normalization (preserving the current behaviour for PRODUCT when + no client-specific valid values are configured). + + Existing (non-empty) values for the derived field on a row are preserved and the new + values are merged in (deduplicated, order-stable). + + Args: + df: pd.DataFrame containing rows with PROGRAM and/or PRODUCT fields. + constants: Constants object used to look up VALID_AARETE_DERIVED_PROGRAMS and + VALID_AARETE_DERIVED_PRODUCTS. + filename: Filename passed to LLM calls and logging. + + Returns: + pd.DataFrame with AARETE_DERIVED_PROGRAM and AARETE_DERIVED_PRODUCT set. + """ + if df.empty: + return df + all_exhibit_rows = df.to_dict(orient="records") + + derived_fields = FieldSet( + config.FIELD_JSON_PATH, field_type="derived_dynamic_primary" + ) + + for field in derived_fields.fields: + derived_field_name = field.field_name # e.g. AARETE_DERIVED_PROGRAM + base_field_name = field.base_field # e.g. PROGRAM + valid_values_key = field.valid_values # e.g. "VALID_AARETE_DERIVED_PROGRAMS" + + # Resolve valid values from constants (None / empty → fallback mode) + valid_values: list[str] = constants.get_constant(valid_values_key) or [] + use_llm = bool(valid_values) + # Build a lowercase → canonical lookup for post-LLM enforcement + valid_values_by_lower: dict[str, str] = {v.lower(): v for v in valid_values} + for answer_dict in all_exhibit_rows: + raw_value = answer_dict.get(base_field_name) + if string_utils.is_empty(raw_value): + continue + + # Normalise base values to a flat list of non-empty strings, + # handling both real lists and serialized list strings (e.g. '["A", "B"]') + base_values = _normalize_dynamic_primary_entities(raw_value) + + if not base_values: + continue + + if use_llm: + try: + if derived_field_name == "AARETE_DERIVED_PROGRAM": + derived_values = prompt_calls.prompt_derive_aarete_program( + base_values, valid_values, filename + ) + elif derived_field_name == "AARETE_DERIVED_PRODUCT": + derived_values = prompt_calls.prompt_derive_aarete_product( + base_values, valid_values, filename + ) + except Exception: + logging.exception( + f"LLM derivation failed for {derived_field_name}; skipping" + ) + derived_values = [] + # Enforce valid-values constraint: filter out any LLM hallucinations, + # returning the canonical value exactly as it appears in valid_values + derived_values = [ + valid_values_by_lower[v.lower()] + for v in derived_values + if v.lower() in valid_values_by_lower + ] + else: + # Fallback: trim + title-case, preserving all-uppercase words as + # acronyms (e.g. CHIP, DSNP, STAR+PLUS stay unchanged). + def _title_preserve_acronyms(text: str) -> str: + return " ".join( + word if word.isupper() else word.title() + for word in text.split() + ) + + derived_values = [_title_preserve_acronyms(v) for v in base_values] + + # Merge with any existing values already present on this row + existing_raw = answer_dict.get(derived_field_name) + existing_list = _normalize_dynamic_primary_entities(existing_raw) + + merged = list(dict.fromkeys(existing_list + derived_values)) + answer_dict[derived_field_name] = merged + + return pd.DataFrame(all_exhibit_rows) + + def set_carveout_indicators(all_exhibit_rows: list[dict]) -> list[dict]: """ Sets CARVEOUT_IND and DEFAULT_IND based on CARVEOUT_CD values. @@ -901,6 +1267,20 @@ def set_carveout_indicators(all_exhibit_rows: list[dict]) -> list[dict]: def one_to_n_cleaning( all_exhibit_rows: list[dict], exhibit_text: str, constants: Constants, filename: str ): + ################################ Split Dynamic Primary Entities ################################ + with timing_utils.timed_block( + "one_to_n_cleaning.split_dynamic_primary_entities", context=filename + ): + all_exhibit_rows = split_dynamic_primary_entities(all_exhibit_rows, filename) + + ################################ Derive AARETE_DERIVED_LOB/NETWORK ################################ + with timing_utils.timed_block( + "one_to_n_cleaning.derive_aarete_lob_network", context=filename + ): + all_exhibit_rows = map_lob_network_to_aarete_derived( + all_exhibit_rows, constants, filename + ) + ################################ Crosswalk Fields ################################ with timing_utils.timed_block( "one_to_n_cleaning.crosswalk_fields", context=filename @@ -917,12 +1297,6 @@ def one_to_n_cleaning( all_exhibit_rows, exhibit_text, filename ) - ################################ Fill NA Mapping ################################ - with timing_utils.timed_block( - "one_to_n_cleaning.fill_na_mapping", context=filename - ): - all_exhibit_rows = aarete_derived.fill_na_mapping(all_exhibit_rows) - ################################ Split REIMB_DATES ################################ with timing_utils.timed_block( "one_to_n_cleaning.split_reimb_dates", context=filename diff --git a/src/pipelines/shared/extraction/row_funcs.py b/src/pipelines/shared/extraction/row_funcs.py index b8d1091..7c8cff3 100644 --- a/src/pipelines/shared/extraction/row_funcs.py +++ b/src/pipelines/shared/extraction/row_funcs.py @@ -3,7 +3,6 @@ from copy import deepcopy import pandas as pd import src.pipelines.shared.prompts.prompt_calls as prompt_calls from src.constants.constants import Constants -from src.pipelines.shared.postprocessing import aarete_derived from src.pipelines.shared.extraction import one_to_one_funcs from src.utils import string_utils from src.prompts.fieldset import FieldSet @@ -100,9 +99,6 @@ def merge_one_to_one_into_one_to_n( ) ) - # one_to_one merge happens after one_to_n cleaning. Re-run fill mapping so - # propagated PROGRAM/PRODUCT values can derive LOB in rows where it is empty. merged_rows = one_to_n_results.to_dict(orient="records") - merged_rows = aarete_derived.fill_na_mapping(merged_rows) return pd.DataFrame(merged_rows) diff --git a/src/pipelines/shared/postprocessing/aarete_derived.py b/src/pipelines/shared/postprocessing/aarete_derived.py index c362b89..b404ec4 100644 --- a/src/pipelines/shared/postprocessing/aarete_derived.py +++ b/src/pipelines/shared/postprocessing/aarete_derived.py @@ -1,14 +1,22 @@ import logging +import pandas as pd import src.config as config import src.utils.string_utils as string_utils from src.constants.constants import Constants from src.constants.investment_columns import FIELD_FORMAT_MAPPING from src.crosswalk.crosswalk_builder import CrosswalkBuilder +from src.pipelines.saas.prompts import prompt_calls from src.prompts.fieldset import FieldSet from src.utils.crosswalk_utils import apply_crosswalk from src.utils.formatting_utils import normalize_field_value +def _canonical_lob(val: str, valid_lobs_lower: dict[str, str]) -> str: + """Return the canonical-cased LOB from valid_lobs (case-insensitive match). + Falls back to title case if no match found.""" + return valid_lobs_lower.get(val.strip().lower(), val.strip().title()) + + def fill_na_from_field(results_dict, to_field, from_field, crosswalk_path): from_str = results_dict.get(from_field) if string_utils.is_empty(from_str): @@ -18,156 +26,197 @@ def fill_na_from_field(results_dict, to_field, from_field, crosswalk_path): return to_value if to_value else results_dict.get(to_field) -def fill_na_mapping(answer_dicts): +def fill_na_mapping(df: pd.DataFrame, constants: Constants) -> pd.DataFrame: """ - Fill in missing values in fields using mappings from other fields. - Always checks all crosswalks (PROGRAM and PRODUCT) and merges unique values. + Derive AARETE_DERIVED_LOB from AARETE_DERIVED_PROGRAM and AARETE_DERIVED_PRODUCT. + Uses crosswalk mapping when available; falls back to LLM when the mapping is empty. + Merges unique LOB values from both sources. Args: - answer_dicts (list[dict]): List of answer dictionaries + df (pd.DataFrame): DataFrame with extracted fields. + constants (Constants): Shared Constants instance (loaded once at startup). Returns: - list[dict]: List of answer dictionaries with filled/merged values + pd.DataFrame: DataFrame with filled/merged AARETE_DERIVED_LOB values. """ + if df.shape[0] == 0: + return df + + filename = df["FILE_NAME"].iloc[0] if "FILE_NAME" in df.columns else "unknown" + payer_name_raw = df["PAYER_NAME"].iloc[0] if "PAYER_NAME" in df.columns else "" + payer_state_raw = df["PAYER_STATE"].iloc[0] if "PAYER_STATE" in df.columns else "" + payer_name = ( + str(payer_name_raw) if not string_utils.is_empty(payer_name_raw) else "" + ) + payer_state = ( + str(payer_state_raw) if not string_utils.is_empty(payer_state_raw) else "" + ) + + answer_dicts = df.to_dict(orient="records") + # Crosswalk values are the AARETE_DERIVED_LOB values - pass these to the LLM + lob_mapping_values = constants.CROSSWALK_LOB.mapping.values() + valid_lobs = sorted( + { + item + for v in lob_mapping_values + for item in (v if isinstance(v, list) else [v]) + if item and item != "N/A" + } + ) + # Case-insensitive lookup → canonical form (e.g. "MEDICAID" → "Medicaid") + valid_lobs_lower: dict[str, str] = {v.lower(): v for v in valid_lobs} + + def _to_str_list(val) -> list: + """Normalize a field value to a deduplicated list of non-empty strings.""" + if string_utils.is_empty(val): + return [] + if isinstance(val, list): + return [str(v).strip() for v in val if str(v).strip()] + if isinstance(val, str): + return [val.strip()] if val.strip() else [] + return [] + + # --- Pre-compute LOB caches keyed by unique input tuples --- + # Collect unique keys across all rows so each distinct value is resolved once. + unique_aarete_program_keys: set[tuple] = set() + unique_program_keys: set[tuple] = set() + unique_aarete_product_keys: set[tuple] = set() + unique_product_keys: set[tuple] = set() + + for answer_dict in answer_dicts: + adp = _to_str_list(answer_dict.get("AARETE_DERIVED_PROGRAM")) + if adp: + unique_aarete_program_keys.add(tuple(adp)) + p = _to_str_list(answer_dict.get("PROGRAM")) + if p: + unique_program_keys.add(tuple(p)) + adprod = _to_str_list(answer_dict.get("AARETE_DERIVED_PRODUCT")) + if adprod: + unique_aarete_product_keys.add(tuple(adprod)) + prod = _to_str_list(answer_dict.get("PRODUCT")) + if prod: + unique_product_keys.add(tuple(prod)) + + # Crosswalk cache: AARETE_DERIVED_PROGRAM → LOB list + program_crosswalk_cache: dict[tuple, list] = {} + if constants.CROSSWALK_PROGRAM_LOB.mapping: + for key in unique_aarete_program_keys: + program_crosswalk_cache[key] = [ + constants.CROSSWALK_PROGRAM_LOB.mapping[v] + for v in key + if v in constants.CROSSWALK_PROGRAM_LOB.mapping + ] + + # LLM cache: PROGRAM → LOB list (one call per unique program list) + program_llm_cache: dict[tuple, list] = {} + for key in unique_program_keys: + program_llm_cache[key] = ( + prompt_calls.prompt_program_to_lob( + list(key), + valid_lobs, + filename, + payer_name=payer_name, + payer_state=payer_state, + ) + or [] + ) + + # Crosswalk cache: AARETE_DERIVED_PRODUCT → LOB list + product_crosswalk_cache: dict[tuple, list] = {} + if constants.CROSSWALK_PRODUCT_LOB.mapping: + for key in unique_aarete_product_keys: + product_crosswalk_cache[key] = [ + constants.CROSSWALK_PRODUCT_LOB.mapping[v] + for v in key + if v in constants.CROSSWALK_PRODUCT_LOB.mapping + ] + + # LLM cache: PRODUCT → LOB list (one call per unique product list) + product_llm_cache: dict[tuple, list] = {} + for key in unique_product_keys: + product_llm_cache[key] = ( + prompt_calls.prompt_product_to_lob( + list(key), + valid_lobs, + filename, + payer_name=payer_name, + payer_state=payer_state, + ) + or [] + ) + + # --- Apply cached results to each row --- for answer_dict in answer_dicts: - # Collect all AARETE_DERIVED_LOB values from different sources all_lob_values = set() # Get existing AARETE_DERIVED_LOB values (if any) # Handle both list and string formats (JSON lists or pipe-delimited strings) existing_lob_list = answer_dict.get("AARETE_DERIVED_LOB", "") - if not string_utils.is_empty(existing_lob_list): - # Normalize to list format if isinstance(existing_lob_list, list): lob_items = existing_lob_list elif isinstance(existing_lob_list, str): - # Handle pipe-delimited format (backward compatibility) - if "|" in existing_lob_list: - lob_items = [v.strip() for v in existing_lob_list.split("|")] - else: - lob_items = [existing_lob_list] + lob_items = [existing_lob_list] else: lob_items = [str(existing_lob_list)] - - # Process each LOB value for lob_val in lob_items: if ( isinstance(lob_val, str) and lob_val.strip() and lob_val.strip() != "N/A" ): - all_lob_values.add(lob_val.strip()) + all_lob_values.add(_canonical_lob(lob_val, valid_lobs_lower)) - # Get AARETE_DERIVED_LOB from AARETE_DERIVED_PROGRAM crosswalk (always check if PROGRAM exists) + # Get AARETE_DERIVED_LOB from AARETE_DERIVED_PROGRAM + PROGRAM (via caches) program_lob_values = set() - if not string_utils.is_empty(answer_dict.get("AARETE_DERIVED_PROGRAM")): - program_filled_value_list = fill_na_from_field( - answer_dict, - "AARETE_DERIVED_LOB", - "AARETE_DERIVED_PROGRAM", - "src/constants/mappings/crosswalk_program_lob.json", - ) - if ( - not string_utils.is_empty(program_filled_value_list) - and "N/A" not in program_filled_value_list - ): - # Normalize to list format (handle both list and string) - if isinstance(program_filled_value_list, list): - program_lob_items = program_filled_value_list - elif isinstance(program_filled_value_list, str): - # Handle pipe-delimited format (backward compatibility) - if "|" in program_filled_value_list: - program_lob_items = [ - v.strip() for v in program_filled_value_list.split("|") - ] - else: - program_lob_items = [program_filled_value_list] - else: - program_lob_items = [str(program_filled_value_list)] + adp_key = tuple(_to_str_list(answer_dict.get("AARETE_DERIVED_PROGRAM"))) + p_key = tuple(_to_str_list(answer_dict.get("PROGRAM"))) + program_lob_list = program_crosswalk_cache.get( + adp_key, [] + ) + program_llm_cache.get(p_key, []) + filtered = [ + _canonical_lob(v, valid_lobs_lower) + for v in program_lob_list + if isinstance(v, str) and v.strip() and v.strip() != "N/A" + ] + if filtered: + program_lob_values.update(filtered) + all_lob_values.update(filtered) - # Process each program LOB value - for program_lob_val in program_lob_items: - if ( - isinstance(program_lob_val, str) - and program_lob_val.strip() - and program_lob_val.strip() != "N/A" - ): - program_lob_values.add(program_lob_val.strip()) - # Update all_lob_values after processing all program LOB values - all_lob_values.update(program_lob_values) - - # Get AARETE_DERIVED_LOB from product crosswalk. - # Prefer AARETE_DERIVED_PRODUCT, fallback to PRODUCT when derived is empty. + # Get AARETE_DERIVED_LOB from AARETE_DERIVED_PRODUCT + PRODUCT (via caches) product_lob_values = set() - product_source_field = None - if not string_utils.is_empty(answer_dict.get("AARETE_DERIVED_PRODUCT")): - product_source_field = "AARETE_DERIVED_PRODUCT" - elif not string_utils.is_empty(answer_dict.get("PRODUCT")): - product_source_field = "PRODUCT" - - if product_source_field is not None: - product_filled_value_list = fill_na_from_field( - answer_dict, - "AARETE_DERIVED_LOB", - product_source_field, - "src/constants/mappings/crosswalk_product_lob.json", - ) - if ( - not string_utils.is_empty(product_filled_value_list) - and "N/A" not in product_filled_value_list - ): - # Normalize to list format (handle both list and string) - if isinstance(product_filled_value_list, list): - product_lob_items = product_filled_value_list - elif isinstance(product_filled_value_list, str): - # Handle pipe-delimited format (backward compatibility) - if "|" in product_filled_value_list: - product_lob_items = [ - v.strip() for v in product_filled_value_list.split("|") - ] - else: - product_lob_items = [product_filled_value_list] - else: - product_lob_items = [str(product_filled_value_list)] - - # Process each product LOB value - for product_lob_val in product_lob_items: - if ( - isinstance(product_lob_val, str) - and product_lob_val.strip() - and product_lob_val.strip() != "N/A" - ): - product_lob_values.add(product_lob_val.strip()) - # Update all_lob_values after processing all product LOB values - all_lob_values.update(product_lob_values) + adprod_key = tuple(_to_str_list(answer_dict.get("AARETE_DERIVED_PRODUCT"))) + prod_key = tuple(_to_str_list(answer_dict.get("PRODUCT"))) + product_lob_list = product_crosswalk_cache.get( + adprod_key, [] + ) + product_llm_cache.get(prod_key, []) + filtered = [ + _canonical_lob(v, valid_lobs_lower) + for v in product_lob_list + if isinstance(v, str) and v.strip() and v.strip() != "N/A" + ] + if filtered: + product_lob_values.update(filtered) + all_lob_values.update(filtered) # Set the merged, deduplicated value if all_lob_values: answer_dict["AARETE_DERIVED_LOB"] = sorted(all_lob_values) - if string_utils.is_empty(answer_dict.get("LOB")): - answer_dict["LOB"] = sorted(all_lob_values) - # Set relationship flags if values came from PROGRAM or PRODUCT # Only set if field is empty or "N/A" (preserve LLM-determined Inclusive/Exclusive) if program_lob_values: existing_relationship = answer_dict.get("LOB_PROGRAM_RELATIONSHIP", "") - # Original code (CHANGED - now preserves LLM values): - # answer_dict["LOB_PROGRAM_RELATIONSHIP"] = "Exclusive" if string_utils.is_empty(existing_relationship): answer_dict["LOB_PROGRAM_RELATIONSHIP"] = "Exclusive" if product_lob_values: existing_relationship = answer_dict.get("LOB_PRODUCT_RELATIONSHIP", "") - # Original code (CHANGED - now preserves LLM values): - # answer_dict["LOB_PRODUCT_RELATIONSHIP"] = "Exclusive" if string_utils.is_empty(existing_relationship): answer_dict["LOB_PRODUCT_RELATIONSHIP"] = "Exclusive" elif string_utils.is_empty(answer_dict.get("AARETE_DERIVED_LOB")): # If still empty after all attempts, keep it as is (will be N/A or empty) pass - # Handle cases where PROGRAM/PRODUCT is NA/blank (LLM was bypassed) - # Set relationship to "Exclusive" as default when PROGRAM/PRODUCT is missing and relationship is not set - # This covers both cases: LOB exists but PROGRAM/PRODUCT is NA, or both LOB and PROGRAM/PRODUCT are NA + # Handle cases where PROGRAM/PRODUCT is NA/blank + # Set relationship to "Exclusive" as default when missing and relationship is not set if string_utils.is_empty(answer_dict.get("AARETE_DERIVED_PROGRAM")): if string_utils.is_empty(answer_dict.get("LOB_PROGRAM_RELATIONSHIP")): answer_dict["LOB_PROGRAM_RELATIONSHIP"] = "Exclusive" @@ -176,7 +225,7 @@ def fill_na_mapping(answer_dicts): if string_utils.is_empty(answer_dict.get("LOB_PRODUCT_RELATIONSHIP")): answer_dict["LOB_PRODUCT_RELATIONSHIP"] = "Exclusive" - return answer_dicts + return pd.DataFrame(answer_dicts) def get_crosswalk_fields(answer_dicts: list, constants: Constants): @@ -248,5 +297,4 @@ def get_crosswalk_fields(answer_dicts: list, constants: Constants): # The crosswalk should only create/update the target field (to_field_name), # not modify the source field (from_field_name) # If from_field_name needs to be a list, that should be handled elsewhere - return answer_dicts diff --git a/src/pipelines/shared/postprocessing/postprocess.py b/src/pipelines/shared/postprocessing/postprocess.py index fdaca5a..0b5cced 100644 --- a/src/pipelines/shared/postprocessing/postprocess.py +++ b/src/pipelines/shared/postprocessing/postprocess.py @@ -4,7 +4,7 @@ import logging import src.config as config from src.constants.constants import Constants from src.constants.investment_columns import FIELD_FORMAT_MAPPING -from src.pipelines.shared.postprocessing import postprocessing_funcs +from src.pipelines.shared.postprocessing import postprocessing_funcs, aarete_derived from src.utils import timing_utils, string_utils @@ -130,9 +130,6 @@ def standard_postprocess(df, constants: Constants): # Process PATIENT_AGE_RANGE into PATIENT_AGE_MIN and PATIENT_AGE_MAX df = postprocessing_funcs.process_patient_age_range(df) - # Add AARETE_DERIVED_PRODUCT - df = postprocessing_funcs.add_aarete_derived_product(df) - df = postprocessing_funcs.update_grouper_base_rate_and_grouper_pct_rate(df) # Fill empty claim type from CONTRACT_TITLE as a safety net diff --git a/src/pipelines/shared/postprocessing/postprocessing_funcs.py b/src/pipelines/shared/postprocessing/postprocessing_funcs.py index ad1899e..9a5d453 100644 --- a/src/pipelines/shared/postprocessing/postprocessing_funcs.py +++ b/src/pipelines/shared/postprocessing/postprocessing_funcs.py @@ -733,14 +733,11 @@ def add_aarete_derived_amendment_num(df: pd.DataFrame) -> pd.DataFrame: return df -def add_aarete_derived_product(df): +def add_aarete_derived_program_product(df): + """No-op pass-through: AARETE_DERIVED_PROGRAM/PRODUCT are now derived upstream + in one_to_n_cleaning (one_to_n_funcs.add_aarete_derived_program_product) before + the DataFrame is assembled. This stub is retained for backward compatibility. """ - Add AARETE_DERIVED_PRODUCT column to df, based on PRODUCT column. - - The new column should be Title Case of the PRODUCT column value. - """ - if "PRODUCT" in df.columns: - df["AARETE_DERIVED_PRODUCT"] = df["PRODUCT"] return df diff --git a/src/pipelines/shared/preprocessing/hybrid_smart_chunking_funcs.py b/src/pipelines/shared/preprocessing/hybrid_smart_chunking_funcs.py index bd8e399..81e2278 100644 --- a/src/pipelines/shared/preprocessing/hybrid_smart_chunking_funcs.py +++ b/src/pipelines/shared/preprocessing/hybrid_smart_chunking_funcs.py @@ -470,14 +470,25 @@ def prompt_hsc_single_field( f"context with len {len(context)}: {context[:500]}........{context[-500:]}" ) # log only the beginning and end of the context to avoid clutter - # LLM call + # LLM call. The static extraction rules live in + # ONE_TO_ONE_SINGLE_FIELD_INSTRUCTION so they cache once across all + # ~30 fields per document instead of being re-billed every call. + # The dynamic prompt still carries the per-field question and the + # retrieved chunk; only the rule block is cached. with timing_utils.timed_block( f"hsc.field.{field.field_name}.llm_call", context=filename ): llm_answer_raw = llm_utils.invoke_claude( - prompt, "sonnet_latest", filename, max_tokens=8192 + prompt, + "sonnet_latest", + filename, + max_tokens=8192, + cache=True, + instruction=prompt_templates.ONE_TO_ONE_SINGLE_FIELD_INSTRUCTION(), + usage_label="ONE_TO_ONE_SINGLE_FIELD", ) logging.debug(f"Raw LLM answer for {field.field_name}: {llm_answer_raw}") + try: llm_answer_final = _parser(llm_answer_raw) # returns dict field_value = llm_answer_final.get(field.field_name) diff --git a/src/pipelines/vendors/shared/generic_processor.py b/src/pipelines/vendors/shared/generic_processor.py index 93ef317..360b821 100644 --- a/src/pipelines/vendors/shared/generic_processor.py +++ b/src/pipelines/vendors/shared/generic_processor.py @@ -127,7 +127,7 @@ class VendorProcessor: Generic vendor processor driven by config.yaml + vendor_fields.json. Matches the interface expected by runner.py: - - process_file(file_object, constants, run_timestamp) -> pd.DataFrame + - process_file(file_object, constants, run_timestamp) -> tuple[pd.DataFrame, None] - SHEET_COLUMN_SCHEMAS -> dict[str, list[str]] """ @@ -222,7 +222,9 @@ class VendorProcessor: # Public entry point # ------------------------------------------------------------------ - def process_file(self, file_object, constants, run_timestamp) -> pd.DataFrame: + def process_file( + self, file_object, constants, run_timestamp + ) -> tuple[pd.DataFrame, None]: """Process a single vendor contract file. Args: @@ -231,8 +233,10 @@ class VendorProcessor: run_timestamp: Timestamp string for output naming. Returns: - Flat DataFrame combining 1-to-1 and SLA/KPI rows (broadcast style). - Each SLA/KPI row carries all 1-to-1 field values, matching saas behaviour. + (flat_df, None) — matches the (cc_df, dashboard_df) contract used by + the saas pipeline. Vendors do not produce a dashboard variant, so the + second slot is always None. flat_df combines 1-to-1 and SLA/KPI rows + (broadcast style); each SLA/KPI row carries all 1-to-1 field values. """ filename, contract_text = file_object logging_utils.set_current_file(filename) @@ -307,7 +311,7 @@ class VendorProcessor: logging.info( f"{datetime_str()} [{self.vendor_name}] Writing complete — {filename}" ) - return flat_df + return flat_df, None # ------------------------------------------------------------------ # Extraction helpers diff --git a/src/prompts/cache_registry.py b/src/prompts/cache_registry.py new file mode 100644 index 0000000..73605ee --- /dev/null +++ b/src/prompts/cache_registry.py @@ -0,0 +1,622 @@ +"""Central prompt cache registry. + +Single source of truth for which prompts are eligible for prompt caching, +which model they actually run on at runtime, and which cache class they +use. This is consumed by the cache-warming layer in +`src.pipelines.runner` and by reporting / contract tests. + +Why this exists +--------------- +Before the registry, cache warming and runtime call sites were split: +- `prompt_templates.get_cacheable_instructions()` produced the warmed list +- runtime code in `pipelines/.../prompt_calls.py` chose its own model id +The two could (and did) drift. For example `SPLIT_SERVICE_TERM` was warmed +on `sonnet_latest` but invoked on `haiku_latest`, so warm-up tokens were +spent against a cache key the runtime never read. + +Cache classes +------------- +- INSTRUCTION: the system instruction block is cached; cache reads + are expected on every call after warm-up. +- CONTEXT: a dynamic `context_for_caching` block is cached; + reads are only expected when the same context + repeats (e.g. the same exhibit text across fields). +- INSTRUCTION_PLUS_CONTEXT: both blocks are cached. +- NONE: explicitly opted out; either the runtime model + does not support caching in this Bedrock setup, + or the static instruction is too small to qualify + and is not yet padded. + +Reporting should evaluate INSTRUCTION and CONTEXT cache classes +separately so per-context variability does not make instruction caching +look broken. +""" + +from __future__ import annotations + +import logging +import re +from dataclasses import dataclass +from typing import Callable, Iterable, Iterator + +import src.config as config +import src.prompts.prompt_templates as prompt_templates + + +# Default minimum tokens for a cache_control block to be honored. +# Per-model overrides live in `llm_utils._MODEL_MIN_CACHE_TOKENS`. This +# constant is the conservative fallback used when an entry has no +# explicit min_instruction_tokens set and the resolved model is unknown. +MIN_CACHE_TOKENS = 1024 + +# Same heuristic used by `prompt_call_tracking._estimate_tokens` so warm-up +# audits agree with runtime token reporting. +_WORDS_TO_TOKENS_MULTIPLIER = 1.3 + + +# --------------------------------------------------------------------------- +# Cache classes +# --------------------------------------------------------------------------- + + +class CacheClass: + INSTRUCTION = "instruction" + CONTEXT = "context" + INSTRUCTION_PLUS_CONTEXT = "instruction+context" + NONE = "none" + + +_VALID_CACHE_CLASSES = { + CacheClass.INSTRUCTION, + CacheClass.CONTEXT, + CacheClass.INSTRUCTION_PLUS_CONTEXT, + CacheClass.NONE, +} + + +@dataclass(frozen=True) +class CachePromptEntry: + """One cacheable prompt declaration. + + Attributes: + usage_label: Stable label passed to `invoke_claude(usage_label=...)` + and emitted in runtime logs. Must match what runtime call sites + pass. + instruction_func: Zero-arg callable returning the static + instruction text. May be None if the prompt only uses context + caching (cache_class == CONTEXT) and has no static instruction + we want to warm. + runtime_model: Model alias or full id this prompt is invoked with + at runtime (e.g. "sonnet_latest"). The warming layer resolves + this through `config.resolve_model_id`. + cache_class: One of the CacheClass values. + min_instruction_tokens: Optional explicit minimum size for the + instruction block. When None (the default) the warming layer + uses the per-model threshold from + `llm_utils._min_cache_tokens(resolved_model_id)` — 1024 for + Sonnet 4.5, 4096 for Haiku 4.5, etc. Set this only when an + entry needs a stricter floor than the model defaults. + notes: Free text — usually the reason this entry deviates from the + default (e.g. why something is NONE). + """ + + usage_label: str + instruction_func: Callable[[], str] | None + runtime_model: str + cache_class: str + min_instruction_tokens: int | None = None + notes: str = "" + + def __post_init__(self) -> None: + if self.cache_class not in _VALID_CACHE_CLASSES: + raise ValueError( + f"Unknown cache_class={self.cache_class!r} on {self.usage_label}. " + f"Expected one of {sorted(_VALID_CACHE_CLASSES)}." + ) + if ( + self.cache_class + in ( + CacheClass.INSTRUCTION, + CacheClass.INSTRUCTION_PLUS_CONTEXT, + ) + and self.instruction_func is None + ): + raise ValueError( + f"{self.usage_label}: instruction-cached entries require " + f"an instruction_func." + ) + + +# --------------------------------------------------------------------------- +# Token estimation +# --------------------------------------------------------------------------- + + +def estimate_instruction_tokens(text: str | None) -> int: + """Estimate token count for an instruction string. + + Uses the same words*1.3 heuristic as runtime prompt-call tracking so + warm-up audits and runtime reports stay aligned. + """ + if not text: + return 0 + words = len(re.findall(r"\S+", text)) + return int(round(words * _WORDS_TO_TOKENS_MULTIPLIER)) + + +# --------------------------------------------------------------------------- +# Registry +# --------------------------------------------------------------------------- +# Conventions: +# - usage_label MUST match what call sites pass to `invoke_claude(...)`. +# - runtime_model is the alias the call site passes, not the resolved id. +# - Instruction-only entries that are below 1024 tokens after the latest +# pad expansion are still listed as INSTRUCTION; the warming layer logs +# a below-threshold warning so the gap is visible without silently +# hiding the prompt. + +_REGISTRY: tuple[CachePromptEntry, ...] = ( + # ---------- one_to_n primary / breakout family ---------- + CachePromptEntry( + usage_label="REIMBURSEMENT_PRIMARY", + instruction_func=prompt_templates.REIMBURSEMENT_PRIMARY_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + ), + CachePromptEntry( + usage_label="METHODOLOGY_BREAKOUT", + instruction_func=prompt_templates.METHODOLOGY_BREAKOUT_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + ), + CachePromptEntry( + usage_label="DYNAMIC_CODE_ASSIGNMENT", + instruction_func=prompt_templates.DYNAMIC_CODE_ASSIGNMENT_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + ), + CachePromptEntry( + usage_label="FEE_SCHEDULE_BREAKOUT", + instruction_func=prompt_templates.FEE_SCHEDULE_BREAKOUT_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + ), + CachePromptEntry( + usage_label="GROUPER_BREAKOUT", + instruction_func=prompt_templates.GROUPER_BREAKOUT_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + ), + CachePromptEntry( + usage_label="OUTLIER_BREAKOUT", + instruction_func=prompt_templates.OUTLIER_BREAKOUT_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + ), + CachePromptEntry( + usage_label="CARVEOUT_CHECK", + instruction_func=prompt_templates.CARVEOUT_CHECK_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + ), + CachePromptEntry( + usage_label="DYNAMIC_ASSIGNMENT", + instruction_func=prompt_templates.DYNAMIC_ASSIGNMENT_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + ), + CachePromptEntry( + usage_label="REIMB_DATES_ASSIGNMENT", + instruction_func=prompt_templates.REIMB_DATES_ASSIGNMENT_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + ), + CachePromptEntry( + usage_label="LESSER_OF_DISTRIBUTION", + instruction_func=prompt_templates.LESSER_OF_DISTRIBUTION_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + ), + CachePromptEntry( + usage_label="LESSER_OF_CHECK", + instruction_func=prompt_templates.LESSER_OF_CHECK_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + ), + CachePromptEntry( + usage_label="VALIDATE_REIMBURSEMENTS", + instruction_func=prompt_templates.VALIDATE_REIMBURSEMENTS_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + ), + CachePromptEntry( + usage_label="LOB_RELATIONSHIP", + instruction_func=prompt_templates.LOB_RELATIONSHIP_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + notes=( + "Runtime label is `prompt_lob_relationship` (see alias). " + "Currently below 1024-token threshold; warming layer logs a " + "below-threshold warning until the instruction is expanded." + ), + ), + # ---------- exhibit-level (instruction + context cache) ---------- + CachePromptEntry( + usage_label="EXHIBIT_LEVEL", + instruction_func=prompt_templates.EXHIBIT_LEVEL_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION_PLUS_CONTEXT, + notes=( + "Calls also pass context_for_caching for the exhibit text; " + "instruction reads are stable, context reads only repeat per " + "exhibit." + ), + ), + CachePromptEntry( + usage_label="DYNAMIC_PRIMARY", + instruction_func=prompt_templates.DYNAMIC_PRIMARY_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION_PLUS_CONTEXT, + notes="Exhibit text is sent as context_for_caching across fields.", + ), + CachePromptEntry( + usage_label="DYNAMIC_PRIMARY_ENTITIES", + instruction_func=prompt_templates.DYNAMIC_PRIMARY_ENTITIES_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.CONTEXT, + notes=( + "Static instruction is intentionally short (~180 tokens) and " + "below the 1024-token Sonnet 4.5 cache minimum; padding it " + "would dwarf the actual prompt. Caching here relies on the " + "context_for_caching block (the exhibit text) repeating " + "within a run. CONTEXT class skips instruction warm-up to " + "avoid spending warm-tokens on a block that cannot cache." + ), + ), + CachePromptEntry( + usage_label="DYNAMIC_PRIMARY_ENTITY_CLASSIFICATION", + instruction_func=prompt_templates.DYNAMIC_PRIMARY_ENTITY_CLASSIFICATION_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION_PLUS_CONTEXT, + notes=( + "Classification step that runs after entity extraction. " + "Instruction expanded over the 1024-token Sonnet 4.5 cache " + "minimum so the static block caches; the dynamic entity list " + "in context_for_caching is too small to cache reliably and " + "is expected to miss on the context block." + ), + ), + CachePromptEntry( + usage_label="DYNAMIC_PRIMARY_ENTITIES_ASSIGNMENT", + instruction_func=prompt_templates.DYNAMIC_PRIMARY_ENTITIES_ASSIGNMENT_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION_PLUS_CONTEXT, + notes="Field-level assignment step for DYNAMIC_PRIMARY_ENTITIES; exhibit text passed as context_for_caching.", + ), + # ---------- LOB / program / product mapping ---------- + CachePromptEntry( + usage_label="MAP_LOB_TO_VALID", + instruction_func=prompt_templates.MAP_LOB_TO_VALID_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION_PLUS_CONTEXT, + notes="Maps raw LOB values to valid crosswalk keys; valid-LOB list passed as context_for_caching.", + ), + CachePromptEntry( + usage_label="MAP_NETWORK_TO_VALID", + instruction_func=prompt_templates.MAP_NETWORK_TO_VALID_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION_PLUS_CONTEXT, + notes="Maps raw NETWORK values to valid crosswalk keys; valid-network list passed as context_for_caching.", + ), + CachePromptEntry( + usage_label="AARETE_DERIVED_PROGRAM", + instruction_func=prompt_templates.AARETE_DERIVED_PROGRAM_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION_PLUS_CONTEXT, + notes="Maps extracted PROGRAM names to AARETE_DERIVED_PROGRAM values; valid-values list passed as context_for_caching.", + ), + CachePromptEntry( + usage_label="AARETE_DERIVED_PRODUCT", + instruction_func=prompt_templates.AARETE_DERIVED_PRODUCT_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION_PLUS_CONTEXT, + notes="Maps extracted PRODUCT names to AARETE_DERIVED_PRODUCT values; valid-values list passed as context_for_caching.", + ), + CachePromptEntry( + usage_label="PROGRAM_TO_LOB", + instruction_func=prompt_templates.PROGRAM_TO_LOB_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + notes="Derives LOB values from raw PROGRAM + AARETE_DERIVED_PROGRAM; no context_for_caching at runtime.", + ), + CachePromptEntry( + usage_label="PRODUCT_TO_LOB", + instruction_func=prompt_templates.PRODUCT_TO_LOB_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + notes="Derives LOB values from raw PRODUCT + AARETE_DERIVED_PRODUCT; no context_for_caching at runtime.", + ), + # ---------- service term split (Sonnet 4.5 — Haiku 4.5 won't cache) ---------- + CachePromptEntry( + usage_label="SPLIT_SERVICE_TERM", + instruction_func=prompt_templates.SPLIT_SERVICE_TERM_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + notes=( + "Moved off haiku_latest because Haiku 4.5's 4096-token " + "minimum cache size means the ~1600-token SPLIT_SERVICE_TERM " + "instruction was never cached. Sonnet 4.5's 1024-token " + "minimum makes it cache cleanly. With heavy repetition of " + "service terms across exhibits, cache reads on Sonnet beat " + "uncached Haiku on cost despite the higher per-token rate." + ), + ), + # ---------- preprocessing ---------- + CachePromptEntry( + usage_label="EXHIBIT_HEADER", + instruction_func=prompt_templates.EXHIBIT_HEADER_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + ), + CachePromptEntry( + usage_label="EXHIBIT_LINKAGE", + instruction_func=prompt_templates.EXHIBIT_LINKAGE_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + ), + CachePromptEntry( + usage_label="EXHIBIT_TITLE_MATCH", + instruction_func=prompt_templates.EXHIBIT_TITLE_MATCH_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + ), + # ---------- utility ---------- + CachePromptEntry( + usage_label="DATE_FIX", + instruction_func=prompt_templates.DATE_FIX_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + ), + CachePromptEntry( + usage_label="DERIVED_TERM_DATE", + instruction_func=prompt_templates.DERIVED_TERM_DATE_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + ), + CachePromptEntry( + usage_label="CHECK_PROVIDER_NAME_MATCH", + instruction_func=prompt_templates.CHECK_PROVIDER_NAME_MATCH_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + ), + CachePromptEntry( + usage_label="SPECIAL_CASE_ASSIGNMENT", + instruction_func=prompt_templates.SPECIAL_CASE_ASSIGNMENT_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + ), + CachePromptEntry( + usage_label="SPLIT_REIMB_DATES", + instruction_func=prompt_templates.SPLIT_REIMB_DATES_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + ), + # ---------- code extraction family ---------- + CachePromptEntry( + usage_label="CODE_EXPLICIT", + instruction_func=prompt_templates.CODE_EXPLICIT_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + ), + CachePromptEntry( + usage_label="CODE_CATEGORY", + instruction_func=prompt_templates.CODE_CATEGORY_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + ), + CachePromptEntry( + usage_label="CODE_IMPLICIT_SPECIAL", + instruction_func=prompt_templates.CODE_IMPLICIT_SPECIAL_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + ), + CachePromptEntry( + usage_label="CODE_IMPLICIT", + instruction_func=prompt_templates.CODE_IMPLICIT_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + ), + CachePromptEntry( + usage_label="CODE_IMPLICIT_ARBITRATION", + instruction_func=prompt_templates.CODE_IMPLICIT_ARBITRATION_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + ), + CachePromptEntry( + usage_label="CODE_LAST_CHECK", + instruction_func=prompt_templates.CODE_LAST_CHECK_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + ), + CachePromptEntry( + usage_label="FILL_BILL_TYPE", + instruction_func=prompt_templates.FILL_BILL_TYPE_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + ), + # ---------- one_to_one ---------- + CachePromptEntry( + usage_label="TIN_NPI_TEMPLATE", + instruction_func=prompt_templates.TIN_NPI_TEMPLATE_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + ), + CachePromptEntry( + usage_label="ONE_TO_ONE_SINGLE_FIELD", + instruction_func=prompt_templates.ONE_TO_ONE_SINGLE_FIELD_INSTRUCTION, + runtime_model="sonnet_latest", + cache_class=CacheClass.INSTRUCTION, + ), +) + + +# Aliases let runtime usage_labels that don't match the canonical +# instruction-function name still resolve to one registry entry. +# Callers should prefer canonical labels; this is a compatibility shim. +_USAGE_LABEL_ALIASES: dict[str, str] = { + "code_implicit_rag": "CODE_IMPLICIT", + "prompt_provider_info": "TIN_NPI_TEMPLATE", + "validate_reimbursements_for_llm": "VALIDATE_REIMBURSEMENTS", + "fill_bill_type": "FILL_BILL_TYPE", + "prompt_lob_relationship": "LOB_RELATIONSHIP", +} + + +def _build_index() -> dict[str, CachePromptEntry]: + index: dict[str, CachePromptEntry] = {} + for entry in _REGISTRY: + if entry.usage_label in index: + raise ValueError( + f"Duplicate registry entry for usage_label={entry.usage_label}" + ) + index[entry.usage_label] = entry + return index + + +_INDEX: dict[str, CachePromptEntry] = _build_index() + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + + +def all_entries() -> tuple[CachePromptEntry, ...]: + """Return every registered entry (including NONE / opted-out ones).""" + return _REGISTRY + + +def get_entry(usage_label: str) -> CachePromptEntry | None: + """Look up an entry by usage_label. Resolves aliases.""" + if usage_label in _INDEX: + return _INDEX[usage_label] + canonical = _USAGE_LABEL_ALIASES.get(usage_label) + if canonical is not None: + return _INDEX.get(canonical) + return None + + +def _resolve_min_tokens(entry: CachePromptEntry, resolved_model_id: str) -> int: + """Pick the threshold to enforce: explicit on the entry > per-model > default.""" + if entry.min_instruction_tokens is not None: + return entry.min_instruction_tokens + # Imported lazily so this module stays importable without llm_utils. + from src.utils.llm_utils import _min_cache_tokens + + return _min_cache_tokens(resolved_model_id) + + +def iter_warming_targets() -> Iterator[tuple[str, str, str]]: + """Yield (usage_label, resolved_model_id, instruction_text) tuples for + every entry whose instruction block is supposed to be cached. + + Skips: + - cache_class == NONE + - cache_class == CONTEXT (no static instruction to warm) + - entries whose resolved model does not support prompt caching + - entries whose instruction function fails to render + """ + # Imported here to avoid a circular import: llm_utils → cache_registry + # is allowed, but cache_registry → llm_utils only at call time. + from src.utils.llm_utils import _supports_prompt_cache + + for entry in _REGISTRY: + if entry.cache_class in (CacheClass.NONE, CacheClass.CONTEXT): + continue + if entry.instruction_func is None: + continue + + resolved_model_id = config.resolve_model_id(entry.runtime_model) + if not _supports_prompt_cache(resolved_model_id): + logging.info( + "cache_registry: skipping warm-up for %s — model %s " + "(resolved %s) does not support prompt caching.", + entry.usage_label, + entry.runtime_model, + resolved_model_id, + ) + continue + + try: + instruction_text = entry.instruction_func() + except Exception as exc: # field bundles can be unavailable in CLI/tests + logging.debug( + "cache_registry: skipping %s — instruction function raised: %s", + entry.usage_label, + exc, + ) + continue + + token_estimate = estimate_instruction_tokens(instruction_text) + threshold = _resolve_min_tokens(entry, resolved_model_id) + if token_estimate < threshold: + # Warn but still warm — the call still tracks + # cache_creation_input_tokens in the response, and the runtime + # cache_miss_reason makes the gap visible. + logging.warning( + "cache_registry: %s instruction is ~%d tokens, below the " + "%d-token minimum for %s; cache_control block may not be " + "honored.", + entry.usage_label, + token_estimate, + threshold, + resolved_model_id, + ) + + yield entry.usage_label, resolved_model_id, instruction_text + + +def audit_below_threshold() -> list[tuple[str, int, int]]: + """Return (usage_label, estimated_tokens, min_required) for every + INSTRUCTION-class entry whose instruction is below threshold. The + threshold is resolved per-model: Sonnet 4.5 = 1024, Haiku 4.5 = 4096, + etc. Useful for a startup audit / contract test. + """ + findings: list[tuple[str, int, int]] = [] + for entry in _REGISTRY: + if entry.cache_class not in ( + CacheClass.INSTRUCTION, + CacheClass.INSTRUCTION_PLUS_CONTEXT, + ): + continue + if entry.instruction_func is None: + continue + try: + text = entry.instruction_func() + except Exception: + continue + resolved_model_id = config.resolve_model_id(entry.runtime_model) + threshold = _resolve_min_tokens(entry, resolved_model_id) + tokens = estimate_instruction_tokens(text) + if tokens < threshold: + findings.append((entry.usage_label, tokens, threshold)) + return findings + + +def instruction_cache_entries() -> Iterable[CachePromptEntry]: + """Entries where the instruction block is expected to be cached.""" + return tuple( + e + for e in _REGISTRY + if e.cache_class + in (CacheClass.INSTRUCTION, CacheClass.INSTRUCTION_PLUS_CONTEXT) + ) + + +def context_cache_entries() -> Iterable[CachePromptEntry]: + """Entries where a dynamic context block is cached separately.""" + return tuple( + e + for e in _REGISTRY + if e.cache_class in (CacheClass.CONTEXT, CacheClass.INSTRUCTION_PLUS_CONTEXT) + ) diff --git a/src/prompts/fieldset.py b/src/prompts/fieldset.py index df1e21e..ae53ff9 100644 --- a/src/prompts/fieldset.py +++ b/src/prompts/fieldset.py @@ -166,7 +166,7 @@ class Field: """Resolves placeholders in the prompt using the valid_values property.""" # If no valid_values to resolve - if string_utils.is_empty(self.valid_values): # + if string_utils.is_empty(self.valid_values): return self.prompt if not string_utils.is_empty(self.valid_values) and not constants: @@ -177,12 +177,13 @@ class Field: resolved_prompt = self.prompt if "{valid_values}" in resolved_prompt: resolved_value = self.valid_values - if hasattr(constants, resolved_value): + # Only call hasattr if resolved_value is a string + if isinstance(resolved_value, str) and hasattr(constants, resolved_value): try: resolved_prompt = resolved_prompt.replace( "{valid_values}", str(constants.get_constant(resolved_value)) ) - except: + except Exception: resolved_prompt = resolved_prompt.replace( "{valid_values}", str(resolved_value) ) diff --git a/src/prompts/investment_prompts.json b/src/prompts/investment_prompts.json index 5e61080..391153d 100644 --- a/src/prompts/investment_prompts.json +++ b/src/prompts/investment_prompts.json @@ -99,8 +99,7 @@ "field_name": "LOB", "relationship": "one_to_n", "field_type": "dynamic_primary", - "prompt": "A distinct, high-level category of health insurance coverage. Choose LOBs ONLY from the following values: {valid_values}. Do NOT extract Programs, Products, or other entities listed. Choose ONLY from the valid values presented. Only return values that are explicitly written - do NOT fill this value out by deducing the answer from sub-Programs that may fall under one of the valid answers. If only the sub-Program is seen, return 'N/A'. Do NOT extract an LOB that appears only incidentally as part of a rate name, reimbursement methodology, or pricing reference (e.g., 'Medicaid Base DRG rate'). Only extract an LOB if the text explicitly declares or restricts the scope of the exhibit or agreement to that LOB.", - "valid_values": "VALID_LOBS", + "prompt": "A distinct, high-level category of health insurance coverage. Only return values that are explicitly written in the contract. Do NOT extract an LOB that appears only incidentally as part of a rate name, reimbursement methodology, or pricing reference (e.g., 'Medicaid Base DRG rate'). Only extract an LOB if the text explicitly declares or restricts the scope of the exhibit or agreement to that LOB.", "retrieval_question": "What Line of Business does this contract apply to?" }, { @@ -115,8 +114,7 @@ "field_name": "NETWORK", "relationship": "one_to_n", "field_type": "dynamic_primary", - "prompt": "A type of managed care structures that defines what providers a patient can use. Choose Networks ONLY from the following values: {valid_values}. Only return values that are explicitly written.", - "valid_values": "VALID_NETWORKS", + "prompt": "A type of managed care structures that defines what providers a patient can use. Only return values that are explicitly written in the contract.", "retrieval_question": "What Network (HMO, PPO, etc) does this contract apply to?" }, { @@ -131,32 +129,29 @@ "field_name": "PRODUCT", "relationship": "one_to_n", "field_type": "dynamic_primary", - "prompt": "The brand name of a health plan or product offered by the Payer. Choose Products ONLY from the following values: {valid_values}. Choose ONLY from the valid values presented; if no value from the list is explicitly mentioned, return 'N/A'. Do NOT include the name of the Payer itself (such as 'Molina', 'Centene' etc) - rather, look for the Product that these healthplans offer. Do NOT extract LOBs (such as 'Medicaid', 'Medicare', 'Marketplace', 'Duals') or Programs (such as 'Medicaid Managed Care', 'Medicare Advantage', 'ACA') as products - those belong in the LOB and PROGRAM fields respectively. Return values exactly as written in the valid values list.", - "valid_values": "VALID_PRODUCTS", + "prompt": "The brand name of a health plan or product offered by the Payer. Do NOT include the name of the Payer itself (such as 'Molina', 'Centene' etc) - rather, look for the Product that these health plans offer. List any products explicitly stated in the text.", "retrieval_question": "What Product does this contract apply to?" }, { "field_name": "AARETE_DERIVED_PRODUCT", "relationship": "one_to_n", - "field_type": "TBD", - "prompt": "TBD", - "base_field": "PRODUCT" + "field_type": "derived_dynamic_primary", + "base_field": "PRODUCT", + "valid_values": "VALID_AARETE_DERIVED_PRODUCTS" }, { "field_name": "PROGRAM", "relationship": "one_to_n", "field_type": "dynamic_primary", - "prompt": "The brand name of a health plan or product offered by the State or Federal government. List any Programs explicitly stated in the text. Choose ONLY from the following: {valid_values}. Note that 'Medicare' on it's own does NOT mean 'Medicare Advantage'. If only 'Medicare' is seen, do not write 'Medicare Advantage'. IMPORTANT - CHIP vs CHIPP distinction: 'CHIP' (Children's Health Insurance Program) is a valid standalone value. If you see 'CHIP HMO' or just 'CHIP' (without 'Perinate'), extract 'CHIP'. However, if you see 'CHIP Perinate' or 'CHIP-P' (Children's Health Insurance Program Perinate), extract 'CHIPP' (NOT 'CHIP'). These are two distinct programs - do not confuse them. Choose the answer from the list that most closely matches what is found in the text. Use reasonable judgment for obvious synonyms and equivalent terms (e.g., 'Medicare-Medicaid Program' matches 'Medicare-Medicaid Plan'). Do NOT extract a program value when it appears only as a regulatory citation, legal reference, or compliance note (e.g., 'section 2713 of ACA', 'pursuant to 42 CFR', 'under ERISA', 'as defined by ACA'). Only extract a program if the text explicitly identifies it as the type of coverage or program governing the scope of the agreement or exhibit.", - "valid_values": "VALID_PROGRAMS", + "prompt": "The brand name of a health plan or product offered by the State or Federal government. List any Programs explicitly stated in the text. Do NOT extract a program value when it appears only as a regulatory citation, legal reference, or compliance note (e.g., 'section 2713 of ACA', 'pursuant to 42 CFR', 'under ERISA', 'as defined by ACA'). Only extract a program if the text explicitly identifies it as the type of coverage or program governing the scope of the agreement or exhibit.", "retrieval_question": "What Program(s) does this contract apply to?" }, { "field_name": "AARETE_DERIVED_PROGRAM", "relationship": "one_to_n", - "field_type": "crosswalk", - "prompt": "TBD", - "crosswalk": "CROSSWALK_PROGRAM", - "base_field": "PROGRAM" + "field_type": "derived_dynamic_primary", + "base_field": "PROGRAM", + "valid_values": "VALID_AARETE_DERIVED_PROGRAMS" }, { "field_name": "CONTRACT_TITLE", diff --git a/src/prompts/prompt_templates.py b/src/prompts/prompt_templates.py index 1f4bc9e..81a84d9 100644 --- a/src/prompts/prompt_templates.py +++ b/src/prompts/prompt_templates.py @@ -192,10 +192,23 @@ def get_cacheable_instructions(): if config.FIELDS in ["all", "one_to_n"]: try: cacheable_instructions["DYNAMIC_PRIMARY"] = DYNAMIC_PRIMARY_INSTRUCTION() + cacheable_instructions["DYNAMIC_PRIMARY_ENTITIES"] = ( + DYNAMIC_PRIMARY_ENTITIES_INSTRUCTION() + ) + cacheable_instructions["DYNAMIC_PRIMARY_ENTITY_CLASSIFICATION"] = ( + DYNAMIC_PRIMARY_ENTITY_CLASSIFICATION_INSTRUCTION() + ) + cacheable_instructions["MAP_LOB_TO_VALID"] = MAP_LOB_TO_VALID_INSTRUCTION() + cacheable_instructions["MAP_NETWORK_TO_VALID"] = ( + MAP_NETWORK_TO_VALID_INSTRUCTION() + ) cacheable_instructions["CARVEOUT_CHECK"] = CARVEOUT_CHECK_INSTRUCTION() cacheable_instructions["DYNAMIC_ASSIGNMENT"] = ( DYNAMIC_ASSIGNMENT_INSTRUCTION() ) + cacheable_instructions["DYNAMIC_PRIMARY_ENTITIES_ASSIGNMENT"] = ( + DYNAMIC_PRIMARY_ENTITIES_ASSIGNMENT_INSTRUCTION() + ) cacheable_instructions["REIMB_DATES_ASSIGNMENT"] = ( REIMB_DATES_ASSIGNMENT_INSTRUCTION() ) @@ -289,6 +302,61 @@ Extract Exhibit-Level attributes from a section of a contract. - If there are no correct answers for a given field, write 'N/A' as the answer for that field. - Be consistent and deterministic; avoid creative paraphrasing. +[DETAILED GUIDANCE FOR EXHIBIT-LEVEL EXTRACTION] +- Exhibit-level attributes are those that apply uniformly across all rates and services listed in the exhibit, not just to individual line items. +- Look for information in exhibit headers, introductory paragraphs, or statements that explicitly cover the entire exhibit. +- Avoid inferring answers from individual rate descriptions unless they are explicitly stated to apply to all. +- For fields like Claim Type and Bill Type, even if every service in the exhibit suggests the same type, do not assume it applies exhibit-wide unless stated. +- Common exhibit headers might include terms like "Exhibit A: Professional Services", "Schedule of Rates", or "Fee Schedule for All Services". +- Pay attention to qualifiers like "all rates", "entire exhibit", "applicable to all", or similar universal statements. + +[VALIDATION RULES] +- Ensure that the extracted value is directly supported by the text in a way that applies to the whole exhibit. +- If the text mentions different values for different parts of the exhibit, do not extract any value. +- Cross-reference with the exhibit structure: headers, footers, and overarching statements take precedence. +- Maintain strict adherence to valid value lists; no synonyms or variations unless explicitly allowed. + +[EXAMPLES OF VALID EXTRACTION] +- If the exhibit header says "Exhibit B: Inpatient Services - All Rates Apply to Medicare Claims", then Claim Type could be "Medicare". +- If a paragraph states "The following rates apply to all Blue Cross Blue Shield bill types", then Bill Type might be extractable if consistent. +- If individual rates mention different claim types, leave Claim Type as 'N/A' even if one type dominates. + +[COMMON PITFALLS TO AVOID] +- Do not extract from individual rate lines unless they explicitly state exhibit-wide application. +- Avoid assuming uniformity based on majority; it must be explicitly universal. +- Do not use service descriptions to infer claim or bill types at exhibit level. +- Resist the temptation to paraphrase or interpret loosely; stick to exact matches. +- Do not infer from context outside the exhibit text provided. +- Avoid over-generalizing from partial statements. + +[ADDITIONAL EXTRACTION PRINCIPLES] +- Prioritize explicit statements over implicit ones. +- Look for contractual language that binds the entire exhibit. +- Consider the hierarchical structure: exhibit > section > rate. +- Validate against the contract's overall purpose and stated scope. +- Ensure answers are defensible with direct quotes from the text. +- Maintain neutrality; do not favor one interpretation over another without textual support. + +[FIELD-SPECIFIC NOTES] +- For dates and periods: Look for effective dates, term periods, or renewal clauses that apply to the whole exhibit. +- For payer information: Check for payer names, TINs, or identifiers mentioned in headers or introductory text. +- For methodology: Identify pricing methodologies stated to apply to all rates in the exhibit. +- For modifiers: Only if explicitly stated to apply universally across all services and rates. + +[QUALITY ASSURANCE CHECKS] +- Does the answer apply to 100% of the rates, not 99% or even all visible ones? +- Is the supporting text in the exhibit header, title, or a universal statement? +- Would removing any part of the exhibit change the answer? +- Is the answer consistent with the exhibit's stated purpose? + +[OUTPUT FORMAT REQUIREMENTS] +- Provide a brief explanation for each field before stating the final value. +- The explanation should reference the specific text that supports the answer. +- If no supporting text exists, explain why 'N/A' is used. +- Ensure the JSON dictionary uses exact field names as specified. +- Validate JSON syntax: proper quotes, commas, and structure. +- Include all specified fields in the output, even if 'N/A'. + [OUTPUT FORMAT] Briefly explain your answer, then put the final answer in the properly-formatted JSON dictionary. {JSON_DICT_FORMAT_INSTRUCTIONS}""" @@ -327,16 +395,82 @@ def DYNAMIC_PRIMARY_INSTRUCTION() -> str: Contains extraction rules for text-based fields (allows obvious assumptions). """ return f"""[OBJECTIVE] -Extract attribute values for dynamic fields from contract text, paying close attention to detail. +Extract attribute values for dynamic fields from contract text with precision and consistency. [GENERAL INSTRUCTIONS] -- Return all valid, explicitly stated values. If multiple values are found, include all of them as separate items in the JSON list. -- If none of the valid values appear explicitly in the text, return ["N/A"]. -- Make appropriate and obvious assumptions, but don't take huge leaps of logic. - - For example, you can assume that "Children's Health Insurance Program Perinate (CHIP-P)" is equivalent to "CHIP Perinate", but do not assume that "Children's Health Insurance Program Perinate (CHIP-P)" is equivalent to "Medicaid". +- Return all valid values that are explicitly stated in the text. +- If multiple valid values are present, include each as a separate item in the JSON list. +- If none of the valid values appear explicitly, return ["N/A"] and do not guess. +- Make only obvious, narrowly scoped assumptions. Do not make broad semantic leaps. + - For example, "Children's Health Insurance Program Perinate (CHIP-P)" is equivalent to "CHIP Perinate", but it is not equivalent to "Medicaid". + +[EXTRACTION GUIDANCE] +- Prefer exact matches to the valid values list. Only use synonyms when they are clearly equivalent and explicit in the text. +- Exclude boilerplate text, headers, legends, or unrelated contract language that does not directly answer the field. +- Treat different phrases as separate values unless they clearly refer to the same single concept. +- Do not infer a value from a nearby section or table unless the field definition explicitly requires it. + +[AMBIGUITY AND CONFIDENCE] +- If the text is ambiguous between two or more valid values, include all plausible values rather than selecting one arbitrarily. +- If the correct value cannot be determined from the text alone, return ["N/A"]. +- Avoid using a single phrase that is only tangentially related to the field as the answer. + +[QUALITY CHECKS] +- Does each returned value appear explicitly or clearly in the text? +- Are all returned values valid according to the field's expected set? +- Is the output deterministic and repeatable across similar inputs? +- If no valid values are present, did you correctly return ["N/A"]? + +[SCOPE BOUNDARIES] +- Each call extracts the values for ONE field (the [ATTRIBUTE TO EXTRACT] block names it). Do not return values intended for a sibling field even when that information is visible in the same context. +- Do not concatenate multiple distinct values into a single string. Each distinct value is its own list element. +- A field with a closed valid-values list takes its answers ONLY from that list. A near-synonym maps to its canonical entry only when the contract makes the equivalence explicit (e.g., a parenthetical alias). + +[SOURCE TEXT INTERPRETATION] +- Treat exhibit headers, table headings, signature blocks, and footers as scaffolding, not as field values. +- A program or methodology mentioned only as a pricing benchmark (e.g., "100% of Medicare allowable") does not establish that the contract IS for that program. The pricing reference is part of the methodology, not the LOB or PRODUCT. +- A benefit example listing "what other plan types might also pay" is illustrative and does not populate the field for this contract. + +[REASONING DISCIPLINE] +- Cite the specific phrase in the context that supports each returned value. Keep explanations to one or two short sentences per value. +- Do not restate the rules above. Do not invent or extrapolate context. +- If two values are tied on evidence and the field instruction does not allow both, return ["N/A"] rather than picking arbitrarily. + +[WORKED EXAMPLES] +The examples below illustrate the rules already stated; they are not new rules. Use them as calibration only. + +Example — explicit alias maps to canonical valid value. +Context: "Marketplace plans (also known as ACA Exchange products) are eligible for the rates in this Exhibit." +Field has valid value "ACA". +Correct: ["ACA"] — the contract makes the equivalence explicit via parenthetical alias. +Incorrect: ["Marketplace"] when "Marketplace" is not in the valid-values list. + +Example — pricing benchmark is not the field value. +Context: "Provider shall be reimbursed at 100% of Medicare Allowable for outpatient services rendered to Members." +Field: LOB. +Correct: ["N/A"] — Medicare here is a pricing benchmark inside the methodology. The contract does not state that the LOB is Medicare. +Incorrect: ["Medicare"] inferred from the pricing reference alone. + +Example — multiple distinct values must be separate list elements. +Context: "This Agreement covers Commercial PPO and Marketplace HMO products." +Field: NETWORK with valid values including "PPO" and "HMO". +Correct: ["PPO", "HMO"] +Incorrect: ["PPO and HMO"] or ["PPO/HMO"] + +Example — values mentioned in a contrastive clause do not populate the field. +Context: "Unlike Medicare Advantage plans, this contract is for a Medicaid Managed Care population." +Field: LOB. +Correct: ["Medicaid"] +Incorrect: ["Medicare", "Medicaid"] — Medicare is referenced in a contrastive clause that explicitly excludes it from the contract scope. + +Example — heading is scaffolding, not a value. +Context: An exhibit titled "EXHIBIT B: COMMERCIAL FEE SCHEDULE" containing a table of CPT-level rates with no other LOB mention in the body. +Field: LOB with valid value "Commercial". +Correct: ["Commercial"] — the exhibit header is bound to the rates and explicitly names the LOB scope. +Incorrect: ["N/A"] when the exhibit header itself is the explicit scope statement. [OUTPUT FORMAT] -Briefly explain your answer before putting the final answer in a properly-formatted JSON list. +Briefly explain your answer why each entity is extracted before putting the final answer in a properly-formatted JSON list. {JSON_LIST_FORMAT_INSTRUCTIONS}""" @@ -368,6 +502,407 @@ Here is the text to analyze: return (context_text, attribute_text, parser) +def DYNAMIC_PRIMARY_ENTITIES_INSTRUCTION( + combined_field_prompt: str | None = None, +) -> str: + """Static instruction for combined dynamic primary entity extraction.""" + combined_fields_block = "" + if not string_utils.is_empty(combined_field_prompt): + combined_fields_block = f""" +[FIELDS TO CONSOLIDATE] +Use these field definitions and instructions to extract one combined list of entities: +{combined_field_prompt} +""" + + return f"""[OBJECTIVE] +Extract a single consolidated list of dynamic primary entities for the exhibit. + +[INSTRUCTIONS] +- Use the provided field definitions for LOB, PROGRAM, PRODUCT, and NETWORK as guidance. +- Return all distinct entities that may be relevant to any of those four fields. +- Pay close attention to exhibit headers, subheaders, table headings, and any text that explicitly defines the scope of the exhibit. +- Do not classify entities into categories in this step. +- If no entities are found, return ["N/A"]. +- CRITICAL INSTRUCTION: Extract only entities that are explicitly and directly stated in the contract text. Preserve the exact wording used in the contract without modification. + +[DO NOT EXTRACT COMPOSITE TERMS] +- Extract each atomic component separately — do NOT bundle multiple categories into a single string. +- If the contract text contains a compound phrase that mixes a LOB, PROGRAM, PRODUCT, or NETWORK together, +extract the individual components as separate list entries rather than the composite phrase as one entry. + +Examples: +- "CHIP HMO" → extract "CHIP" and "HMO" separately, NOT "CHIP HMO" as one entry. +- "Medicaid STAR" → extract "Medicaid" and "STAR" separately, NOT "Medicaid STAR" as one entry. +- "Medicare DSNP" → extract "Medicare" and "DSNP" separately. +- "Medicaid PPO" → extract "Medicaid" and "PPO" separately. +- "Ohio Medicaid" → extract "Medicaid" only; state prefixes are geographic modifiers, not entities. +- "Texas Medicaid STAR+PLUS" → extract "Medicaid" and "STAR+PLUS" separately. +Only extract the composite as a single entry when it is clearly a proper branded name (e.g., "Blue Cross PPO", "Aetna Medicare Advantage") OR when the qualifier is integral to the program's official identity. The following are atomic program names — do NOT decompose them: +- "CHIP Perinate" / "CHIP Perinatal" / "CHIP-P" → single PROGRAM entry (Perinate is not a separate entity; it is the qualifier that makes this distinct from base CHIP) +- "STAR+PLUS" → single PROGRAM entry +- "CHIP HMO" is an exception to the above: CHIP and HMO ARE separate categories, so decompose it. + +[ENTITY NAME CLEANING] +Before adding an entity to the output list, apply all of the following rules in order: + +1. STRIP GENERIC CATEGORY WORDS — Remove trailing or embedded descriptor words "Product", "Products", "Program", "Programs", "Plan", "Plans" when they are NOT part of an official proper name. + - "Health Insurance Marketplace Product" → "Health Insurance Marketplace" + - "Medicaid Managed Care Program" → "Medicaid Managed Care" + - "CHIP Program" → "CHIP" + Exception: keep the word when it is integral to the official program brand (e.g., "STAR+PLUS Program" may keep "Program" if that is how the contract spells it). + +2. REMOVE PAYER / COMPANY NAME PREFIXES — Do not include the payer or company name as a prefix unless the payer name IS the entity (i.e., the product is identified solely by the payer brand). + - "Molina Health Insurance Marketplace Product" → "Health Insurance Marketplace" (then apply rule 1) + - "Molina Health Benefit Exchange Product" → "Health Benefit Exchange" + - "CareSource Indiana Marketplace" → after applying rule 3 below → "Marketplace" + Keep the payer name when it IS the meaningful identifier: "PCHP", "Parkland Community Health Plan", "BlueCross BlueShield HMO" stay intact. + Heuristic: if removing the payer prefix leaves a well-known LOB, program, network, or industry-standard value, remove the payer prefix. + +3. REMOVE U.S. STATE NAME PREFIXES — Remove a U.S. state name that is merely a geographic qualifier when removing it leaves a recognizable entity. + - "Indiana Marketplace" → "Marketplace" + - "Ohio Medicaid" → "Medicaid" + - "Texas CHIP" → "CHIP" + - "CareSource Indiana Marketplace" → first strip payer "CareSource", then strip state "Indiana" → "Marketplace" + Keep the state when it IS the distinct identifier of a state-specific branded plan and no simpler term applies. + +4. DEDUPLICATE ACRONYM / LONG-FORM PAIRS — When both a full-name form and a standalone abbreviation of the same concept are present as separate extracted items, collapse them into ONE entry using the most informative form found in the contract: + - If the contract already wrote the combined form "Full Name (ABBR)" (e.g., "Dual Special Needs Plan (DSNP)"), use that combined form exactly and discard the standalone abbreviation. + - If only the standalone abbreviation is present (e.g., only "DSNP" with no long-form entry), keep it exactly as-is. + - If only the long-form is present with no abbreviation entry (e.g., only "Dual Special Needs Plan"), keep it exactly as-is — do NOT invent or append an abbreviation. + Examples: + - Both "Dual Special Needs Plan (DSNP)" and "DSNP" extracted → keep "Dual Special Needs Plan (DSNP)", discard "DSNP" + - Only "DSNP" extracted → keep "DSNP" as-is + - Only "Dual Special Needs Plan" extracted (no parenthetical abbreviation in the contract) → keep "Dual Special Needs Plan" as-is, do NOT change to "DSNP" + +5. CANONICAL PROGRAM FORMS — The only normalization applied is for bare qualifiers that are not standalone program names: + - "Perinate", "Perinatal", and "CHIP-P" are NOT standalone entities. They are qualifiers that must always remain attached to "CHIP". Normalize bare "Perinate" or "Perinatal" to "CHIP Perinate". Never extract them alone. + - If you have already extracted the abbreviation separately, do not add the long form again. + +{combined_fields_block} + +[OUTPUT FORMAT] +Briefly explain your reasoning for extracting each entity. Return only a valid JSON list of strings. +{JSON_LIST_FORMAT_INSTRUCTIONS}""" + + +def DYNAMIC_PRIMARY_ENTITIES( + context: str, +) -> Tuple[str, str, Callable[[str], list]]: + """Prompt payload for extracting a single combined dynamic primary entity list.""" + context_text = f"""[CONTEXT] +Here is the text to analyze: +{context.replace('"', "'")}""" + + question_text = ( + "Extract one consolidated list of dynamic primary entities from this context." + ) + + return (context_text, question_text, _json_list_parser) + + +def DYNAMIC_PRIMARY_ENTITY_CLASSIFICATION_INSTRUCTION() -> str: + """Static instruction for classifying extracted dynamic primary entities.""" + return f"""[OBJECTIVE] +Classify extracted dynamic primary entities into one of these categories: LOB, PROGRAM, PRODUCT, NETWORK. + +[GENERAL INSTRUCTIONS] +- Classify each entity into the SINGLE best matching category, or exclude it if not valid. +- Some entities are compound strings that combine components from multiple categories (e.g., "CHIP HMO", "Medicaid STAR"). For these, decompose and emit each component into its own category instead of placing the entire string into one field. +- Not every extracted entity is necessarily valid; exclude entities that are irrelevant, ambiguous, or unsupported. + +[ANTI-DUPLICATION PROTOCOL — MANDATORY] +You MUST follow these four steps in order. Do not skip any step. + +STEP 1 — ENTITY ASSIGNMENT TABLE +Before building any output list, write out an assignment table with one row per entity: + "" → (or EXCLUDED) +Every entity must appear in this table exactly once with exactly one category. +ACRONYM/LONG-FORM DEDUPLICATION: If the input contains both a combined "Full Name (ABBR)" entry AND a standalone abbreviation for the same concept, collapse them into one. Keep the combined form (e.g., "Dual Special Needs Plan (DSNP)") and mark the standalone abbreviation (e.g., "DSNP") as EXCLUDED. + If only the standalone abbreviation is present (no long-form entry), keep it exactly as-is. + If only the long-form is present with no abbreviation entry, keep it exactly as-is — do NOT invent or append an abbreviation. + +STEP 2 — BUILD OUTPUT LISTS FROM TABLE ONLY +Build the four output lists strictly from your Step 1 table. +Each entity goes into exactly one list — the list matching its assigned category. +Do NOT copy any entity into a second list. + +STEP 3 — CROSS-CHECK FOR DUPLICATES +Scan all four lists combined. If any entity string appears in more than one list: + - Remove it from all lower-priority lists. + - Priority order (highest → lowest): LOB → NETWORK → PROGRAM → PRODUCT. + - Keep it only in the single highest-priority list where it appears. + +STEP 4 — OUTPUT +Only after completing Steps 1–3, output the final JSON dictionary. + +[CATEGORY DEFINITIONS] +- LOB: Broad administrative insurance coverage category that defines who is covered under the contract. + Typical examples: Medicaid, Medicare, Commercial, Marketplace, Duals. + The following terms should be classified as LOB — never place them in PROGRAM or PRODUCT: + Health Insurance Exchange (HIE), Health Benefit Exchange (HBE), Affordable Care Act (ACA), + Individual and Family Market (IFM), Health Connector, Commercial-Exchange, Marketplace. + An entity belongs in LOB if it names the overarching payer program type or insurance coverage category — + not a specific plan brand, delivery model, or sub-program variant. +- NETWORK: A type of managed care structures that defines what providers a patient can use. + Typical examples: HMO, PPO, EPO, POS, PFFS, Indemnity. + An entity belongs in NETWORK if it describes how the provider network is structured or how beneficiaries access care. +- PROGRAM: Government/state/public program names and public coverage program variants like CHIP, SNP, DSNP, STAR, STAR+PLUS. +- PRODUCT: Payer-branded or plan-branded commercial product names and plan labels like example Parkland Community Health Plan(PCHP). + +[CLASSIFY BY MEANING, NOT BY WORDS IN THE NAME] +Always classify based on what the entity fundamentally IS, not on individual words that appear in its name. +A word like "Product", "Plan", "Program", or "Network" in an entity name does NOT determine its category. +You must look at the entity as a whole and determine its true nature. + +Examples of this rule in practice: +- "Medicaid Product" → LOB (Medicaid is a broad coverage category; "Product" in the name is irrelevant) +- "Medicare Advantage" → LOB (Medicare Advantage is an overarching coverage category/LOB, not a specific program variant) +- "Medicaid Managed Care" → LOB (still Medicaid as the coverage category) +- "Blue Cross Medicaid Plan" → PRODUCT (this is a payer-branded plan, not the Medicaid program itself) +- "CHIP Program" → PROGRAM (CHIP is a government program; "Program" in the name confirms it) +- "Aetna Medicare" → PRODUCT (payer-branded; not the Medicare program itself) + +[COMPOUND ENTITY DECOMPOSITION] +When an extracted entity is a compound string that fuses components from different categories, decompose it. +Do NOT place the entire compound string into one field — emit each component into its correct category. + +Decomposition rules: +- " " pattern → LOB component goes to LOB, program variant goes to PROGRAM. + Example: "Medicaid STAR" → LOB=["Medicaid"], PROGRAM=["STAR"] + Example: "Medicare DSNP" → LOB=["Medicare"], PROGRAM=["DSNP"] +- " " pattern → program name goes to PROGRAM, network type goes to NETWORK. + Example: "CHIP HMO" → PROGRAM=["CHIP"], NETWORK=["HMO"] + Example: "Medicaid PPO" → LOB=["Medicaid"], NETWORK=["PPO"] + +EXCEPTION — Do NOT decompose these established program variant names; they are atomic and must remain intact as single PROGRAM entries: +- "CHIP Perinate" / "CHIP Perinatal" / "CHIP-P" → PROGRAM ("Perinate"/"Perinatal" is not a separate entity; it is the qualifier that distinguishes this from base CHIP — do NOT split into "CHIP" + "Perinate") +- "STAR+PLUS" → PROGRAM (single atomic program name) +- "Medicare Advantage" → LOB (always; it is an overarching coverage category, not a program variant) +- "SNP" (Special Needs Plan) when used as a standalone term → PROGRAM (not NETWORK; SNP describes a program type, not a network delivery model) +- " " pattern → the state prefix is a geographic modifier, not a program name. Classify the core LOB into LOB; do NOT create a PROGRAM entry. + Example: "Ohio Medicaid" → LOB=["Medicaid"] (discard "Ohio" as a geographic qualifier) + Example: "Texas Medicaid" → LOB=["Medicaid"] +- When an entity contains an LOB keyword (Medicaid, Medicare, Commercial, Marketplace, Duals) combined with a specific program variant, always place the LOB keyword into LOB and the program variant into PROGRAM — never keep the compound string intact in PROGRAM or PRODUCT. + +[LOB EXCLUSIVITY RULE] +Entities that name a broad coverage category (e.g., Medicaid, Medicare, Commercial, Marketplace, Duals) belong ONLY in LOB. +Do NOT place them in PROGRAM, PRODUCT, or NETWORK — even if they superficially resemble a program or product name. + +[PROGRAM VS PRODUCT DIFFERENTIATION] +- PROGRAM is for government/state/federal program constructs. +- PRODUCT is for payer or plan branding tied to specific plans/offerings. +- If an entity looks like a branded plan name or payer-specific plan label, classify as PRODUCT only. +- If an entity looks like a public program name or public program variant, classify as PROGRAM only. +- If still ambiguous, pick whichever single category best fits the evidence — do not default to either; choose the stronger match. +- NEVER place the same entity in both PROGRAM and PRODUCT. + +[ENTITY NORMALIZATION BEFORE CLASSIFICATION] +Before writing any entity to an output list, apply these rules: + +- ACRONYM/LONG-FORM PAIRS: When the input contains both a combined "Full Name (ABBR)" form AND a standalone abbreviation for the same concept, keep the combined form and discard the standalone abbreviation. + Example: input has both "Dual Special Needs Plan (DSNP)" and "DSNP" → output "Dual Special Needs Plan (DSNP)" once only. + Do NOT normalize a standalone form to a different canonical form: if only "Dual Special Needs Plan" is in the input (no abbreviation), write "Dual Special Needs Plan" — do NOT change it to "DSNP". + If only "DSNP" is in the input, write "DSNP" — do NOT expand it. + +- CHIP Perinate: "Perinate", "Perinatal", and "CHIP-P" are NOT standalone entities. They are qualifiers that belong attached to "CHIP". Normalize bare "Perinate" or "Perinatal" to "CHIP Perinate". Do NOT classify standalone "Perinate" or "Perinatal" as a PROGRAM entry. + If both "CHIP" and "CHIP Perinate" appear, keep both — they are distinct (CHIP is the parent program, CHIP Perinate is a variant). + +[FINAL CHECKLIST BEFORE OUTPUT] +- Step 1 table written: every entity appears exactly once with exactly one category (or EXCLUDED). +- Step 2 lists built only from the table — no entity invented and no entity dropped except those marked EXCLUDED. +- Step 3 cross-check: scan the four lists; if any entity string appears twice, retain it only in the highest-priority list per LOB → NETWORK → PROGRAM → PRODUCT. +- Step 3b normalization check: scan the PROGRAM list for duplicate-concept violations — if both a combined "Full Name (ABBR)" form and a standalone abbreviation for the same concept are present, collapse to the combined form only. +- Final dictionary has exactly the keys "LOB", "PROGRAM", "PRODUCT", "NETWORK". +- Each value is a JSON list (possibly empty); no value is a single string or null. +- Casing of every entity string must match exactly how it appeared in the contract input (no invention of abbreviations or expansions). + +[OUTPUT FORMAT] +Write your Step 1 assignment table, then output the final JSON dictionary. +Return only a valid JSON dictionary with exactly these keys: "LOB", "PROGRAM", "PRODUCT", "NETWORK". +Each field value must be a JSON list (possibly empty). +{JSON_DICT_FORMAT_INSTRUCTIONS}""" + + +def DYNAMIC_PRIMARY_ENTITY_CLASSIFICATION( + dynamic_primary_entities: list[str], +) -> Tuple[str, str, Callable[[str], dict]]: + """Build prompt payload for dynamic primary entity classification.""" + entities_text = ", ".join(f'"{entity}"' for entity in dynamic_primary_entities) + + context_text = f"""[CONTEXT] +Dynamic primary entities extracted from the contract: +{entities_text}""" + + attribute_text = """[CLASSIFICATION TASK] +Classify each dynamic primary entity into ONLY one of the following categories: LOB, PROGRAM, PRODUCT, NETWORK. +Follow all rules and the anti-duplication protocol defined in the instruction.""" + + return (context_text, attribute_text, _json_dict_parser) + + +def AARETE_DERIVED_PROGRAM_INSTRUCTION() -> str: + """Static instruction for AARETE_DERIVED_PROGRAM prompt caching.""" + return f"""[OBJECTIVE] +Map extracted PROGRAM names to standardized AARETE_DERIVED_PROGRAM values. + +[INSTRUCTIONS] +- For each PROGRAM value provided, find the best semantically matching value from the valid output values list. +- Only return values from the valid output values list. Do NOT invent or modify values. +- If a PROGRAM value does not clearly match any valid output value, exclude it. +- Return a flat JSON list of the matched derived values (no duplicates, preserve relevance order). +- If no PROGRAM values match, return an empty JSON list []. +- CRITICAL — Do NOT force-fit: if a PROGRAM value is only loosely or partially related to a valid output value, do NOT map it. A match requires genuine semantic equivalence or a clear subset/superset relationship in the same program category. For example, "Medicare Advantage" is NOT a match for "DSNP" — DSNP is a specific subset of Medicare Advantage plans, not synonymous with it. Only map to DSNP when the program explicitly indicates dual-eligible Special Needs Plan membership. + +[OUTPUT FORMAT] +Return your final answer as a valid JSON list of matched AARETE_DERIVED_PROGRAM values. +Example: ["CHIP", "MMC"] or [] +{JSON_LIST_FORMAT_INSTRUCTIONS}""" + + +def AARETE_DERIVED_PROGRAM( + program_values: list[str], + valid_values: list[str], +) -> Tuple[str, str, Callable[[str], list]]: + """Build prompt payload for mapping PROGRAM values to AARETE_DERIVED_PROGRAM. + + Returns: + Tuple of (context_text, prompt_text, parser_function). + context_text is cached at the instruction level; prompt_text carries the + per-call dynamic content. + """ + entities_text = ", ".join(f'"{v}"' for v in program_values) + context_text = f"""[PROGRAMS TO MAP] +{entities_text}""" + + prompt = f"""[TASK] +Map each PROGRAM value listed in [PROGRAMS TO MAP] to the best matching AARETE_DERIVED_PROGRAM value. + +Valid AARETE_DERIVED_PROGRAM values: {", ".join(valid_values)} + +Briefly explain your reasoning for each mapping, then return your final answer as a JSON list.""" + + return (context_text, prompt, _json_list_parser) + + +def AARETE_DERIVED_PRODUCT_INSTRUCTION() -> str: + """Static instruction for AARETE_DERIVED_PRODUCT prompt caching.""" + return f"""[OBJECTIVE] +Map extracted PRODUCT names to standardized AARETE_DERIVED_PRODUCT values. + +[INSTRUCTIONS] +- For each PRODUCT value provided, find the best semantically matching value from the valid output values list. +- Only return values from the valid output values list. Do NOT invent or modify values. +- If a PRODUCT value does not clearly match any valid output value, exclude it. +- Return a flat JSON list of the matched derived values (no duplicates, preserve relevance order). +- If no PRODUCT values match, return an empty JSON list []. + +[OUTPUT FORMAT] +Return your final answer as a valid JSON list of matched AARETE_DERIVED_PRODUCT values. +Example:["Parkland Community Health Plan"] or [] +{JSON_LIST_FORMAT_INSTRUCTIONS}""" + + +def AARETE_DERIVED_PRODUCT( + product_values: list[str], + valid_values: list[str], +) -> Tuple[str, str, Callable[[str], list]]: + """Build prompt payload for mapping PRODUCT values to AARETE_DERIVED_PRODUCT. + + Returns: + Tuple of (context_text, prompt_text, parser_function). + """ + entities_text = ", ".join(f'"{v}"' for v in product_values) + context_text = f"""[PRODUCTS TO MAP] +{entities_text}""" + + prompt = f"""[TASK] +Map each PRODUCT value listed in [PRODUCTS TO MAP] to the best matching AARETE_DERIVED_PRODUCT value. + +Valid AARETE_DERIVED_PRODUCT values: {", ".join(valid_values)} + +Briefly explain your reasoning for each mapping, then return your final answer as a JSON list.""" + + return (context_text, prompt, _json_list_parser) + + +def MAP_LOB_TO_VALID_INSTRUCTION() -> str: + """Static instruction for MAP_LOB_TO_VALID prompt caching.""" + return f"""[OBJECTIVE] +Map raw classified LOB (Line of Business) values to standardized valid LOB values. + +[INSTRUCTIONS] +- For each raw LOB value provided, find the best semantically matching value from the valid LOB values list. +- Only return values from the valid LOB values list. Do NOT invent or modify values. +- If a raw LOB value does not clearly match any valid LOB value, exclude it. +- Return a flat JSON list of the matched valid LOB values (no duplicates, preserve relevance order). +- If no raw LOB values match, return an empty JSON list []. +- A match requires genuine semantic equivalence — e.g., "Medi-Cal" maps to "Medicaid", "Medicare Part A/B" maps to "Medicare". +- Do NOT force-fit: if a raw value is only loosely related to a valid LOB, do not map it. + +[OUTPUT FORMAT] +Return your final answer as a valid JSON list of matched valid LOB values. +Example: ["Medicaid", "Medicare"] or [] +{JSON_LIST_FORMAT_INSTRUCTIONS}""" + + +def MAP_LOB_TO_VALID( + lob_values: list[str], + valid_lobs: list[str], +) -> Tuple[str, str, Callable[[str], list]]: + """Build prompt payload for mapping raw classified LOB values to valid LOB values.""" + entities_text = ", ".join('"' + v + '"' for v in lob_values) + context_text = f"""[LOB VALUES TO MAP] +{entities_text}""" + + prompt = f"""[TASK] +Map each raw LOB value listed in [LOB VALUES TO MAP] to the best matching valid LOB value. + +Valid LOB values: {", ".join(valid_lobs)} + +Briefly explain your reasoning for each mapping, then return your final answer as a JSON list.""" + + return (context_text, prompt, _json_list_parser) + + +def MAP_NETWORK_TO_VALID_INSTRUCTION() -> str: + """Static instruction for MAP_NETWORK_TO_VALID prompt caching.""" + return f"""[OBJECTIVE] +Map raw classified NETWORK values to standardized valid NETWORK values. + +[INSTRUCTIONS] +- For each raw NETWORK value provided, find the best semantically matching value from the valid NETWORK values list. +- Only return values from the valid NETWORK values list. Do NOT invent or modify values. +- If a raw NETWORK value does not clearly match any valid NETWORK value, exclude it. +- Return a flat JSON list of the matched valid NETWORK values (no duplicates, preserve relevance order). +- If no raw NETWORK values match, return an empty JSON list []. +- A match requires genuine semantic equivalence — e.g., "Health Maintenance Organization" maps to "HMO", "Preferred Provider Organization" maps to "PPO". +- Do NOT force-fit: if a raw value is only loosely related to a valid NETWORK value, do not map it. + +[OUTPUT FORMAT] +Return your final answer as a valid JSON list of matched valid NETWORK values. +Example: ["HMO", "PPO"] or [] +{JSON_LIST_FORMAT_INSTRUCTIONS}""" + + +def MAP_NETWORK_TO_VALID( + network_values: list[str], + valid_networks: list[str], +) -> Tuple[str, str, Callable[[str], list]]: + """Build prompt payload for mapping raw classified NETWORK values to valid NETWORK values.""" + entities_text = ", ".join('"' + v + '"' for v in network_values) + context_text = f"""[NETWORK VALUES TO MAP] +{entities_text}""" + + prompt = f"""[TASK] +Map each raw NETWORK value listed in [NETWORK VALUES TO MAP] to the best matching valid NETWORK value. + +Valid NETWORK values: {", ".join(valid_networks)} + +Briefly explain your reasoning for each mapping, then return your final answer as a JSON list.""" + + return (context_text, prompt, _json_list_parser) + + def REIMB_DATES_ASSIGNMENT_INSTRUCTION() -> str: """Static instruction for REIMB_DATES_ASSIGNMENT prompt caching. Contains all date assignment rules and examples. @@ -579,6 +1114,63 @@ Here is the definition and valid values. Use this information to help you find t return (context_text, question_text, parser) +def DYNAMIC_PRIMARY_ENTITIES_ASSIGNMENT_INSTRUCTION() -> str: + """Instruction for assigning the relevant set (full or subset) of dynamic primary entities per reimbursement row.""" + return f"""[OBJECTIVE] +Assign the set of dynamic primary entities (either the full set or a relevant subset) that applies to the given Service Term and Reimbursement Term. + +[INSTRUCTIONS] +- Use the candidate entity list provided. +- Assign only those entities (LOB, PROGRAM, PRODUCT, NETWORK) that are directly supported by the local context around the reimbursement term and service term. +- If all candidate entities are relevant, assign the full set; if only a subset is relevant, assign only that subset. +- If none apply, return ["N/A"]. + +[CATEGORY DEFINITIONS] +- LOB: Broad line-of-business categories (e.g., Medicaid, Medicare, Commercial, Marketplace). +- NETWORK: Network model types (e.g., HMO, PPO, EPO, POS). +- PROGRAM: Government/state/public program names and public coverage program variants (e.g., CHIP, ACA, Medicare Advantage). +- PRODUCT: Payer-branded or plan-branded commercial product names and plan labels (e.g., Blue Cross PPO, Aetna Medicare). + +[CRITICAL CONSIDERATIONS] +- Pay close attention to exhibit headers, section headers, and local context that may indicate which entities apply. +- The assignment should reflect only those entities that are explicitly or obviously relevant to the specific reimbursement term and service term. +- Do not include entities that are not supported by the immediate context, even if they appear elsewhere in the document. +- If the context is ambiguous or does not support any entity, return ["N/A"]. + +[OUTPUT FORMAT] +Briefly explain your answer and why the selected values are assigned for this service term and reimbursement term. +Return a valid JSON dictionary with key "DYNAMIC_PRIMARY_ENTITIES" and value as a list. +{JSON_DICT_WITH_LISTS_FORMAT_INSTRUCTIONS}""" + + +def DYNAMIC_PRIMARY_ENTITIES_ASSIGNMENT( + service_term: str, + reimb_term: str, + candidate_entities_text: str, + exhibit_text_simplified: str, + page_num: str, +) -> Tuple[str, str, Callable[[str], dict]]: + """Prompt payload to assign row-level subset from dynamic primary entity candidates.""" + context_text = f"""[CONTEXT] +Here is the section of the contract to analyze: +[START CONTEXT] +{exhibit_text_simplified} +[END CONTEXT]""" + + question_text = f"""[REIMBURSEMENT TERM] +Service Term: "{service_term}" +Reimbursement Term: "{reimb_term}" +Page: {page_num} + +[CANDIDATE DYNAMIC PRIMARY ENTITIES] +{candidate_entities_text} + +Return only the subset of candidate entities that applies to this reimbursement term.""" + + parser = _create_json_dict_parser(field_names=["DYNAMIC_PRIMARY_ENTITIES"]) + return (context_text, question_text, parser) + + def REIMBURSEMENT_PRIMARY(context) -> Tuple[str, Callable[[str], list]]: """ Returns prompt for reimbursement primary extraction. @@ -660,109 +1252,98 @@ Example output format: def LESSER_OF_DISTRIBUTION_INSTRUCTION() -> str: - return f"""TASK: Identify "lesser of" or "not to exceed" statements that apply to a service. + return f"""TASK +Identify "lesser of" or "not to exceed" constraints that apply to the given SERVICE and base METHODOLOGY. -DEFINITIONS: -"Lesser of" / "not to exceed" = overarching payment rules specifying payment at the lesser of multiple values or capped amounts. -Examples: "lesser of rates below and billed charges", "shall not exceed 110% of Medicare" +DEFINITION +"Lesser of" / "not to exceed" means an overarching payment ceiling (or comparison rule) that limits reimbursement. These constraints wrap a base methodology; they do not replace it. -DECISION LOGIC: +DECISION FRAMEWORK -0. UNDERSTAND THE TASK +0) GOAL +- Determine whether any applicable constraint should be layered onto the provided METHODOLOGY for this SERVICE. - We are looking for overarching "lesser of" or "not to exceed" constraints that apply to SERVICE. - METHODOLOGY is the base rate - we may wrap it with constraints we find. +1) COLLECT CANDIDATE CONSTRAINTS +- From CROSS-EXHIBIT templates (if provided): include them; they are pre-verified. +- From intra-exhibit text: include generic references such as "this exhibit", "rates below", "contracted rates", "set forth below". +- Exclude any statement that references a specific external exhibit/attachment/schedule by name (e.g., Attachment A, Exhibit 2, Schedule B), unless it came from CROSS-EXHIBIT templates input. -1. COLLECT APPLICABLE STATEMENTS +CRITICAL EXTERNAL-REFERENCE RULE +- If an intra-exhibit statement says "lesser of ... Attachment A ..." or similar named exhibit reference, ignore that statement. +- Example IGNORE: "lesser of payment rates established in Attachment A or 100% of Medicare" +- Example USE: "lesser of contracted rates or billed charges" - From cross-exhibit templates (if provided): - → Include these-they're already verified to apply. +2) FILTER BY SERVICE SCOPE +- Keep only constraints whose scope applies to the current SERVICE. +- Broad scope ("all services", "all covered services", "all services in this exhibit") applies generally. +- Narrow scope must align with SERVICE type (e.g., inpatient-only does not apply to outpatient). +- Statements using "listed below", "rates below", or "set forth below" apply to rates that appear after the statement in the exhibit context. - From intra-exhibit text: - → INCLUDE statements referencing "this exhibit", "rates below", "contracted rates" (generic) - → EXCLUDE any statement that references a specific exhibit name (Attachment A, Schedule B, Exhibit 2, etc.) +EXCEPTION CLAUSE HANDLING +If a statement contains exceptions ("except ...", "excluding ...") and SERVICE falls inside the exception: +1. Treat the full payment rule in that same grammatical unit as not applicable. +2. Keep only independently readable overarching constraints that stand on their own. +3. Re-scan for additional standalone constraints elsewhere; do not stop after excluding one rule. - CRITICAL: If a lesser-of statement mentions "Attachment A", "Schedule B", or any exhibit name → IGNORE IT. - Example to IGNORE: "lesser of payment rates established in Attachment A or 100% of Medicare" - Example to USE: "lesser of contracted rates or billed charges" +INDEPENDENCE TEST +- Keep: standalone sentence like "In no event shall reimbursement exceed billed charges." +- Drop: combined sentence where cap is part of excluded rule and cannot stand alone. -2. FILTER BY SERVICE SCOPE +CONDITIONAL ALTERNATIVE METHODOLOGY RULE +If "lesser of" / "not to exceed" applies only under a specific trigger ("if not listed", "where there is no rate", "when no fee schedule exists") and creates an alternative reimbursement method: +- Treat it as separate methodology logic, not a ceiling on the primary METHODOLOGY. +- Do not merge it into the primary lesser-of output. - Keep only statements whose scope matches SERVICE: - - "All services in this exhibit" / "All services" / "All covered services" → applies to any service - - "Inpatient services only" → does NOT apply to outpatient services - - Lesser-of statements containing "listed below", "rates below", "set forth below" - → applies to ANY service rate that appears after that statement in the exhibit text, - regardless of section headers +3) COMBINE AND FORMAT OUTPUT - EXCEPTION CLAUSE HANDLING: - - If a lesser-of statement has an exception clause (e.g., "except for services in Table X", - "excluding procedures listed below"), and SERVICE appears in that exception list: +CONSTRUCTION RULES +- Replace generic placeholders like "rates" or "contracted rates" with the full METHODOLOGY value when needed. +- Preserve contract wording as much as possible; do not paraphrase away legal meaning. +- Do not add service codes/descriptions to the returned constraint text. +- Deduplicate identical constraints. +- Preserve nested logic explicitly when multiple comparisons exist (e.g., lesser-of around greater-of). - a) The ENTIRE payment rule in that sentence/paragraph does NOT apply to SERVICE. - This includes all comparisons, caps, and conditions within that grammatical unit. +METHODOLOGY-PRESERVATION RULE +- Final output must retain the FULL original METHODOLOGY text (all tiers, rates, conditions). +- Constraint language is an outer wrapper; it must not summarize or truncate the base methodology. +- If the built string drops METHODOLOGY, reinsert/re-nest so the base remains intact. - b) ONLY overarching constraints still apply to SERVICE. - Test: Can this constraint be read independently without referencing the excepted statement? - If YES → it still applies - If NO → it's part of the excepted rule and does NOT apply +QUALITY CHECK BEFORE RETURN +1. Is every included constraint applicable to this SERVICE? +2. Did you exclude disallowed external exhibit references from intra-exhibit text? +3. Does final text still include full METHODOLOGY? +4. Are nested comparisons represented faithfully? +5. Are duplicate constraints removed? - c) ALWAYS re-scan the exhibit text for standalone constraints after excluding the payment rule. - Do not stop at step (a) - even if the main rule is excluded, independent constraints MUST be applied. +OUTPUT BEHAVIOR +- Return one JSON list item containing the final constrained methodology text when applicable. +- Return ["N/A"] when no applicable constraint remains. - Examples: - - SEPARATE sentence: "In no event shall reimbursement exceed the amount billed by Provider." - → Still applies (passes independence test - standalone constraint) - - SAME sentence: "Compensation shall be the lesser of 80% of the scheduled fee or usual - and customary charges, capped at the Provider's submitted amount" - → Does NOT apply (entire grammatical unit is the excepted rule) +RETURN ["N/A"] WHEN +- No applicable lesser-of/not-to-exceed statements are found. +- Remaining intra-exhibit statements only reference named external exhibits/attachments. +- Candidate constraints exist but are scoped to different service types. +- Only conditional alternative-methodology statements exist (no overarching ceiling on base methodology). - If nothing applies → return N/A +REFERENCE EXAMPLES +- ["lesser of $100, billed charges, or 110% Medicare"] +- ["$30.00 not to exceed Provider's billed charges"] +- ["the lesser of [the greater of 100% of the State Medicaid fee schedule or Affinity's standard fee schedule] and billed charges"] +- ["N/A"] - CONDITIONAL METHODOLOGY RULE: - -If a “lesser of” or “not to exceed” statement: - →Applies only under a specific condition (e.g., “where there is no rate,” “if not listed,” “when no fee schedule exists”), and - →Defines an alternative reimbursement method rather than limiting the base METHODOLOGY, - -Treat it as a separate reimbursement methodology. - -Do NOT apply it as a constraint to the primary METHODOLOGY. - -Do NOT merge it into the output. - -Only include “lesser of” or “not to exceed” language when it operates as an overarching ceiling that applies to the base METHODOLOGY itself. +NESTED COMPARISON FAITHFULNESS +- When the contract uses "the lesser of [the greater of A or B] and C", the returned text must preserve both the inner comparison and the outer comparison. Do not flatten nested comparisons into a comma-separated list of three items. +- When a comparison contains four or more terms, preserve the original conjunction structure (commas vs explicit "or") rather than imposing a uniform style. The downstream parser is sensitive to this. +- If a "not to exceed" clause is followed by an additional cap (e.g. "...not to exceed billed charges and not to exceed $1,000"), include both caps in the returned text; do not silently keep only the first one. -3. COMBINE & FORMAT +NEGATIVE CONSTRAINT TESTS +- A statement that simply says payment "shall be" some amount is NOT a lesser-of/not-to-exceed constraint. Return ["N/A"] if no comparison or ceiling is present in the candidate. +- A statement that says payment "shall not exceed" billed charges qualifies as a ceiling and should be retained when SERVICE scope matches. +- A statement that says payment is "the lesser of" but lists only one item ("lesser of contracted rate") is degenerate and does not produce a usable constraint; treat it as ["N/A"]. - Substitution: - - Replace generic "rates" / "contracted rates" with METHODOLOGY value - - Preserve original phrasing exactly - do not paraphrase, abbreviate, or alter wording - - Do not include service code or description in the output - - Deduplicate identical constraints - - Join multiple terms preserving meaning through nesting or other structures: e.g., "[the lesser of [the greater of A or B] and C]", "lesser of X, Y, or Z". Ensure brackets and nesting are used appropriately to maintain the original meaning. - - Validation: - - The output string MUST contain METHODOLOGY value - - If METHODOLOGY is a simple rate (e.g., "$30.00") and is missing from the final string, prepend it - - If METHODOLOGY is a complex formula (e.g., "80% of State Medicaid fee schedule", "the lesser of (i) X or (ii) Y"), verify it appears as the base of the output - - The output MUST contain the FULL original METHODOLOGY - every rate, tier, and condition. - The lesser-of constraint wraps METHODOLOGY as an outer ceiling; it never replaces or summarizes it. - Before returning, verify METHODOLOGY appears intact. If any part is missing, re-nest: - "the lesser of [full METHODOLOGY] or [constraint ceiling]" - - Examples: - - ["lesser of $100, billed charges, or 110% Medicare"] - - ["$30.00 not to exceed Provider's billed charges"] - - ["the lesser of [the greater of 100% of the State Medicaid fee schedule or Affinity's standard fee schedule] and billed charges"] - - ["N/A"] - -Example - Intra-exhibit statement references another exhibit → N/A: -- METHODOLOGY: "per visit basis at payment rates established in Attachment A" -- CROSS-EXHIBIT TEMPLATES: None -- EXHIBIT TEXT contains: "lesser of payment rates established in Attachment A or 100% of Medicare" -- Analysis: The only lesser-of statement found references "Attachment A" → IGNORE IT -- Result: ["N/A"] (no applicable statements remain) - -RETURN ["N/A"] WHEN: -- No applicable lesser-of statements found -- All statements found reference other exhibits by name (and no cross-exhibit templates provided) -- Statements found but scoped to different service types +INVARIANTS BEFORE RETURN +- The returned JSON list contains exactly one string when a constraint applies, or exactly ["N/A"] when none applies. Do not return multiple constraint strings as separate list items. +- The single string concatenates ALL applicable comparisons into one expression preserving the original logical structure. {JSON_LIST_FORMAT_INSTRUCTIONS}""" @@ -884,6 +1465,20 @@ Reimbursement: "lesser of Allowable Charges or Company's fee schedule in effect - GLOBAL must be a constraint that governs OTHER rates across the contract, not a rate itself. - When in doubt between STANDALONE and GLOBAL, ask: "Is this a rate, or a rule about rates?" If it's a rate, choose STANDALONE. +**EDGE CASES AND PRECEDENCE:** + +- **Mixed signals across fields**: If the Service field references one exhibit and the Reimbursement field references a different exhibit, classify by the more specific reference. If both are equally specific, use the Reimbursement field reference (it is closer to the rate-defining text). +- **"Notwithstanding" prefix**: A statement that begins with "Notwithstanding any rates set forth..." is almost always GLOBAL. The notwithstanding clause is the strongest signal that this rule overrides other rates. +- **"In no event" prefix**: "In no event shall payment exceed..." is GLOBAL when the scope is contract-wide ("any service", "all claims", "any provider"). It is STANDALONE only when paired with a single specific service. +- **Implicit external references**: "Per the attached fee schedule" with no exhibit name still counts as an EXHIBIT_SPECIFIC (OTHER) reference; set exhibit_reference to "the attached fee schedule" verbatim. +- **Self-referential current-exhibit phrasing**: "The rates in this Exhibit" / "the schedule below" / "as set forth in this Schedule" are EXHIBIT_SPECIFIC (CURRENT). Their target_exhibit is "CURRENT" and exhibit_reference is null. + +**FIELD-LEVEL CONSISTENCY:** +- target_exhibit must be "ALL" only when scope is "GLOBAL". +- target_exhibit must be "CURRENT" only when scope is "EXHIBIT_SPECIFIC" and the reference points to this exhibit. +- target_exhibit must be "OTHER" only when scope is "EXHIBIT_SPECIFIC" and the reference points to a different exhibit; exhibit_reference must be non-null in this case. +- target_exhibit must be null when scope is "STANDALONE"; exhibit_reference must also be null. + **OUTPUT FORMAT:** Briefly explain your reasoning, then return a properly-formatted JSON dictionary with the following structure: {{ @@ -1046,6 +1641,7 @@ def DYNAMIC_CODE_ASSIGNMENT_INSTRUCTION() -> str: return f"""[OBJECTIVE] Analyze a Service Term from a Payer-Provider contract and classify it as one or more of the listed field values. +The Service Term describes a medical service, procedure, or provider category. Your job is to determine which classification codes apply based solely on what is explicitly written in the Service Term. [INSTRUCTION] - FIRST: Determine if some OTHER code is Explicitly written in the Service Term, then the answer will always be N/A. @@ -1057,6 +1653,29 @@ Analyze a Service Term from a Payer-Provider contract and classify it as one or - If a field has valid values, choose ONLY from those valid values. - Make appropriate assumptions only if the answer is obvious. +[CLASSIFICATION GUIDANCE] +- Explicit label rule: A field value is only valid if the Service Term contains language that directly maps to that value. Do not derive a value from an adjacent or implied concept. +- Specificity rule: When a Service Term uses a specific provider type (e.g., "Physician", "Hospital"), map only to the most specific applicable value. Do not broaden to a parent category unless the parent category is also explicitly named. +- Multi-value rule: If the Service Term lists multiple distinct items (e.g., "Inpatient and Outpatient services"), assign separate values for each applicable classification. Return them as a list (array). +- Conflict rule: If two fields would normally take the same source label but map to different valid value sets, assign each field independently using its own valid values list. Do not let one field's assignment affect another. +- Ambiguity rule: If the text is ambiguous and could plausibly map to two or more valid values with equal confidence, include all plausible values as a list. Do not arbitrarily choose one. +- Abbreviation rule: Recognize common healthcare abbreviations (e.g., "IP" = Inpatient, "OP" = Outpatient, "Prof" = Professional, "Fac" = Facility) only when the abbreviation is unambiguous in context. +- N/A default: When no field value can be determined with confidence from the Service Term text, return "N/A" for that field. Never guess. + +[CODE-PRESENCE PRECEDENCE] +- The presence of any explicit billing identifier (CPT, HCPCS, Revenue, ICD-10, MS-DRG, APR-DRG, NDC, POS code, Bill Type code) in the Service Term causes EVERY classification field other than the matching code field to be "N/A". +- This applies even when the surrounding text appears to describe a service category. Example: "POS 12: Home" sets POS-related fields if any, but Bill Type, Claim Type, and Provider Type are "N/A" because an explicit code is present. +- If the Service Term contains BOTH an explicit code AND a place-of-service phrase WITHOUT a code (e.g., "POS 21 - Inpatient Hospital"), the code still wins and non-code fields remain "N/A". + +[CROSS-FIELD INDEPENDENCE] +- Each field in the dictionary must be evaluated against its own valid-values list independently. Do not let one field's match propagate to another field even when both fields share source vocabulary. +- When two valid-value sets overlap (for example, "Outpatient" appearing as both a Claim Type and a Bill Type modifier), assign each field only based on the rules of that field's value list. + +[OUTPUT DETERMINISM] +- Sort multi-value lists alphabetically so equivalent inputs produce identical outputs across runs. +- Strip leading/trailing whitespace inside string values; collapse internal whitespace to single spaces. +- Do not include explanatory prose inside JSON values; explanations belong before the JSON only. + [FIELDS] Populate a JSON dictionary with the following fields: {fields_text} @@ -1151,6 +1770,21 @@ Return 'N/A' when information is missing. - For multiple values: separate with commas, not arrays - Return N/A when no relevant information is found +[GROUPER FAMILY DISTINCTIONS] +- MS-DRG, APR-DRG, AP-DRG, and CMG are distinct grouper families. Do not collapse them into a single label even when multiple appear together; each should be captured under its own grouper-system value. +- Version qualifiers (e.g. "MS-DRG v40", "APR-DRG version 38") belong with the grouper-system value when the field expects a system identifier; they belong with the version field when the schema separates the two. +- Severity-of-illness or risk-of-mortality modifiers (SOI 1-4, ROM 1-4) are part of the APR-DRG family; do not invent them when only an MS-DRG number appears. +- A bare number that the contract calls a "Grouper Code" without naming the family is still ambiguous — return the number, but do NOT guess MS-DRG vs APR-DRG. + +[CONVERSION FACTOR vs RATE] +- A grouper conversion factor is the per-weight dollar amount applied to a relative weight (e.g. "$5,000 per relative weight"). It is REIMB_CONVERSION_FACTOR. +- A flat amount keyed to a specific grouper code (e.g. "MS-DRG 470 = $12,000") is a REIMB_FEE_RATE for that code, NOT a conversion factor. +- A percentage applied to a baseline schedule under a grouper methodology is a REIMB_PCT_RATE; the grouper system identifies which schedule, but the numeric output remains the percent value. + +[OUTPUT NORMALIZATION] +- Strip currency symbols only when the schema requires bare numerics; otherwise preserve them as written. +- Do not append "per case", "per claim", or "per stay" qualifiers into numeric fields; those qualifiers describe scope, not the value. + [FIELDS] Populate a JSON dictionary with the following fields: {fields_text} @@ -1398,20 +2032,20 @@ Extract explicit procedure, revenue, diagnosis, and other healthcare codes from [INSTRUCTIONS] - For each field, return a list of all codes EXPLICITLY written for that field. The code itself must be written, not language that describes the code. - FIELD ASSIGNMENT (map by code system, not by the word "procedure" or - "diagnosis" in the text): - - PROCEDURE_CD (CPT4): Only CPT (5 digits or 5 alphanumeric, e.g. 99213, 0124A) - and HCPCS (1 letter + 4 alphanumeric, e.g. J1098, S2900). Put these ONLY in - PROCEDURE_CD. - - DIAG_CD: Includes ICD-10-CM (letter then digits, e.g. Z00.00) - and ICD-10-PCS (7 alphanumeric characters). When the text mentions both - diagnosis/procedure codes and uses "ICD-10" or lists codes that look like - ICD-10 (e.g. 7-character alphanumeric), put those in DIAG_CD. Do NOT put - ICD-10 codes in PROCEDURE_CD. - - When the same sentence or list contains both ICD-10-style codes and - CPT/HCPCS-style codes (5-char), assign each by format: ICD-10 to DIAG_CD, - CPT/HCPCS to PROCEDURE_CD. Do not put all codes in one field. - - REVENUE_CD: Revenue codes only (3-4 digits, may end in X). When in doubt: - PROCEDURE_CD = CPT/HCPCS only; DIAG_CD = ICD-10 only (including ICD-10-PCS). + "diagnosis" in the text): + - PROCEDURE_CD (CPT4): Only CPT (5 digits or 5 alphanumeric, e.g. 99213, 0124A) + and HCPCS (1 letter + 4 alphanumeric, e.g. J1098, S2900). Put these ONLY in + PROCEDURE_CD. + - DIAG_CD: Includes ICD-10-CM (letter then digits, e.g. Z00.00) + and ICD-10-PCS (7 alphanumeric characters). When the text mentions both + diagnosis/procedure codes and uses "ICD-10" or lists codes that look like + ICD-10 (e.g. 7-character alphanumeric), put those in DIAG_CD. Do NOT put + ICD-10 codes in PROCEDURE_CD. + - When the same sentence or list contains both ICD-10-style codes and + CPT/HCPCS-style codes (5-char), assign each by format: ICD-10 to DIAG_CD, + CPT/HCPCS to PROCEDURE_CD. Do not put all codes in one field. + - REVENUE_CD: Revenue codes only (3-4 digits, may end in X). When in doubt: + PROCEDURE_CD = CPT/HCPCS only; DIAG_CD = ICD-10 only (including ICD-10-PCS). - Note that codes may be present, but not explicitly labeled as codes. For example, you may simply see "J1098", which is a PROCEDURE_CD. You may see "155" which is a REVENUE_CD, etc. - If the text explicitly gives a range of codes (e.g. "Surgery codes 10021 to 69990", "codes X to Y", "X–Y"), return that range in the format 'LowestCode-HighestCode' (e.g. 10021-69990). A single range string is acceptable and preferred when the source text describes a single range; do not list every code in the range. - If the text gives a range of codes without listing each code individually, return the range in the format 'LowestCode-HighestCode'. @@ -1422,6 +2056,30 @@ Extract explicit procedure, revenue, diagnosis, and other healthcare codes from - Return only valid codes that match the stated format (e.g. CPT exactly 5 digits, HCPCS 1 letter + 4 digits); do not invent or guess codes. - If a code appears only in an exclusion, exception, or negation context (e.g., "except for", "excluding", "not including", "other than", "does not include"), do NOT extract it. Only extract codes that are explicitly being applied, covered, or referenced as the subject of the service description. +[CODE-SYSTEM SHAPE REFERENCE] +- CPT4: exactly 5 characters, all digits OR 4 digits followed by one capital letter (Category II/III). Examples: 99213, 0124A, 99072. +- HCPCS Level II: 1 capital letter (A-V) followed by 4 digits. Examples: J1098, S2900, G0008. Never put HCPCS in DIAG_CD. +- ICD-10-CM: 1 capital letter (A-Z), 2 digits, optional dot and 1-4 alphanumerics. Examples: Z00.00, M54.5, J45.20. These go in DIAG_CD only. +- ICD-10-PCS: exactly 7 alphanumeric characters, no dots, mixing letters and digits. Examples: 0DTJ4ZZ, B30FZZZ. These go in DIAG_CD only. +- Revenue Codes: 3 or 4 digits, sometimes ending in a literal X to indicate a family. Examples: 170, 173, 0450, 36X. +- MS-DRG / APR-DRG: 3-digit numeric code typically 1-999. Examples: 470, 783, 795. These belong in GROUPER_CD, not REVENUE_CD. + +[DISAMBIGUATION RULES] +- A naked 3-digit number with no labeling tokens is ambiguous between Revenue and Grouper. Use surrounding nouns ("revenue", "rev code", "DRG", "grouper", "MS-DRG") to assign; if no nouns appear within the same sentence, leave the field empty rather than guess. +- A 4-digit number starting with 0 is almost always a Revenue Code (e.g. 0450 ER); a 3-digit number in the 700-900 range with "DRG" nearby is a Grouper Code. +- "Status Indicator F4" / "SI F4" are Outpatient Prospective Payment System indicators, NOT codes; do not place them in any code field. + +[RANGE FORMATTING DISCIPLINE] +- Use the en-dash style "Low-High" with no spaces (e.g. "10021-69990"). Convert "to", "through", and "—" to a hyphen. +- Do not enumerate every code inside a range; one range string per range is correct. +- When the contract lists a comma-separated set ("J1098, J1100, J1110"), preserve the discrete codes as separate list items, not a range. + +[NEGATIVE FILTERS] +- Phrases that mention a code system without quoting actual codes ("CPT codes apply", "all HCPCS-billed services", "according to ICD-10") do NOT contribute extracted codes; return [] for that field unless a literal code appears. +- Approximate references like "around code 99213" or "similar to J1098" count as the literal code present in the phrase; extract the literal token, not the approximation language. +- Citation references inside parentheses or brackets ("see CPT 99213", "[ICD-10 Z00.00]") are still explicit code mentions and should be extracted. +- Do not include modifier suffixes (-26, -TC, -25, -GA) as standalone codes; they are not codes by themselves and belong to the procedure code they qualify. + [FIELDS] Populate a JSON dictionary with the following fields: {fields_text} @@ -1646,6 +2304,49 @@ Match a medical Service Term to its corresponding Procedure Code description usi - There can be multiple correct answers. When this is the case, return them all as separate items in the JSON list. - There may be no correct answers, especially when the Service is much more specific than any of the descriptions. When this is the case, simply return an empty list []. +[ADDITIONAL CLARIFICATIONS] +- A synonym match means the Description and isolated Service are meaningfully equivalent, not just related. +- Do not accept broad parent categories when the service is much narrower. +- Do not accept narrow subtypes when the service is broader. +- If the only overlap is context words (payer/program/place/provider/reimbursement language), reject the match. +- If two candidates seem possible but neither is clearly synonymous, prefer returning [] over forcing a closest option. + +[DETERMINISTIC DECISION RULE] +- Use the same strict threshold for every candidate in a single prompt call. +- Do not switch from strict to permissive matching midway through evaluation. +- If evidence quality differs across candidates, keep only those that pass the same synonym standard. +- When uncertainty remains after review, abstain with [] rather than guessing. + +[CALIBRATION EXAMPLES] +Example 1: +- Input Service: "Medical Services provided by an Anesthesiologist" +- Isolated Service: "Medical Services" +- Candidate: "Medical Services" -> acceptable. + +Example 2: +- Input Service: "Anesthesia Services provided in an Outpatient Laboratory" +- Isolated Service: "Anesthesia Services" +- Candidate: "Anesthesia" -> acceptable when clearly equivalent. + +Example 3: +- Input Service: "Nutritional Evaluations" +- Candidate: "Evaluations" -> reject (too broad). + +Example 4: +- Input Service: "Cardiac MRI and Echocardiography" +- Action: evaluate both services independently. + +Example 5: +- Input Service is very specific and available candidates are only generic umbrellas. +- Action: return []. + +[QUALITY CHECKLIST] +Before finalizing: +1. Every selected description is supported by the isolated service meaning. +2. No selected description is based only on provider/place/payer/program/reimbursement framing. +3. No description is selected merely because it is the closest available candidate. +4. If confidence is low, return []. + [OUTPUT FORMAT] Explain your answer, ensuring each point in the instructions is addressed. Then return your final answer in a properly-formatted JSON list. {JSON_LIST_FORMAT_INSTRUCTIONS}""" @@ -1769,6 +2470,67 @@ Analyze a given medical Service Term to determine which Bill Type Code Descripti - "home dialysis" --> ["Home"] - "dialysis" --> ["Dialysis Center"] +[PLACE-OF-SERVICE PRECEDENCE RULES] +- A place of service mentioned with a preposition that anchors location ("at", "in", "rendered in", "performed at", "delivered in") binds the bill type to that location even if other terms describe what happens there. +- A place of service used as a modifier of a procedure ("Inpatient Surgery", "Outpatient Lab") still binds the bill type to that location; the procedure itself is secondary. +- When both a setting and a sub-setting are present (e.g. "Hospital Outpatient"), select the most specific valid description that matches; do not return both the broad and the specific value when one already implies the other. +- Provider type alone (physician, surgeon, anesthesiologist, technician, therapist) is NOT a place of service and does not yield a bill type by itself. +- Payment-source language ("Medicare reimbursement", "Medicaid fee schedule") is NOT a place of service. + +[OUTPUT INTEGRITY] +- Use the exact case and punctuation from [VALID DESCRIPTIONS]; do not paraphrase or shorten the labels. +- Sort the returned list alphabetically when more than one description matches, so equivalent inputs produce the same output across runs. +- Never invent a description that is not in [VALID DESCRIPTIONS]. + +[DETAILED GUIDANCE FOR BILL TYPE DETERMINATION] +- Bill type codes are primarily determined by the setting or location where healthcare services are delivered. +- Common place-of-service indicators include: hospital, clinic, home, nursing facility, dialysis center, ambulatory surgery center, etc. +- When the service term explicitly mentions a location (e.g., "inpatient hospital services", "home health care"), use that location to match against valid descriptions. +- If no location is specified, look for service type indicators that strongly imply a setting (e.g., "emergency room visit" implies hospital setting). +- Avoid inferring locations from service names alone unless the connection is direct and unambiguous. + +[VALIDATION RULES] +- Ensure that the matched description directly corresponds to a place of service mentioned in the term. +- Cross-reference with the service type to confirm compatibility (e.g., surgery services in a hospital setting). +- If multiple locations are possible, only include those that are explicitly supported by the term. +- Reject matches that require significant interpretation or assumption beyond what's stated. + +[EXTENDED EXAMPLES] +- "Inpatient psychiatric care" --> ["Inpatient Hospital"] (location explicitly stated) +- "Outpatient chemotherapy" --> ["Outpatient Hospital"] (location and service type clear) +- "Skilled nursing facility rehabilitation" --> ["Skilled Nursing Facility"] (facility type specified) +- "Community-based mental health services" --> ["Community Mental Health Center"] (if available in valid descriptions) +- "Telemedicine consultation" --> [] (no specific location, ambiguous) +- "Laboratory testing" --> [] (could be multiple settings, not specified) +- "Emergency department visit" --> ["Hospital"] (implies hospital setting) +- "Home infusion therapy" --> ["Home"] (location explicitly home) + +[COMMON PITFALLS TO AVOID] +- Do not match based on service type alone without location context. +- Avoid assuming hospital settings for all medical services. +- Do not infer outpatient when inpatient is possible or vice versa without clear indication. +- Resist the temptation to match to the closest available description when no clear match exists. +- Do not use provider type (e.g., physician, nurse) to determine bill type. + +[ADDITIONAL CONSIDERATIONS] +- Consider the hierarchical nature of healthcare settings: hospital > clinic > home. +- Account for specialized facilities like dialysis centers or surgery centers. +- Recognize that some services can occur in multiple settings but only match when specified. +- Maintain consistency with standard healthcare billing classifications. + +[QUALITY ASSURANCE CHECKS] +- Does the matched bill type directly reflect a location mentioned in the service term? +- Is the match unambiguous and not requiring inference? +- Would the same match apply if the service term were slightly rephrased? +- Is the bill type appropriate for the type of service described? + +[OUTPUT FORMAT REQUIREMENTS] +- Provide a brief explanation justifying the match or lack thereof. +- Reference specific words in the service term that support the decision. +- If no match, explain why none of the valid descriptions apply. +- Ensure the JSON list format is correct, with proper brackets and quotes. +- Include all applicable bill types when multiple are clearly supported. + [OUTPUT FORMAT] Briefly explain your answer, then return your final answer in a properly-formatted JSON list. {JSON_LIST_FORMAT_INSTRUCTIONS}""" @@ -1856,10 +2618,56 @@ Briefly explain your answer, then put the final answer in a properly-formatted J def ONE_TO_ONE_SINGLE_FIELD_INSTRUCTION() -> str: - return ( - "[OBJECTIVE]\n" - "Answer a single, deterministic question about the contract text. Return result in JSON dictionary format as instructed." - ) + """Static instruction for the single-field one-to-one HSC prompt. + + The runtime template (`ONE_TO_ONE_SINGLE_FIELD_TEMPLATE`) injects the + contract excerpt and the per-field question dynamically. This block + captures every rule that does NOT vary per call so it can be cached + and reused across the ~30 fields HSC asks per document. Sized above + Sonnet's 1024-token minimum so cache_control is honored. + """ + return f"""[OBJECTIVE] +Answer a single, deterministic question about a healthcare payer-provider contract excerpt. Extract the answer EXACTLY as stated in the excerpt, return it in the requested JSON shape, and never invent values. + +[EXCERPT INTERPRETATION] +- Treat the text between `## START CONTRACT TEXT ##` and `## END CONTRACT TEXT ##` as the only authoritative source. Do not import knowledge from outside the excerpt — no inferring values from training data, no completing partial dates from common conventions, no expanding abbreviations the contract did not expand. +- The excerpt is a retrieval-ranked subset of a longer document. It may contain multiple paragraphs, tables, footnotes, signature blocks, or running headers from different parts of the contract. Read each paragraph independently before deciding which one answers the question. +- Headers, page numbers, and exhibit labels (e.g. "Exhibit B", "Schedule 2.1") are scaffolding, not answers. Use them to locate the relevant paragraph but do NOT return them as the value of an unrelated field. +- When the excerpt contains conflicting candidate values, prefer the value stated in the most specific scope (line item > section > exhibit > whole-contract preamble). If specificity is equal, prefer the most recent date or amendment. + +[ANSWER PRECISION] +- Copy the answer verbatim from the excerpt where the field is a name, identifier, code, address, or quoted phrase. Preserve original capitalization and punctuation. +- Strip non-essential surrounding tokens (e.g. trailing periods, leading "the", role qualifiers like "Dr." when the field asks only for the name) only when the field definition makes that explicit. +- For dates, normalize to a single canonical format only when the field definition requests it; otherwise echo the contract's own format. +- For numeric percentages, preserve the percent sign and decimal precision as written in the contract. Do NOT round, do NOT convert between percent and decimal. +- For TINs and NPIs, strip non-digit characters (hyphens, spaces, parentheses) before returning. TIN must be 9 digits; NPI must be 10. Reject any candidate that does not meet the digit count. +- For multi-token answers (e.g. payer name with legal suffix), include the full noun phrase as it appears, not a truncated form. + +[NULL AND UNKNOWN HANDLING] +- Return "N/A" when the field is genuinely absent from the excerpt, or when every candidate value fails the field's validation rules. +- Do NOT return placeholders like "TBD", "Unknown", "See attached", "[Pending]", or boilerplate values. Treat any of these as missing and return "N/A". +- Do NOT guess. If two values are equally plausible and the excerpt does not disambiguate, "N/A" is the correct answer. +- Do NOT hallucinate values that look reasonable but are not in the text. The downstream pipeline depends on faithful "N/A" reporting more than on guessed positives. + +[CROSS-PARTY DISAMBIGUATION] +- Healthcare contracts have at least two parties: a Payer (insurance company, health plan, MCO) and a Provider (physician, group, hospital, ancillary). Each field asks about ONE of these roles. +- Do NOT swap parties. The provider's TIN is not the payer's TIN. The contract effective date is not the amendment effective date unless the field explicitly asks for the amendment. +- Signatory blocks list both parties; pick the signature line that matches the requested role. + +[FORMATTING DISCIPLINE] +- The final answer must be a JSON dictionary with exactly one key — the field name as supplied in the question — and one string value. +- Do not include nested JSON, arrays, or additional keys. +- Strip whitespace at the start and end of the value. Collapse internal whitespace runs to a single space unless the field is a code or identifier where exact spacing matters. +- Escape internal double quotes as needed for valid JSON. + +[REASONING DISCIPLINE] +- Provide a brief explanation BEFORE the JSON dictionary citing the specific phrase or sentence that supports the answer. Quote no more than necessary. +- Keep the explanation under three sentences. Skip restating the rules. +- Do NOT include the explanation inside the JSON value. + +[OUTPUT] +Briefly explain your answer with a short verbatim quote, then return the final answer in a properly-formatted JSON dictionary `{{"": ""}}`. +{JSON_DICT_FORMAT_INSTRUCTIONS}""" def TIN_NPI_TEMPLATE_INSTRUCTION() -> str: @@ -1869,23 +2677,45 @@ Analyze the following healthcare payer-provider contract text and extract provid IMPORTANT DISTINCTION: - Providers: Physicians, physician groups, hospitals, clinics who DELIVER healthcare, CEOs and other signatories that may sign on behalf of any of the previous entities - Payers: Insurance companies, health plans who PAY for healthcare services +- When in doubt: if an entity appears as the party being paid or delivering services, it is a provider. If it appears as the party setting rates or processing claims, it is a payer. [EXTRACTION INSTRUCTIONS] - First, analyze the text thoroughly and identify all Provider Entities - A Provider Entity is considered valid as long as either a TIN, NPI, or NAME is present. If any one or two of those fields are missing, use "N/A" for those fields. - Return TIN and NPIs that are found on the page even if they aren't explicitly labeled as such. -- If the table contains multiple name columns (e.g. 'System Name' and 'Facility Name'), YOU MUST extract the specific 'Facility Name' or 'Hospital Name' column. +- If the table contains multiple name columns (e.g. 'System Name' and 'Facility Name'), YOU MUST extract the specific 'Facility Name' or 'Hospital Name' column. Do not use a parent organization name when a more specific facility name is available. - IMPORTANT VALIDATION REQUIREMENTS: * TIN must be exactly 9 digits (no more, no less) * NPI must be exactly 10 digits (no more, no less) * CAREFULLY double-check the digit count for each TIN and NPI before including them in the response + * If a number contains hyphens (e.g. 12-3456789), strip hyphens first, then validate the digit count + * Reject any TIN or NPI that does not meet the exact digit requirement — do not pad or truncate to force a match + +[TIN / NPI DISAMBIGUATION] +- TIN (Tax Identification Number): exactly 9 digits, often formatted as XX-XXXXXXX. Also called EIN (Employer Identification Number), Federal Tax ID, or Tax ID Number. +- NPI (National Provider Identifier): exactly 10 digits. Also called Provider NPI, Individual NPI, or Group NPI. +- If a number is labeled as both TIN and NPI, classify by digit count: 9 digits → TIN, 10 digits → NPI. +- If a number is unlabeled, use context clues: proximity to "Tax ID", "Federal ID", "EIN", "NPI", or "Provider ID". +- Do not confuse Medicaid Provider IDs, Medicare Provider IDs, contract reference numbers, or claim IDs with TINs or NPIs. + +[MULTIPLE ENTITIES] +- If a single TIN is associated with multiple NPIs, create one dictionary per NPI. +- If a single TIN is associated with multiple distinct names (DBA names, aliases), create one dictionary per name. +- If multiple TINs are present for the same provider group, create separate dictionaries for each TIN. +- Signatories (e.g., CEOs, medical directors) signing on behalf of a group are considered providers; extract their TIN/NPI if present. + +[EXCLUSION RULES] +- Do NOT extract the payer's TIN or NPI even if present in the contract. +- Do NOT extract government agency identifiers (e.g., CMS identifiers, state Medicaid agency IDs). +- Do NOT extract reference numbers, contract numbers, or claim submission IDs. +- Do NOT extract example or placeholder values that are clearly presented as illustrative samples. [FORMATTING INSTRUCTIONS] - Briefly explain your reasoning. After your explanation, include the heading "FINAL PROVIDER INFO:" followed by a properly formatted JSON array with your final output - In your response, include ONLY ONE SINGLE properly formatted JSON object. CRITICAL: The object MUST be of the form **dict[list[str, str]]** - Each associated TIN-NPI-Name combination must be an object (DICT) within this array - All provider objects must have these exact keys: "TIN", "NPI", "NAME" -- ALl dict values must be strings - if there are multiple different NAMEs or NPIs for an identical TIN, put them in different dictionaries. +- All dict values must be strings - if there are multiple different NAMEs or NPIs for an identical TIN, put them in different dictionaries. - Use "N/A" for any missing values - Strip ALL non-digit characters from any found TIN or NPI (e.g., remove hyphens, spaces, parentheses). - Your final JSON array must begin with an opening bracket '[' and end with a closing bracket ']' @@ -1997,8 +2827,104 @@ Determine if two provider names refer to the same healthcare organization. - Health system names that are NOT in PROVIDER NAME B, even if the listed hospitals belong to that system. - Corporate umbrella names unless they are explicitly listed in PROVIDER NAME B. +[NORMALIZATION DETAILS] +Apply these before comparing names: +1. Case-insensitive matching. +2. Ignore punctuation and extra whitespace. +3. Ignore legal suffixes that do not change identity: LLC, L.L.C., Inc, Incorporated, Corp, Corporation, LP, LLP, PC, PA, PLLC, Co. +4. Treat DBA aliases as valid identity links. +5. Treat obvious orthographic variants as equivalent: St. = Saint, & = and, Hosp = Hospital. +6. If PROVIDER NAME B contains multiple candidate names, evaluate each candidate independently. + +[DETERMINISTIC DECISION PROCESS] +Use this exact order for every comparison: +1. Normalize A and each candidate in B using the rules above. +2. Check exact/close-variation/DBA/acronym/common-word-variant rules. +3. Apply CRITICAL non-match exclusions. +4. If at least one B candidate is a confident identity match to A, return ["Y"]. +5. Otherwise return ["N"]. + +[DISAMBIGUATION POLICY] +- This is identity matching, not ownership, affiliation, or network membership matching. +- Shared city/state/location words are not identity proof. +- Generic words like "Medical", "Hospital", "Center", or "Health" alone are not identity proof. +- If evidence is ambiguous, return ["N"]. +- Do not infer hidden legal relationships unless explicitly present in the names. + +[MULTI-NAME HANDLING] +When PROVIDER NAME B contains multiple entities, split on common separators (|, ;, /, comma lists, "and" joining distinct orgs) and evaluate each candidate independently. + +[ACRONYM SAFETY] +Accept acronym matches only when the expansion is clear and unambiguous. When multiple plausible expansions exist, return ["N"]. + +[FEW-SHOT EXAMPLES] +Example 1: +- A: Lakeview Hospital +- B: Hospital Corporation of Utah d.b.a. Lakeview Hospital +- Result: ["Y"] +- Why: Explicit DBA identity link. + +Example 2: +- A: MHS +- B: Memorial Hospital System +- Result: ["Y"] +- Why: Clear acronym expansion with no ambiguity. + +Example 3: +- A: CommonSpirit Health +- B: St. Mary Medical Center +- Result: ["N"] +- Why: Parent/system vs operating facility is not direct identity. + +Example 4: +- A: Saint Marks Hospital +- B: St. Mark's Hospital LLC +- Result: ["Y"] +- Why: Orthographic variation and legal suffix differences only. + +Example 5: +- A: Northwell Health +- B: Lenox Hill Hospital | Staten Island University Hospital +- Result: ["N"] +- Why: System-level name does not equal either specific listed facility name. + +Example 6: +- A: Valley Medical Group +- B: Valley Medical Center +- Result: ["N"] +- Why: Group vs center are distinct entities without explicit alias evidence. + +Example 7: +- A: Mercy Clinic +- B: Mercy Clinic, LLC +- Result: ["Y"] +- Why: Same entity after legal suffix normalization. + +Example 8: +- A: UMC +- B: University Medical Center | United Medical Care +- Result: ["N"] +- Why: Acronym is ambiguous across multiple plausible expansions. + +[FINAL CHECKLIST BEFORE OUTPUT] +1. Did you compare A against each candidate in B separately? +2. Did you normalize punctuation/case/suffixes first? +3. Did you apply CRITICAL non-match exclusions? +4. Is the match supported by direct identity evidence, not topic similarity? +5. If uncertain, did you return ["N"]? + +[ADDITIONAL FALSE-MATCH GUARDS] +- Geographic descriptors ("of Texas", "of Northern California", "Greater Boston Area") attached to one name but not the other do not by themselves justify a match. They only matter when the rest of the entity name is otherwise identical after normalization. +- Service-line descriptors ("Cardiology", "Pediatrics", "Behavioral Health") attached to one name but not the other are NOT identity evidence. Treat them as additional context, not equivalences. +- A name that is just a generic word followed by "Health", "Care", "Group", or "Services" is too weak to drive a match on its own; require corroborating tokens (a city, a saint name, a founder name, a recognizable proper noun) before returning ["Y"]. +- Numeric identifiers (TINs, NPIs, internal IDs) embedded in either name are decisive: identical IDs lock a ["Y"] regardless of textual differences; conflicting IDs lock a ["N"] even when textual matching would suggest otherwise. + +[REASONING DISCIPLINE] +- Keep the reasoning paragraph short — three sentences or fewer — and avoid restating the rules. Reference only the specific tokens you compared. +- Do not list every rule you considered; cite only the ones that were decisive for this comparison. + [OUTPUT FORMAT] -Briefly explain your reasoning. Then return ONLY 'Y' (match found) or 'N' (no match found) as a single item in a JSON list. +Very briefly explain your reasoning. Then return ONLY 'Y' (match found) or 'N' (no match found) as a single item in a JSON list. Examples: ["Y"] or ["N"] {JSON_LIST_FORMAT_INSTRUCTIONS}""" @@ -2134,6 +3060,59 @@ Rules for Exclusion: * Do NOT include standalone headings that label a category of content within an exhibit's body (e.g., a heading that introduces a block of notes, definitions, billing rules, or procedural instructions). A valid exhibit header answers "which section/exhibit/amendment is this?" — a body-level label answers "what kind of content follows here?" Only the former should be included. Ask: would this heading appear in the document's table of contents as a top-level entry? If not, exclude it. * Do NOT include page-footer text. Footers appear near the bottom of a page adjacent to page numbers, contract/reference numbers, effective dates, or docusign IDs, and often repeat a SHORTENED form of the real exhibit identifier. For example, if the top-of-page header is "ATTACHMENT A-2 Oregon Health & Science University Effective 07/01/2025 Reimbursement Schedule", a bottom-of-page line like "Attachment A Oregon Health & Science University" next to "Page 2" or a contract number IS a footer, NOT a new section header — exclude it. When the same chunk contains a long-form exhibit header AND a short-form repeat of the same exhibit identifier near page metadata, keep only the long-form header. +[BOUNDARY MARKER HANDLING] +* Each ===BOUNDARY_X_HEADER=== marker signals the start of a new chunk. The number X is the key for that chunk in the output dictionary. +* A header may span multiple lines; collect all consecutive lines that form part of the same header title before body text begins. +* If a chunk contains multiple distinct headers (rare but possible), concatenate them with a newline (\n) in the value. +* If a boundary chunk contains only body text with no recognizable header, return "N/A" for that key. +* Do not confuse repeated page headers (running heads like a company name printed at the top of every page) with exhibit/section headers. Running heads are NOT section headers unless they match the keyword list above. + +[MULTI-LINE HEADER ASSEMBLY] +* Some headers are split across lines, e.g.: + "EXHIBIT B" + "FEE SCHEDULE FOR PROFESSIONAL SERVICES" + These two lines together form one header value: "EXHIBIT B\nFEE SCHEDULE FOR PROFESSIONAL SERVICES" +* Sub-labels that qualify or describe an exhibit (e.g., "Effective January 1, 2023") may be included as part of the header if they appear immediately after the header keyword line and before body text. +* Do NOT include effective date lines that appear mid-paragraph or mid-table — only include date qualifiers that appear as standalone lines directly adjacent to the header. + +[HEADER KEYWORD VARIANTS] +The following variants also qualify as section headers: +* Numbered variants: "EXHIBIT 1", "EXHIBIT A-1", "ATTACHMENT 2B", "SCHEDULE III" +* Titled variants: "EXHIBIT A: COMPENSATION SCHEDULE", "ADDENDUM NO. 3 - MENTAL HEALTH SERVICES" +* Combinations: "AMENDMENT TO EXHIBIT B", "SCHEDULE TO ATTACHMENT C" +* Standalone agreement names that serve as document titles: "PROVIDER SERVICES AGREEMENT", "PARTICIPATING HOSPITAL AGREEMENT", "LETTER OF AGREEMENT" +* Signature page identifiers: "SIGNATURE PAGE", "EXECUTION PAGE" when appearing directly under an exhibit or agreement title + +[EXACT PRESERVATION RULE] +* Preserve the exact capitalization, spacing, punctuation, and line breaks of each header as it appears in the source text. +* Do not normalize, abbreviate, or paraphrase. The extracted header text is used downstream for document splitting and any modification will break that process. + +[FEW-SHOT EXAMPLES] +Chunk text: "===BOUNDARY_1_HEADER===\nEXHIBIT A\nFEE SCHEDULE\nThis exhibit sets forth the rates..." +Output key "1": "EXHIBIT A\nFEE SCHEDULE" + +Chunk text: "===BOUNDARY_2_HEADER===\nProvider's Legal Name: ______\nAuthorized Signature: ______" +Output key "2": "N/A" + +Chunk text: "===BOUNDARY_3_HEADER===\nAMENDMENT NO. 2 TO PARTICIPATING PROVIDER AGREEMENT\nEffective April 1, 2024\nThis Amendment..." +Output key "3": "AMENDMENT NO. 2 TO PARTICIPATING PROVIDER AGREEMENT\nEffective April 1, 2024" + +Chunk text: "===BOUNDARY_4_HEADER===\nACME HEALTH PLAN\n123 Main Street\nSome City, TX 75001\nDear Provider," +Output key "4": "N/A" + +[BOUNDARY KEY DISCIPLINE] +- The output keys must be string representations of the boundary numbers as they appear in the markers (e.g. "1", "10", "27"). Do not pad with leading zeros, do not strip them, and do not coerce to integers. +- Every boundary marker present in the input must appear as a key in the output dictionary, even if its value is "N/A". Missing keys are a critical error because downstream code joins by key. +- Do not invent boundary keys that were not present in the input. If only boundaries 3, 4, 5 appear, return only "3", "4", "5". + +[NEAR-DUPLICATE HEADER HANDLING] +- When the same header text appears in consecutive chunks (because a multi-page exhibit reprints its title at the top of every page), report it for each chunk it appears in. Do not deduplicate. +- When two chunks have the same header text except for capitalization, spacing, or trailing punctuation, preserve each exactly as it appears in its own chunk; downstream linkage handles equivalence separately. + +[NOISE EXCLUSION] +- Lines that are exclusively page artifacts — page numbers like "Page 12 of 47", running footers, repeating boilerplate disclaimer text, DocuSign envelope IDs, watermarks — are not headers under any circumstance. +- Form field placeholders ("Provider's Legal Name: ____", "Date Signed: ____") are not headers even when they appear at the top of a chunk. + [OUTPUT FORMAT] Return a JSON dictionary enclosed in |pipes| where: - Keys are boundary numbers (e.g., "1", "2", "3") @@ -3009,3 +3988,100 @@ def SPLIT_SERVICE_TERM_INSTRUCTION() -> str: [OUTPUT FORMAT] {JSON_LIST_FORMAT_INSTRUCTIONS} """ + + +# PROGRAM to LOB prompt instruction and function +def PROGRAM_TO_LOB_INSTRUCTION() -> str: + """Static instruction for PROGRAM_TO_LOB prompt caching. + Contains objective, rules, definitions, and output format. + """ + return f"""[OBJECTIVE] +Given a PROGRAM value (or list of values) and a list of valid LOBs, extract the most appropriate LOB(s) that correspond to the PROGRAM(s). + +[INSTRUCTIONS] +- Use only the provided PROGRAM value(s) as input. +- Return only valid LOB(s) from the provided list that are directly and unambiguously associated with the PROGRAM(s). +- If no valid LOB can be determined, return [\"N/A\"]. +- If Payer Name is provided, use it as a secondary tie-breaker when the program name alone is ambiguous. Do not override a clear program-name signal with payer context alone. +- If Payer State is provided, use it to resolve state-specific program name ambiguity — the same program keyword can map to different LOBs depending on the state. + +[DEFINITIONS] +- PROGRAM: Government/state/public program names and public coverage program variants (e.g., CHIP, ACA, Medicare Advantage). +- LOB: Broad line-of-business categories (e.g., Medicaid, Medicare, Commercial, Marketplace). + +[LOB DISAMBIGUATION GUIDE] +- Rely primarily on explicit program name keywords to identify the LOB; use Payer Name as a secondary signal only when the program name is ambiguous. +- Use Payer State to resolve cases where the same program keyword maps to different LOBs across states. +- When in doubt or unable to confidently determine the LOB from the program name, payer name, or payer state, return ["N/A"]. Do not guess. +- CRITICAL — Medicare Advantage vs. Duals: "Medicare Advantage" (MA, MAPD, Part C) is a Medicare LOB, NOT Duals. Do NOT classify a program as Duals simply because it is a Medicare Advantage plan. Only assign the Duals LOB when the program explicitly targets individuals enrolled in BOTH Medicare and Medicaid — look for keywords such as "D-SNP", "DSNP", "Dual Special Needs Plan", "FIDE-SNP", "MMAI", "Medicare-Medicaid", or "Dual Eligible". Absent these specific indicators, default to Medicare for Medicare Advantage programs. + +[OUTPUT FORMAT] +Briefly explain why each LOB was selected. Return a valid JSON list of strings for the LOB(s). +{JSON_LIST_FORMAT_INSTRUCTIONS}""" + + +def PROGRAM_TO_LOB( + program_values, valid_lobs, payer_name: str = "", payer_state: str = "" +) -> Tuple[str, Callable[[str], list]]: + """Returns ONLY dynamic content for PROGRAM-to-LOB derivation. + Call PROGRAM_TO_LOB_INSTRUCTION() separately for the cached instruction. + + Returns: + Tuple of (prompt_string, parser_function) where parser expects JSON list output. + """ + context = f"PROGRAM values: {program_values}\nValid LOBs: {valid_lobs}" + if payer_name and payer_name.strip(): + context += f"\nPayer Name: {payer_name.strip()}" + if payer_state and payer_state.strip(): + context += f"\nPayer State: {payer_state.strip()}" + return (context, _json_list_parser) + + +# PRODUCT to LOB prompt instruction and function +def PRODUCT_TO_LOB_INSTRUCTION() -> str: + """Static instruction for PRODUCT_TO_LOB prompt caching. + Contains objective, rules, definitions, and output format. + """ + return f"""[OBJECTIVE] +Given a PRODUCT value (or list of values) and a list of valid LOBs, extract the most appropriate LOB(s) that correspond to the PRODUCT(s). + +[INSTRUCTIONS] +- Use only the provided PRODUCT value(s) as input. +- Return only valid LOB(s) from the provided list that are directly and unambiguously associated with the PRODUCT(s). +- If no valid LOB can be determined, return [\"N/A\"]. +- If Payer Name is provided, use it as a secondary tie-breaker when the product name alone is ambiguous. Do not override a clear product-name signal with payer context alone. +- If Payer State is provided, use it to resolve state-specific product name ambiguity — the same branded product name can map to different LOBs depending on the state. + +[DEFINITIONS] +- PRODUCT: Payer-branded or plan-branded commercial product names and plan labels (e.g., Blue Cross PPO, Aetna Medicare). +- LOB: Broad line-of-business categories (e.g., Medicaid, Medicare, Commercial, Marketplace). + +[LOB DISAMBIGUATION GUIDE] +- Rely primarily on explicit product name keywords to identify the LOB; use Payer Name as a secondary signal only when the product name is ambiguous. +- Use Payer State to resolve cases where the same product name maps to different LOBs across states. +- When in doubt or unable to confidently determine the LOB from the product name, payer name, or payer state, return ["N/A"]. Do not guess. +- CRITICAL — Medicare Advantage vs. Duals: "Medicare Advantage" (MA, MAPD, Part C) is a Medicare LOB, NOT Duals. Do NOT classify a product as Duals simply because it is a Medicare Advantage product. Only assign the Duals LOB when the product explicitly targets individuals enrolled in BOTH Medicare and Medicaid — look for keywords such as "D-SNP", "DSNP", "Dual Special Needs Plan", "FIDE-SNP", "MMAI", "Medicare-Medicaid Plan", or "Dual Eligible". Absent these specific indicators, default to Medicare for Medicare Advantage products. +- For products with ambiguous names that do not clearly signal a single LOB, return ["N/A"]. Do not guess based on naming conventions or payer context alone. +- For Essential Plan products in New York, assign to Medicaid LOB, as they are state-funded and administered as part of the Medicaid program, even though they are technically a separate product line. +- For Quality Blue, Together Blue and True Performance Programs in BCBS New York, assign to Marketsplace LOB, as they are exchange-based products, even though their names do not include typical Marketplace indicators. + +[OUTPUT FORMAT] +Briefly explain why each LOB was selected. Return a valid JSON list of strings for the LOB(s). +{JSON_LIST_FORMAT_INSTRUCTIONS}""" + + +def PRODUCT_TO_LOB( + product_values, valid_lobs, payer_name: str = "", payer_state: str = "" +) -> Tuple[str, Callable[[str], list]]: + """Returns ONLY dynamic content for PRODUCT-to-LOB derivation. + Call PRODUCT_TO_LOB_INSTRUCTION() separately for the cached instruction. + + Returns: + Tuple of (prompt_string, parser_function) where parser expects JSON list output. + """ + context = f"PRODUCT values: {product_values}\nValid LOBs: {valid_lobs}" + if payer_name and payer_name.strip(): + context += f"\nPayer Name: {payer_name.strip()}" + if payer_state and payer_state.strip(): + context += f"\nPayer State: {payer_state.strip()}" + return (context, _json_list_parser) diff --git a/src/scripts/prompt_call_log_report.py b/src/scripts/prompt_call_log_report.py new file mode 100644 index 0000000..a2f2048 --- /dev/null +++ b/src/scripts/prompt_call_log_report.py @@ -0,0 +1,254 @@ +import importlib.util +import inspect +import re +from pathlib import Path + +import pandas as pd + +ANTHROPIC_CACHE_MIN_TOKENS = 1024 +WORDS_TO_TOKENS_MULTIPLIER = 1.3 + +ALIASES = { + "date_fix": "DATE_FIX", + "derive_term_date": "DERIVED_TERM_DATE", +} + +ROUND_COLS = [ + "avg_prompt_tokens_est", + "avg_instruction_tokens_est", + "avg_context_tokens_est", + "avg_input_tokens", + "avg_api_cached_input_tokens", + "avg_api_total_billed_input_tokens", +] + +TOKENS_PER_CALL_COLS = [ + "timestamp", + "contract_filename", + "usage_label", + "model_id", + "resolved_model_id", + "status", + "from_local_memory_cache", + "prompt_tokens_est", + "instruction_tokens_est", + "context_tokens_est", + "total_input_tokens_est", + "api_cached_input_tokens", + "api_total_billed_input_tokens", + "est_minus_api_total_tokens", + "longest_component", + "input_tokens", + "output_tokens", + "cache_creation_tokens", + "cache_read_tokens", +] + +INSTRUCTION_AUDIT_COLS = [ + "requested_name", + "normalized_prompt", + "instruction_function_found", + "instruction_chars", + "instruction_tokens_est", + "cache_min_tokens", + "cacheable_at_threshold", + "tokens_to_reach_threshold", + "priority", + "instruction_preview", +] + + +def _min_cache_tokens_for_label( + df: pd.DataFrame, + label: str, +) -> int: + """Return the most restrictive cache-minimum across all model IDs seen for + *label* in the prompt-call log. Falls back to ANTHROPIC_CACHE_MIN_TOKENS + when the column is absent or empty.""" + from src.utils.llm_utils import _min_cache_tokens + + col = "resolved_model_id" if "resolved_model_id" in df.columns else "model_id" + mask = df.get("usage_label", pd.Series(dtype=str)).astype(str).str.strip() == label + model_ids = df.loc[mask, col].dropna().unique() if col in df.columns else [] + if len(model_ids) == 0: + return ANTHROPIC_CACHE_MIN_TOKENS + return max(_min_cache_tokens(str(mid)) for mid in model_ids) + + +def _build_instruction_audit( + df: pd.DataFrame, + prompt_templates_file: Path, +) -> pd.DataFrame: + instruction_texts: dict[str, str] = {} + if not prompt_templates_file.exists(): + raise FileNotFoundError( + f"Prompt templates file not found: {prompt_templates_file}" + ) + + spec = importlib.util.spec_from_file_location( + "prompt_templates_for_audit", prompt_templates_file + ) + if spec is None or spec.loader is None: + raise ImportError(f"Unable to load module from {prompt_templates_file}") + + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + for name, fn in inspect.getmembers(module, inspect.isfunction): + if not name.endswith("_INSTRUCTION"): + continue + label = name.removesuffix("_INSTRUCTION") + try: + text = fn() + except TypeError: + text = "" + instruction_texts[label] = text if isinstance(text, str) else str(text) + + prompt_names = sorted( + {str(v) for v in df.get("usage_label", pd.Series(dtype=str)).dropna().unique()} + ) + + rows: list[dict] = [] + for raw_name in prompt_names: + normalized = ALIASES.get(raw_name.strip(), raw_name.strip().upper()) + text = instruction_texts.get(normalized, "") + tokens = int(round(len(re.findall(r"\S+", text)) * WORDS_TO_TOKENS_MULTIPLIER)) + threshold = _min_cache_tokens_for_label(df, raw_name.strip()) + gap = max(0, threshold - tokens) + rows.append( + { + "requested_name": raw_name, + "normalized_prompt": normalized, + "instruction_function_found": normalized in instruction_texts, + "instruction_chars": len(text), + "instruction_tokens_est": tokens, + "cache_min_tokens": threshold, + "cacheable_at_threshold": tokens >= threshold, + "tokens_to_reach_threshold": gap, + "priority": "OK" if gap == 0 else ("HIGH" if gap >= 300 else "MEDIUM"), + "instruction_preview": ( + (text[:240].replace("\n", " ") + "...") if text else "" + ), + } + ) + + if not rows: + return pd.DataFrame(columns=INSTRUCTION_AUDIT_COLS) + + return pd.DataFrame(rows).sort_values( + by=["cacheable_at_threshold", "tokens_to_reach_threshold", "normalized_prompt"], + ascending=[True, False, True], + ) + + +def run_reports( + prompt_call_log: Path, + output_dir: Path, + prompt_templates_file: Path, +) -> dict[str, Path]: + output_dir.mkdir(parents=True, exist_ok=True) + + if not prompt_call_log.exists(): + raise FileNotFoundError(f"Prompt call log not found: {prompt_call_log}") + df = pd.read_csv(prompt_call_log) + + tokens_per_call = df[[c for c in TOKENS_PER_CALL_COLS if c in df.columns]].copy() + tokens_per_call = tokens_per_call.sort_values( + by=["contract_filename", "usage_label", "timestamp"] + ) + + success_df = df[df["status"] == "success"] + calls_per_contract = ( + success_df.groupby(["contract_filename", "usage_label"], dropna=False) + .agg( + call_count=("usage_label", "size"), + avg_prompt_tokens_est=("prompt_tokens_est", "mean"), + avg_instruction_tokens_est=("instruction_tokens_est", "mean"), + avg_context_tokens_est=("context_tokens_est", "mean"), + avg_input_tokens=("input_tokens", "mean"), + avg_api_cached_input_tokens=("api_cached_input_tokens", "mean"), + avg_api_total_billed_input_tokens=("api_total_billed_input_tokens", "mean"), + ) + .reset_index() + .sort_values(by=["contract_filename", "call_count"], ascending=[True, False]) + ) + if not calls_per_contract.empty: + calls_per_contract[ROUND_COLS] = calls_per_contract[ROUND_COLS].round(2) + + longest_component = ( + success_df.groupby(["usage_label", "longest_component"], dropna=False) + .size() + .reset_index(name="occurrences") + .merge( + success_df.groupby(["usage_label"], dropna=False) + .size() + .reset_index(name="total_calls"), + on=["usage_label"], + how="left", + ) + ) + if longest_component.empty: + longest_component = pd.DataFrame( + columns=[ + "usage_label", + "longest_component", + "occurrences", + "percent_of_calls", + ] + ) + else: + longest_component["percent_of_calls"] = ( + (longest_component["occurrences"] / longest_component["total_calls"]) * 100 + ).round(2) + longest_component = longest_component.drop(columns=["total_calls"]).sort_values( + by=["usage_label", "occurrences"], ascending=[True, False] + ) + + instruction_audit = _build_instruction_audit( + df, prompt_templates_file=prompt_templates_file + ) + + tokens_path = output_dir / "runtime_prompt_tokens_per_call.csv" + calls_path = output_dir / "runtime_prompt_calls_per_contract.csv" + longest_path = output_dir / "runtime_longest_component_summary.csv" + audit_path = output_dir / "prompt_instruction_audit.csv" + + tokens_per_call.to_csv(tokens_path, index=False) + calls_per_contract.to_csv(calls_path, index=False) + longest_component.to_csv(longest_path, index=False) + instruction_audit.to_csv(audit_path, index=False) + + return { + "tokens_per_call": tokens_path, + "calls_per_contract": calls_path, + "longest_component": longest_path, + "instruction_audit": audit_path, + } + + +PROMPT_TEMPLATES_FILE = Path("src/prompts/prompt_templates.py") + + +def main() -> None: + prompt_call_log = max( + Path("outputs").glob("**/tracking/*-PROMPT-CALLS.csv"), + key=lambda p: p.stat().st_mtime, + default=None, + ) + if prompt_call_log is None: + raise FileNotFoundError("No prompt call logs found under outputs/**/tracking") + + outputs = run_reports( + prompt_call_log=prompt_call_log, + output_dir=prompt_call_log.parent, + prompt_templates_file=PROMPT_TEMPLATES_FILE, + ) + + print(f"Source log: {prompt_call_log}") + print("Generated report files:") + for key, path in outputs.items(): + print(f"- {key}: {path}") + + +if __name__ == "__main__": + main() diff --git a/src/tests/test_crosswalk_utils.py b/src/tests/test_crosswalk_utils.py index 5cd1888..b6ea656 100644 --- a/src/tests/test_crosswalk_utils.py +++ b/src/tests/test_crosswalk_utils.py @@ -128,28 +128,6 @@ def test_to_csv_no_metadata(tmp_path): assert result_dict == {"A": "one", "B": "two"} -def test_from_json_client_none_merges_all_client_mappings(tmp_path, monkeypatch): - data = { - "metadata": {"from_col": "from_col", "to_col": "to_col"}, - "mapping": {}, - "client_mapping": { - "ClientA": {"Connexus": "Commercial", "Synergy": "Commercial"}, - "ClientB": {"HEALTHfirst": "Medicaid"}, - }, - } - file_path = tmp_path / "client_merge.json" - with open(file_path, "w") as f: - json.dump(data, f) - - monkeypatch.setattr("src.crosswalk.crosswalk_builder.CLIENT", ["None"]) - monkeypatch.setattr("src.crosswalk.crosswalk_builder.STATE", ["None"]) - - builder = CrosswalkBuilder().from_json(file_path) - assert builder.mapping["Connexus"] == "Commercial" - assert builder.mapping["Synergy"] == "Commercial" - assert builder.mapping["HEALTHfirst"] == "Medicaid" - - def test_apply_crosswalk(): mapping = {"A": "1", "B": "2", "C": "3"} assert apply_crosswalk("A", mapping) == ["1"] diff --git a/src/tests/test_one_to_n.py b/src/tests/test_one_to_n.py index e7101f8..082b4bf 100644 --- a/src/tests/test_one_to_n.py +++ b/src/tests/test_one_to_n.py @@ -648,14 +648,10 @@ class TestOneToNFuncs(unittest.TestCase): "src.pipelines.shared.extraction.one_to_n_funcs.aarete_derived.get_crosswalk_fields" ) @patch("src.pipelines.shared.extraction.one_to_n_funcs.get_lob_relationship") - @patch( - "src.pipelines.shared.extraction.one_to_n_funcs.aarete_derived.fill_na_mapping" - ) @patch("src.pipelines.shared.extraction.one_to_n_funcs.split_reimb_dates") def test_one_to_n_cleaning_full_pipeline( self, mock_split_dates, - mock_fill_na, mock_get_lob, mock_crosswalk, ): @@ -664,7 +660,6 @@ class TestOneToNFuncs(unittest.TestCase): mock_crosswalk.return_value = input_data mock_get_lob.return_value = input_data - mock_fill_na.return_value = input_data mock_split_dates.return_value = input_data result = one_to_n_funcs.one_to_n_cleaning( @@ -674,7 +669,6 @@ class TestOneToNFuncs(unittest.TestCase): # Verify all cleaning steps were called mock_crosswalk.assert_called_once() mock_get_lob.assert_called_once() - mock_fill_na.assert_called_once() mock_split_dates.assert_called_once() self.assertEqual(result, input_data) diff --git a/src/tests/test_postprocess.py b/src/tests/test_postprocess.py index 8aaab32..399e177 100644 --- a/src/tests/test_postprocess.py +++ b/src/tests/test_postprocess.py @@ -943,24 +943,37 @@ class TestPostprocessFunctions(unittest.TestCase): self.assertEqual(result_df.loc[0, "CONTRACT_TITLE"], "") self.assertEqual(result_df.loc[1, "CONTRACT_TITLE"], "Another Valid") - def test_fill_na_mapping_derives_lob_from_aarete_derived_product(self): - """fill_na_mapping should derive LOB from AARETE_DERIVED_PRODUCT.""" - rows = [ - { - "PRODUCT": "", - "AARETE_DERIVED_PRODUCT": '["Connexus", "Synergy"]', - "AARETE_DERIVED_LOB": "", - "LOB": "", - "AARETE_DERIVED_PROGRAM": "", - "LOB_PROGRAM_RELATIONSHIP": "", - "LOB_PRODUCT_RELATIONSHIP": "", - } - ] + @patch( + "src.pipelines.shared.postprocessing.aarete_derived.prompt_calls.prompt_product_to_lob", + return_value=["Commercial"], + ) + def test_fill_na_mapping_derives_lob_from_aarete_derived_product( + self, mock_product_lob + ): + """fill_na_mapping should derive LOB from AARETE_DERIVED_PRODUCT via LLM.""" + rows = pd.DataFrame( + [ + { + "FILE_NAME": "test_file", + "PRODUCT": ["Connexus", "Synergy"], + "AARETE_DERIVED_PRODUCT": ["Connexus", "Synergy"], + "AARETE_DERIVED_LOB": "", + "LOB": "", + "AARETE_DERIVED_PROGRAM": "", + "LOB_PROGRAM_RELATIONSHIP": "", + "LOB_PRODUCT_RELATIONSHIP": "", + } + ] + ) - result = aarete_derived.fill_na_mapping(rows) + constants = MagicMock() + constants.CROSSWALK_PROGRAM_LOB.mapping = {} + constants.CROSSWALK_PRODUCT_LOB.mapping = {} + constants.CROSSWALK_LOB.mapping = {} - self.assertEqual(result[0]["AARETE_DERIVED_LOB"], ["Commercial"]) - self.assertEqual(result[0]["LOB"], ["Commercial"]) + result = aarete_derived.fill_na_mapping(rows, constants) + + self.assertEqual(result.iloc[0]["AARETE_DERIVED_LOB"], ["Commercial"]) class TestStandardizeReimbMethodAndFeeScheduleUOM(unittest.TestCase): diff --git a/src/utils/llm_utils.py b/src/utils/llm_utils.py index 0019832..897cd0b 100644 --- a/src/utils/llm_utils.py +++ b/src/utils/llm_utils.py @@ -11,6 +11,7 @@ from collections import OrderedDict import anthropic import src.utils.string_utils as string_utils +import src.utils.prompt_call_tracking as prompt_call_tracking import src.utils.usage_tracking as usage_tracking from botocore.exceptions import ClientError, ReadTimeoutError from src import config @@ -103,22 +104,56 @@ def _validate_model_support( def _supports_prompt_cache(model_id: str) -> bool: """Return True if the given model supports prompt caching via system cache_control. - Enable only for models that are known to support prompt caching: + Enable for models that are known to support prompt caching: - Claude 3.5 Sonnet v2+ (20241022-v2:0) - Claude 3.7 Sonnet - Claude 4 Sonnet - Claude 4.5 Sonnet + - Claude 4.5 Haiku """ supported_ids = { getattr(config, "MODEL_ID_CLAUDE35_SONNET_V2", None), getattr(config, "MODEL_ID_CLAUDE37_SONNET", None), getattr(config, "MODEL_ID_CLAUDE4_SONNET", None), getattr(config, "MODEL_ID_CLAUDE45_SONNET", None), + getattr(config, "MODEL_ID_CLAUDE45_HAIKU", None), } # Filter out None values in case any model IDs are not configured return model_id in {m for m in supported_ids if m is not None} +# Per Anthropic docs (https://docs.anthropic.com/en/docs/prompt-caching), +# the minimum cacheable prompt length differs by model family: +# 1024 tokens — Sonnet 3.7 / 4 / 4.5, Opus 4 / 4.1 +# 2048 tokens — Sonnet 4.6, Haiku 3.5 (deprecated) +# 4096 tokens — Sonnet/Opus Mythos, Opus 4.5/4.6/4.7, Haiku 4.5 +# Caches against blocks below the threshold are silently dropped — the +# request still succeeds and returns content, but `cache_creation_tokens` +# and `cache_read_tokens` will both be 0 forever. That is the +# `no_cache_event` signature surfaced in runtime tracking. +_MODEL_MIN_CACHE_TOKENS_DEFAULT = 1024 +_MODEL_MIN_CACHE_TOKENS: dict[str, int] = { + # Sonnet 4.5 — 1024 + getattr(config, "MODEL_ID_CLAUDE45_SONNET", "_unset_45_sonnet"): 1024, + # Sonnet 4 / 3.7 / 3.5 v2 — 1024 + getattr(config, "MODEL_ID_CLAUDE4_SONNET", "_unset_4_sonnet"): 1024, + getattr(config, "MODEL_ID_CLAUDE37_SONNET", "_unset_37_sonnet"): 1024, + getattr(config, "MODEL_ID_CLAUDE35_SONNET_V2", "_unset_35_sonnet_v2"): 1024, + # Haiku 4.5 — 4096 + getattr(config, "MODEL_ID_CLAUDE45_HAIKU", "_unset_45_haiku"): 4096, +} + + +def _min_cache_tokens(model_id: str) -> int: + """Return the minimum prompt-cache block size honored by the model. + + Falls back to 1024 (the historical Sonnet minimum) when the model is + unknown — that is the most permissive choice; callers only get a + surprise `no_cache_event` if the actual minimum is higher. + """ + return _MODEL_MIN_CACHE_TOKENS.get(model_id, _MODEL_MIN_CACHE_TOKENS_DEFAULT) + + def _classify_cache_outcome( *, has_context_cache: bool, @@ -410,11 +445,12 @@ def _build_claude_3_request_body( if context_for_caching: if cache and model_id and _supports_prompt_cache(model_id): token_estimate = len(context_for_caching.split()) * 1.3 - if token_estimate < 1024: + min_tokens = _min_cache_tokens(model_id) + if token_estimate < min_tokens: logging.warning( f"context_for_caching is ~{token_estimate:.0f} tokens, below the " - "Anthropic 1024-token minimum for prompt caching; context will be " - "sent but the cache_control block may not actually be cached." + f"Anthropic {min_tokens}-token minimum for prompt caching on {model_id}; " + "context will be sent but the cache_control block may not actually be cached." ) user_content.append( { @@ -557,10 +593,30 @@ def invoke_claude( context_for_caching=context_for_caching, ) + # Resolve alias up-front so cache-policy classification in tracking + # logs uses the durable id, not whatever alias the caller happened to + # pass. + resolved_model_id = config.resolve_model_id(model_id) + model_supports_cache = _supports_prompt_cache(resolved_model_id) + # Check cache first - thread-safe access with _cache_lock: if cache_key in claude_cache: claude_cache.move_to_end(cache_key) # Mark as recently used + prompt_call_tracking.log_prompt_call( + contract_filename=filename, + usage_label=usage_label, + model_id=model_id, + cache_enabled=cache, + max_tokens=max_tokens, + prompt=prompt, + instruction=instruction, + context_for_caching=context_for_caching, + status="success", + from_local_memory_cache=True, + resolved_model_id=resolved_model_id, + model_supports_cache=model_supports_cache, + ) # One llm_call_inmem_hit event per in-memory cache return. from src.utils import instrumentation, instrumentation_context @@ -594,49 +650,63 @@ def invoke_claude( ) return claude_cache[cache_key] - # Resolve alias to actual model ID - resolved_model_id = config.resolve_model_id(model_id) - # Validate model support (checks for deprecated and unsupported models) _validate_model_support(model_id, resolved_model_id) # Push call metadata into context so _track_usage_from_response can read it. from src.utils.instrumentation_context import instr_scope - with instr_scope( - _usage_label=usage_label, - _has_context_cache=bool(context_for_caching), - _context_chars=len(context_for_caching) if context_for_caching else 0, - _prompt_chars=len(prompt) if prompt else 0, - _cache_key=cache_key, - ): - if config.RUN_MODE == "local": - response = local_claude_3_and_up( - prompt, - resolved_model_id, - filename, - max_tokens, - cache=cache, - instruction=instruction, - usage_label=usage_label, - context_for_caching=context_for_caching, - ) - elif config.RUN_MODE == "ec2": - response = ec2_claude_3_and_up( - prompt, - resolved_model_id, - filename, - max_tokens, - cache=cache, - instruction=instruction, - usage_label=usage_label, - context_for_caching=context_for_caching, - ) - else: - logging.error( - "Usage: python local_main.py <-local> OR python main.py " - ) - raise + try: + with instr_scope( + _usage_label=usage_label, + _has_context_cache=bool(context_for_caching), + _context_chars=len(context_for_caching) if context_for_caching else 0, + _prompt_chars=len(prompt) if prompt else 0, + _cache_key=cache_key, + ): + if config.RUN_MODE == "local": + response = local_claude_3_and_up( + prompt, + resolved_model_id, + filename, + max_tokens, + cache=cache, + instruction=instruction, + usage_label=usage_label, + context_for_caching=context_for_caching, + ) + elif config.RUN_MODE == "ec2": + response = ec2_claude_3_and_up( + prompt, + resolved_model_id, + filename, + max_tokens, + cache=cache, + instruction=instruction, + usage_label=usage_label, + context_for_caching=context_for_caching, + ) + else: + logging.error( + "Usage: python local_main.py <-local> OR python main.py " + ) + raise + except Exception as e: + prompt_call_tracking.log_prompt_call( + contract_filename=filename, + usage_label=usage_label, + model_id=resolved_model_id, + cache_enabled=cache, + max_tokens=max_tokens, + prompt=prompt, + instruction=instruction, + context_for_caching=context_for_caching, + status="error", + error=str(e), + resolved_model_id=resolved_model_id, + model_supports_cache=model_supports_cache, + ) + raise # Store response in cache with LRU eviction - thread-safe _store_in_cache(cache_key, response) @@ -806,9 +876,34 @@ def local_claude_3_and_up( response_body = json.loads(response.get("body").read()) response_text = _extract_text_from_claude_response(response_body) + ( + input_tokens, + output_tokens, + cache_creation_tokens, + cache_read_tokens, + ) = usage_tracking.extract_usage_from_response(response_body) + # Track usage and costs from API response (including cache tokens) _track_usage_from_response(response_body, filename, model_id) + prompt_call_tracking.log_prompt_call( + contract_filename=filename, + usage_label=usage_label or "UNLABELED", + model_id=model_id, + cache_enabled=cache, + max_tokens=max_tokens, + prompt=prompt, + instruction=instruction, + context_for_caching=context_for_caching, + status="success", + input_tokens=input_tokens, + output_tokens=output_tokens, + cache_creation_tokens=cache_creation_tokens, + cache_read_tokens=cache_read_tokens, + resolved_model_id=model_id, + model_supports_cache=_supports_prompt_cache(model_id), + ) + if config.ENABLE_ANSWER_TRACKING: config.ANSWER_TRACKING_DICT[prompt] = response_text @@ -1050,9 +1145,34 @@ def ec2_claude_3_and_up( response_body = json.loads(response.get("body").read()) response_text = _extract_text_from_claude_response(response_body) + ( + input_tokens, + output_tokens, + cache_creation_tokens, + cache_read_tokens, + ) = usage_tracking.extract_usage_from_response(response_body) + # Track usage and costs from API response (including cache tokens) _track_usage_from_response(response_body, filename, model_id) + prompt_call_tracking.log_prompt_call( + contract_filename=filename, + usage_label=usage_label or "UNLABELED", + model_id=model_id, + cache_enabled=cache, + max_tokens=max_tokens, + prompt=prompt, + instruction=instruction, + context_for_caching=context_for_caching, + status="success", + input_tokens=input_tokens, + output_tokens=output_tokens, + cache_creation_tokens=cache_creation_tokens, + cache_read_tokens=cache_read_tokens, + resolved_model_id=model_id, + model_supports_cache=_supports_prompt_cache(model_id), + ) + return response_text except ValueError as e: @@ -1142,9 +1262,34 @@ def ec2_claude_3_and_up( response_body = json.loads(response.get("body").read()) response_text = _extract_text_from_claude_response(response_body) + ( + input_tokens, + output_tokens, + cache_creation_tokens, + cache_read_tokens, + ) = usage_tracking.extract_usage_from_response(response_body) + # Track usage and costs from API response (including cache tokens) _track_usage_from_response(response_body, filename, model_id) + prompt_call_tracking.log_prompt_call( + contract_filename=filename, + usage_label=usage_label or "UNLABELED", + model_id=model_id, + cache_enabled=cache, + max_tokens=max_tokens, + prompt=prompt, + instruction=instruction, + context_for_caching=context_for_caching, + status="success", + input_tokens=input_tokens, + output_tokens=output_tokens, + cache_creation_tokens=cache_creation_tokens, + cache_read_tokens=cache_read_tokens, + resolved_model_id=model_id, + model_supports_cache=_supports_prompt_cache(model_id), + ) + return response_text except ValueError as e: @@ -1241,32 +1386,58 @@ def warm_prompt_cache( ): """ Warms the prompt cache by sending a minimal request with the instruction. - This should be called once before parallel processing to ensure the cache is registered. + This should be called once before parallel processing to ensure the cache + is registered. + + Cache entries are model-specific in Bedrock — a cache created against + `sonnet_latest` is NOT readable from a runtime call invoked against + `haiku_latest`. The warmed-state map is therefore keyed by + (cache_key, resolved_model_id) so repeated calls with different models + each get warmed exactly once. Args: instruction (str): The instruction text to cache cache_key (str): A unique identifier for this cache (e.g., "REIMBURSEMENT_PRIMARY") - model_id (str): The model ID to use. Defaults to "sonnet_latest" + model_id (str): The model ID or alias to use. Defaults to "sonnet_latest" max_tokens (int): Minimal tokens for cache warming. Defaults to 10 Returns: - bool: True if cache was warmed, False if it was already warm + bool: True if cache was warmed, False if it was already warm or + the resolved model does not support prompt caching. """ global _cache_warmed + # Resolve model ID up-front so the warmed-state key matches what the + # runtime call will use after its own resolve_model_id(). + resolved_model_id = config.resolve_model_id(model_id) + + # Hard skip on models that do not support prompt caching in this + # Bedrock setup. Without this, warming spends create-tokens against a + # cache the runtime call can never read. + if not _supports_prompt_cache(resolved_model_id): + logging.info( + "Skipping cache warming for %s on %s (resolved %s): model does " + "not support prompt caching.", + cache_key, + model_id, + resolved_model_id, + ) + return False + + warmed_key = (cache_key, resolved_model_id) + # Check if already warmed (thread-safe) with _cache_warming_lock: - if _cache_warmed.get(cache_key, False): - logging.debug(f"Cache already warmed for {cache_key}") + if _cache_warmed.get(warmed_key, False): + logging.debug( + f"Cache already warmed for {cache_key} on {resolved_model_id}" + ) return False # Mark as being warmed - _cache_warmed[cache_key] = True + _cache_warmed[warmed_key] = True - logging.info(f"Warming prompt cache for {cache_key}...") - - # Resolve model ID - resolved_model_id = config.resolve_model_id(model_id) + logging.info(f"Warming prompt cache for {cache_key} on {resolved_model_id}...") # Create minimal prompt with the instruction to register the cache minimal_prompt = "Respond with OK." @@ -1323,5 +1494,5 @@ def warm_prompt_cache( logging.error(f"Error warming cache for {cache_key}: {str(e)}") # Mark as not warmed on error so it can be retried with _cache_warming_lock: - _cache_warmed[cache_key] = False + _cache_warmed[warmed_key] = False return False diff --git a/src/utils/program_product_mapping.py b/src/utils/program_product_mapping.py new file mode 100644 index 0000000..f1cb474 --- /dev/null +++ b/src/utils/program_product_mapping.py @@ -0,0 +1,156 @@ +"""Loader for program_product_lob_mapping.csv. + +The CSV lives in the same S3 prefix as the txt files: + s3:////program_product_lob_mapping.csv + s3://// + +Key columns: + MBR_PROGRAM_DESC -> AARETE_DERIVED_PROGRAM valid values + MBR_PRODUCT_DESC -> AARETE_DERIVED_PRODUCT valid values + MBR_PRODUCT_LINE_OF_BUSINESS -> AARETE_DERIVED_LOB value for that row + +The mapping is loaded once at pipeline startup and injected into Constants so every +file processed in the same batch shares the same read-only data. +If the CSV is absent, the pipeline continues normally with empty mapping lists. +""" + +import logging +from dataclasses import dataclass, field +from io import StringIO + +import pandas as pd +from botocore.exceptions import ClientError + +import src.config as config + + +# --------------------------------------------------------------------------- +# Data class +# --------------------------------------------------------------------------- + + +@dataclass +class ProgramProductMapping: + """Holds all derived data extracted from program_product_lob_mapping.csv.""" + + # AARETE_DERIVED_PROGRAM -> AARETE_DERIVED_LOB (non-empty when CSV has program rows) + program_to_lob: dict[str, str] = field(default_factory=dict) + + # AARETE_DERIVED_PRODUCT -> AARETE_DERIVED_LOB (non-empty when CSV has product rows) + product_to_lob: dict[str, str] = field(default_factory=dict) + + # Sorted list of unique AARETE_DERIVED_PROGRAM values from the CSV + valid_aarete_derived_programs: list[str] = field(default_factory=list) + + # Sorted list of unique AARETE_DERIVED_PRODUCT values from the CSV + valid_aarete_derived_products: list[str] = field(default_factory=list) + + @property + def use_crosswalk_for_program(self) -> bool: + """True when the CSV has program->LOB entries (use crosswalk, skip LLM).""" + return bool(self.program_to_lob) + + @property + def use_crosswalk_for_product(self) -> bool: + """True when the CSV has product->LOB entries (use crosswalk, skip LLM).""" + return bool(self.product_to_lob) + + +# --------------------------------------------------------------------------- +# Internal builder +# --------------------------------------------------------------------------- + + +def _build_from_dataframe(df: pd.DataFrame) -> ProgramProductMapping: + """Parse the loaded DataFrame into a ProgramProductMapping.""" + program_to_lob: dict[str, str] = {} + product_to_lob: dict[str, str] = {} + programs: set[str] = set() + products: set[str] = set() + + for _, row in df.iterrows(): + lob_raw = row.get("MBR_PRODUCT_LINE_OF_BUSINESS", "") + lob = str(lob_raw).strip() if pd.notna(lob_raw) else "" + if not lob or lob.lower() in ("nan", "n/a", "none", ""): + continue + + prog_raw = row.get("MBR_PROGRAM_DESC", "") + prog = str(prog_raw).strip() if pd.notna(prog_raw) else "" + if prog and prog.lower() not in ("nan", "n/a", "none", ""): + program_to_lob[prog] = lob + programs.add(prog) + + prod_raw = row.get("MBR_PRODUCT_DESC", "") + prod = str(prod_raw).strip() if pd.notna(prod_raw) else "" + if prod and prod.lower() not in ("nan", "n/a", "none", ""): + product_to_lob[prod] = lob + products.add(prod) + + mapping = ProgramProductMapping( + program_to_lob=program_to_lob, + product_to_lob=product_to_lob, + valid_aarete_derived_programs=sorted(programs), + valid_aarete_derived_products=sorted(products), + ) + logging.info( + f"program_product_mapping loaded: " + f"{len(programs)} programs, {len(products)} products, " + f"program_crosswalk={'enabled' if mapping.use_crosswalk_for_program else 'disabled (LLM fallback)'}, " + f"product_crosswalk={'enabled' if mapping.use_crosswalk_for_product else 'disabled (LLM fallback)'}" + ) + return mapping + + +# --------------------------------------------------------------------------- +# Public loaders +# --------------------------------------------------------------------------- + + +def load_from_s3(bucket: str, key: str) -> ProgramProductMapping | None: + """Load program_product_lob_mapping.csv from S3. + + Args: + bucket: S3 bucket name. + key: S3 object key (e.g. "run-folder/program_product_lob_mapping.csv"). + + Returns: + ProgramProductMapping on success, None if the file does not exist or on failure. + """ + try: + response = config.S3_CLIENT.get_object(Bucket=bucket, Key=key) + content = response["Body"].read().decode("latin-1") + df = pd.read_csv(StringIO(content), sep=",", dtype=str) + return _build_from_dataframe(df) + except ClientError as e: + if e.response["Error"]["Code"] in ("NoSuchKey", "404"): + logging.warning( + f"program_product_lob_mapping.csv not found at " + f"s3://{bucket}/{key}; crosswalk/derived mappings will be empty." + ) + else: + logging.error( + f"Failed to load program_product_mapping from s3://{bucket}/{key}: {e}" + ) + return None + except Exception as e: + logging.error( + f"Failed to load program_product_mapping from s3://{bucket}/{key}: {e}" + ) + return None + + +def load_from_local(path: str) -> ProgramProductMapping | None: + """Load program_product_lob_mapping.csv from a local file path. + + Args: + path: Absolute or relative path to the CSV file. + + Returns: + ProgramProductMapping on success, None on failure. + """ + try: + df = pd.read_csv(path, encoding="latin-1", sep=",", dtype=str) + return _build_from_dataframe(df) + except Exception as e: + logging.error(f"Failed to load program_product_mapping from {path}: {e}") + return None diff --git a/src/utils/prompt_call_tracking.py b/src/utils/prompt_call_tracking.py new file mode 100644 index 0000000..e93150f --- /dev/null +++ b/src/utils/prompt_call_tracking.py @@ -0,0 +1,354 @@ +"""Runtime prompt-call tracking for prompt caching analysis. + +Writes one CSV row per invoke_claude call so we can answer: +- token sizes per prompt component +- number of calls per contract and prompt label +- longest prompt component (context vs instruction vs field prompt) +- which cache class each call exercised and why a cache read was missed +""" + +import csv +import logging +import os +import re +import threading +from datetime import datetime +from pathlib import Path + +from src import config + +WORDS_TO_TOKENS_MULTIPLIER = 1.3 + +# Default minimum tokens for a `cache_control` block to be honored. +# Per-model overrides are pulled lazily from `llm_utils._min_cache_tokens` +# at call time. Falling back to 1024 here matches the historical Sonnet +# minimum and is the most permissive default. +_DEFAULT_MIN_CACHE_TOKENS = 1024 + +_PROMPT_CALL_LOCK = threading.Lock() + + +# Cache policy values. Mirrors the CacheClass enum in cache_registry. +class CachePolicy: + INSTRUCTION = "instruction" + CONTEXT = "context" + INSTRUCTION_PLUS_CONTEXT = "instruction+context" + NONE = "none" + + +# Cache miss reason values. Empty string means "no miss" — either the +# call had a cache read or it was served from local memory cache. +class CacheMissReason: + NONE = "" # cache read happened, or call was served locally + CACHE_DISABLED = "cache_disabled" + MODEL_UNSUPPORTED = "model_unsupported" + NO_CACHEABLE_BLOCK = "no_cacheable_block" + BELOW_1024_THRESHOLD = "below_1024_threshold" + WARMUP_OR_EVICTION = "warmup_or_eviction" + NO_CACHE_EVENT = "no_cache_event" + + +PROMPT_CALL_FIELDS = [ + "timestamp", + "contract_filename", + "usage_label", + "model_id", + # New: full resolved model id (e.g. "us.anthropic.claude-sonnet-4-5-...") + # so logs are durable when MODEL_ALIASES change. `model_id` may still be + # an alias like "sonnet_latest" for back-compat with older log readers. + "resolved_model_id", + "cache_enabled", + # New: separate cache classes — `instruction` and `context` are + # tracked independently because they have different expected + # behaviours. instruction caches read every call; context caches only + # read when the same context block repeats. Reporting must evaluate + # them separately so context variability does not make instruction + # caching look broken. + "cache_policy", + "instruction_cache_eligible", + "context_cache_eligible", + "cache_miss_reason", + "max_tokens", + "status", + "error", + "from_local_memory_cache", + "prompt_tokens_est", + "instruction_tokens_est", + "context_tokens_est", + "total_input_tokens_est", + "api_cached_input_tokens", + "api_total_billed_input_tokens", + "est_minus_api_total_tokens", + "longest_component", + "input_tokens", + "output_tokens", + "cache_creation_tokens", + "cache_read_tokens", +] + + +def _classify_cache_policy( + cache_enabled: bool, + instruction_tokens: int, + context_tokens: int, + model_supports_cache: bool, +) -> str: + """Return the actual cache_policy this call exercised on the wire. + + This is the runtime view: what `_build_claude_3_request_body` actually + placed `cache_control` on, given what the caller passed. It is NOT the + registry's intended class — comparing the two reveals wiring bugs. + """ + if not cache_enabled or not model_supports_cache: + return CachePolicy.NONE + has_instruction = instruction_tokens > 0 + has_context = context_tokens > 0 + if has_instruction and has_context: + return CachePolicy.INSTRUCTION_PLUS_CONTEXT + if has_instruction: + return CachePolicy.INSTRUCTION + if has_context: + return CachePolicy.CONTEXT + return CachePolicy.NONE + + +def _derive_cache_miss_reason( + cache_enabled: bool, + model_supports_cache: bool, + instruction_tokens: int, + context_tokens: int, + cache_read_tokens: int, + cache_creation_tokens: int, + from_local_memory_cache: bool, + min_cache_tokens: int = _DEFAULT_MIN_CACHE_TOKENS, +) -> str: + """Explain why a call did not record a cache read. + + Returns "" when a read happened (or the call was served from local + memory and thus did not hit the API at all). Otherwise returns a + short stable code from `CacheMissReason`. Reporting groups by this + column to surface root causes without manual investigation. + + `min_cache_tokens` is the model-specific Bedrock threshold below + which `cache_control` blocks are silently dropped. Sonnet 4.5 = 1024, + Haiku 4.5 = 4096, etc. + """ + if from_local_memory_cache: + # No API call happened; cache reads are not applicable. + return CacheMissReason.NONE + if cache_read_tokens > 0: + return CacheMissReason.NONE + if not cache_enabled: + return CacheMissReason.CACHE_DISABLED + if not model_supports_cache: + return CacheMissReason.MODEL_UNSUPPORTED + if instruction_tokens == 0 and context_tokens == 0: + return CacheMissReason.NO_CACHEABLE_BLOCK + + # Eligible blocks are those at or above the per-model Bedrock minimum. + instruction_eligible = instruction_tokens >= min_cache_tokens + context_eligible = context_tokens >= min_cache_tokens + if not (instruction_eligible or context_eligible): + return CacheMissReason.BELOW_1024_THRESHOLD + + if cache_creation_tokens > 0: + # Bedrock created (or refreshed) the cache on this call; a future + # call within TTL is what will actually read. + return CacheMissReason.WARMUP_OR_EVICTION + + return CacheMissReason.NO_CACHE_EVENT + + +def _estimate_tokens(text: str | None) -> int: + if not text: + return 0 + words = len(re.findall(r"\S+", text)) + return int(round(words * WORDS_TO_TOKENS_MULTIPLIER)) + + +def _get_prompt_call_log_path() -> str: + # Preferred path: same tracking directory as USAGE files. + batch_id = getattr(config, "BATCH_ID", None) or "adhoc" + file_name = f"{batch_id}-PROMPT-CALLS.csv" + run_timestamp = getattr(config, "PROMPT_CALL_RUN_TIMESTAMP", "") + if run_timestamp: + return str( + Path(config.CONSOLIDATED_OUTPUT_DIRECTORY) + / run_timestamp + / "tracking" + / file_name + ) + + # Fallback for ad-hoc invocations outside the main pipeline. + return str(Path(config.CONSOLIDATED_OUTPUT_DIRECTORY) / "tracking" / file_name) + + +def _ensure_prompt_call_log_file() -> str: + path = _get_prompt_call_log_path() + os.makedirs(os.path.dirname(path), exist_ok=True) + + if not os.path.exists(path): + with open(path, "w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=PROMPT_CALL_FIELDS) + writer.writeheader() + + return path + + +def _longest_component( + prompt_tokens: int, + instruction_tokens: int, + context_tokens: int, +) -> str: + components = { + "field_instructions": prompt_tokens, + "prompt_instructions": instruction_tokens, + "context": context_tokens, + } + if all(v == 0 for v in components.values()): + return "unknown" + return max(components, key=components.get) + + +def log_prompt_call( + contract_filename: str, + usage_label: str, + model_id: str, + cache_enabled: bool, + max_tokens: int, + prompt: str | None, + instruction: str | None, + context_for_caching: str | None, + status: str, + error: str = "", + from_local_memory_cache: bool = False, + input_tokens: int = 0, + output_tokens: int = 0, + cache_creation_tokens: int = 0, + cache_read_tokens: int = 0, + resolved_model_id: str | None = None, + model_supports_cache: bool | None = None, +) -> None: + """Write one prompt-call tracking row. + + This function never raises to avoid impacting the extraction pipeline. + + Args: + resolved_model_id: Full model id after `config.resolve_model_id`. + When None, falls back to `model_id` so older callers still log + something durable. + model_supports_cache: Whether the resolved model supports prompt + caching at all. When None, the function imports + `_supports_prompt_cache` lazily and computes it. Pass it + explicitly from the call site when available to avoid the + import. + """ + if not getattr(config, "ENABLE_PROMPT_CALL_LOGGING", True): + return + + try: + prompt = prompt or "" + instruction = instruction or "" + context_for_caching = context_for_caching or "" + + prompt_tokens_est = _estimate_tokens(prompt) + instruction_tokens_est = _estimate_tokens(instruction) + context_tokens_est = _estimate_tokens(context_for_caching) + + total_input_tokens_est = ( + prompt_tokens_est + instruction_tokens_est + context_tokens_est + ) + api_cached_input_tokens = cache_creation_tokens + cache_read_tokens + api_total_billed_input_tokens = input_tokens + api_cached_input_tokens + + # Resolve model id, cache support, and per-model min threshold if + # the caller didn't pass them. + if resolved_model_id is None: + resolved_model_id = config.resolve_model_id(model_id) + + min_cache_tokens = _DEFAULT_MIN_CACHE_TOKENS + try: + from src.utils.llm_utils import ( + _min_cache_tokens, + _supports_prompt_cache, + ) + + if model_supports_cache is None: + model_supports_cache = _supports_prompt_cache(resolved_model_id) + min_cache_tokens = _min_cache_tokens(resolved_model_id) + except Exception: + if model_supports_cache is None: + model_supports_cache = False + + cache_policy = _classify_cache_policy( + cache_enabled, + instruction_tokens_est, + context_tokens_est, + model_supports_cache, + ) + instruction_cache_eligible = ( + cache_enabled + and model_supports_cache + and instruction_tokens_est >= min_cache_tokens + ) + context_cache_eligible = ( + cache_enabled + and model_supports_cache + and context_tokens_est >= min_cache_tokens + ) + cache_miss_reason = _derive_cache_miss_reason( + cache_enabled=cache_enabled, + model_supports_cache=model_supports_cache, + instruction_tokens=instruction_tokens_est, + context_tokens=context_tokens_est, + cache_read_tokens=cache_read_tokens, + cache_creation_tokens=cache_creation_tokens, + from_local_memory_cache=from_local_memory_cache, + min_cache_tokens=min_cache_tokens, + ) + + row = { + "timestamp": datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"), + "contract_filename": contract_filename, + "usage_label": usage_label, + "model_id": model_id, + "resolved_model_id": resolved_model_id, + "cache_enabled": cache_enabled, + "cache_policy": cache_policy, + "instruction_cache_eligible": instruction_cache_eligible, + "context_cache_eligible": context_cache_eligible, + "cache_miss_reason": cache_miss_reason, + "max_tokens": max_tokens, + "status": status, + "error": error, + "from_local_memory_cache": from_local_memory_cache, + "prompt_tokens_est": prompt_tokens_est, + "instruction_tokens_est": instruction_tokens_est, + "context_tokens_est": context_tokens_est, + "total_input_tokens_est": total_input_tokens_est, + "api_cached_input_tokens": api_cached_input_tokens, + "api_total_billed_input_tokens": api_total_billed_input_tokens, + "est_minus_api_total_tokens": total_input_tokens_est + - api_total_billed_input_tokens, + "longest_component": _longest_component( + prompt_tokens_est, + instruction_tokens_est, + context_tokens_est, + ), + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "cache_creation_tokens": cache_creation_tokens, + "cache_read_tokens": cache_read_tokens, + } + + with _PROMPT_CALL_LOCK: + file_path = _ensure_prompt_call_log_file() + with open(file_path, "a", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=PROMPT_CALL_FIELDS) + writer.writerow(row) + except Exception as e: + # Tracking must not break the pipeline. + logging.error( + f"Failed to log prompt call for {contract_filename} / {usage_label}: {e}" + ) + return