Merged in refactor/split-investment-and-client (pull request #347)
Refactor/split investment and client * refactor: clean up tests * Renamed tests/qcqa to tests/qa_qc for consistency * comment out faulty test - see docstring note at top * split to client/investment * add __init__.py to investment and client to avoid mypy confusion * fix imports * update imports for consistency across investment module * move back to one config * try changing import to relative * add init.py to src * try relative import * try one more sys.path.append * fix path for client main * fix imports in file_processing * make prompts common * move keywords out to `src` * move smart_chunking_funcs to `src` * fix mypy error * start moving to explicit imports * remove old keywords and fix imports for keywords and prompts * change all imports to full paths * fix unit tests * add readme for new way of running things * isort after all that import work * remove unused sys.path.append from some tests Approved-by: Katon Minhas
This commit is contained in:
Alex Galarce
@@ -1 +1,50 @@
|
||||
# Field Extraction
|
||||
# Field Extraction
|
||||
|
||||
## Running the Code
|
||||
|
||||
This project uses Python's module system for imports. To run the code properly:
|
||||
|
||||
1. **DO NOT** run files directly like this:
|
||||
```bash
|
||||
python src/client/main.py # ❌ This won't work
|
||||
```
|
||||
|
||||
2. **INSTEAD**, use Python's module flag (`-m`) from the project root:
|
||||
```bash
|
||||
python -m src.client.main # ✅ This is correct
|
||||
```
|
||||
|
||||
## Why?
|
||||
The code uses absolute imports (e.g., `from src.utils import llm_utils`) to maintain a clear and consistent package structure. Running with `python -m` ensures Python can properly resolve these imports.
|
||||
|
||||
## Development
|
||||
- All imports should use the `src.` prefix (e.g., `from src.utils import llm_utils`)
|
||||
- Always run code from the project root directory using the `-m` flag
|
||||
- Tests are configured to handle these imports automatically via pytest settings in pyproject.toml
|
||||
|
||||
## Configuration
|
||||
The project uses poetry for dependency management and pytest for testing. Key configurations in pyproject.toml:
|
||||
|
||||
```toml
|
||||
[tool.pytest.ini_options]
|
||||
pythonpath=["."]
|
||||
testpaths = [
|
||||
"tests"
|
||||
]
|
||||
```
|
||||
|
||||
## Dependencies
|
||||
To install dependencies:
|
||||
```bash
|
||||
poetry install
|
||||
```
|
||||
|
||||
To run tests:
|
||||
```bash
|
||||
poetry run pytest
|
||||
```
|
||||
|
||||
To run type checking:
|
||||
```bash
|
||||
poetry run mypy .
|
||||
```
|
||||
@@ -2,7 +2,6 @@ import json
|
||||
|
||||
import pandas as pd
|
||||
import pytest
|
||||
|
||||
from crosswalk_utils import CrosswalkBuilder, apply_crosswalk
|
||||
|
||||
|
||||
|
||||
@@ -39,7 +39,7 @@ disable_error_code = ["import-untyped","assignment","name-defined","call-arg","v
|
||||
exclude = ["scripts/"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
pythonpath=["src"]
|
||||
pythonpath=["."]
|
||||
testpaths = [
|
||||
"tests"
|
||||
]
|
||||
|
||||
+8
-9
@@ -1,6 +1,5 @@
|
||||
import claude_funcs
|
||||
import config
|
||||
import prompts
|
||||
import src.utils.llm_utils as llm_utils
|
||||
from src import config, prompts
|
||||
|
||||
|
||||
def run_conditional(combined_results, text_dict, filename):
|
||||
@@ -14,14 +13,14 @@ def run_conditional(combined_results, text_dict, filename):
|
||||
# LOB/Program Check
|
||||
if "," in d["CONTRACT_LOB"]:
|
||||
prompt = prompts.CONDITIONAL_LOB_CHECK(d, text_dict[page_num])
|
||||
lob_answer = claude_funcs.invoke_claude(
|
||||
lob_answer = llm_utils.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, max_tokens=128
|
||||
)
|
||||
d["CONTRACT_LOB"] = lob_answer
|
||||
|
||||
# Prov Type Level 2
|
||||
prompt = prompts.CONDITIONAL_PROV_2(d, text_dict[page_num])
|
||||
prov2_answer = claude_funcs.invoke_claude(
|
||||
prov2_answer = llm_utils.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, max_tokens=128
|
||||
)
|
||||
d["PROV_TYPE_LEVEL_2"] = prov2_answer.strip()
|
||||
@@ -29,7 +28,7 @@ def run_conditional(combined_results, text_dict, filename):
|
||||
# # Inclusion of essential RBRVS "Fee Source" Language (Y/N) - Top Down approach to be used for non-NY
|
||||
if "NY" not in config.STATE:
|
||||
prompt = prompts.CONDITIONAL_RBRVS(text_dict[page_num])
|
||||
rbrvs_answer = claude_funcs.invoke_claude(
|
||||
rbrvs_answer = llm_utils.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE2, filename, max_tokens=16
|
||||
)
|
||||
|
||||
@@ -48,7 +47,7 @@ def run_conditional(combined_results, text_dict, filename):
|
||||
prompt = prompts.CONDITIONAL_IP_OP_SMALL(
|
||||
d["FULL_SERVICE"], d["FULL_METHODOLOGY"]
|
||||
)
|
||||
ip_op_answer = claude_funcs.invoke_claude(
|
||||
ip_op_answer = llm_utils.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, max_tokens=16
|
||||
)
|
||||
if ip_op_answer.strip() in ["IP", "OP"]:
|
||||
@@ -62,7 +61,7 @@ def run_conditional(combined_results, text_dict, filename):
|
||||
d["IP - DSH/IME/UC, included (Y/N)"] = dsh_dict[page_num]
|
||||
else:
|
||||
prompt = prompts.CONDITIONAL_DSH_IME_UC(text_dict[page_num])
|
||||
dsh_answer = claude_funcs.invoke_claude(
|
||||
dsh_answer = llm_utils.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE2, filename, max_tokens=16
|
||||
)
|
||||
d["IP - DSH/IME/UC, included (Y/N)"] = dsh_answer.strip()
|
||||
@@ -79,7 +78,7 @@ def run_conditional(combined_results, text_dict, filename):
|
||||
]
|
||||
):
|
||||
prompt = prompts.CONDITIONAL_STOP_LOSS(text_dict[page_num])
|
||||
sl_answer = claude_funcs.invoke_claude(
|
||||
sl_answer = llm_utils.invoke_claude(
|
||||
prompt,
|
||||
config.MODEL_ID_CLAUDE3_HAIKU,
|
||||
filename,
|
||||
+6
-7
@@ -3,13 +3,12 @@ import os
|
||||
|
||||
import pandas as pd
|
||||
|
||||
import conditional_funcs
|
||||
import config
|
||||
import one_to_n_funcs
|
||||
import postprocess
|
||||
import preprocess
|
||||
import smart_chunking_funcs
|
||||
from regex_funcs import irs_hotfix, npi_hotfix
|
||||
import src.client.conditional_funcs as conditional_funcs
|
||||
import src.client.one_to_n_funcs as one_to_n_funcs
|
||||
import src.client.postprocess as postprocess
|
||||
import src.client.preprocess as preprocess
|
||||
from src import config, smart_chunking_funcs
|
||||
from src.regex.regex_utils import irs_hotfix, npi_hotfix
|
||||
|
||||
|
||||
def run_ac_prompts(file_object):
|
||||
@@ -7,21 +7,22 @@ from datetime import datetime
|
||||
|
||||
random.seed(42)
|
||||
|
||||
import config
|
||||
import consolidate_output
|
||||
import file_processing
|
||||
import io_utils
|
||||
import qa_qc_helpers
|
||||
import string_funcs
|
||||
import tracking
|
||||
from batch_tracking import BatchTracker
|
||||
import src.investment.file_processing as file_processing
|
||||
import src.qa_qc.qa_qc_utils as qa_qc_utils
|
||||
import src.tracking.tracking_utils as tracking_utils
|
||||
import src.utils.io_utils as io_utils
|
||||
import src.utils.string_utils as string_utils
|
||||
# import sys
|
||||
# sys.path.append("src")
|
||||
from src import config, consolidate_output
|
||||
from src.tracking.batch_tracking import BatchTracker
|
||||
|
||||
|
||||
def process_b(item):
|
||||
filename, contract_text = item
|
||||
start_time = datetime.now()
|
||||
|
||||
if string_funcs.contains_reimbursement(contract_text, 0):
|
||||
if string_utils.contains_reimbursement(contract_text, 0):
|
||||
try:
|
||||
file_processing.run_b_prompts(item)
|
||||
end_time = datetime.now()
|
||||
@@ -31,7 +32,7 @@ def process_b(item):
|
||||
|
||||
if os.path.exists(output_file):
|
||||
batch_tracker.update_file_progress(filename, 'B', start_time, end_time)
|
||||
tracking.log_file_processing(
|
||||
tracking_utils.log_file_processing(
|
||||
filename,
|
||||
'B',
|
||||
f"{config.BATCH_ID}/processing/{filename}",
|
||||
@@ -44,7 +45,7 @@ def process_b(item):
|
||||
filename, 'B', start_time, datetime.now(),
|
||||
status='Failed', error=error_msg
|
||||
)
|
||||
tracking.log_file_processing(
|
||||
tracking_utils.log_file_processing(
|
||||
filename,
|
||||
'B',
|
||||
f"{config.BATCH_ID}/processing/{filename}",
|
||||
@@ -61,7 +62,7 @@ def process_b(item):
|
||||
filename, 'B', start_time, datetime.now(),
|
||||
status='Failed', error=error_msg
|
||||
)
|
||||
tracking.log_file_processing(
|
||||
tracking_utils.log_file_processing(
|
||||
filename,
|
||||
'B',
|
||||
f"{config.BATCH_ID}/processing/{filename}",
|
||||
@@ -75,7 +76,7 @@ def process_b(item):
|
||||
filename, 'B', datetime.now(), datetime.now(),
|
||||
status='Skipped', error='No reimbursement information found'
|
||||
)
|
||||
tracking.log_file_processing(
|
||||
tracking_utils.log_file_processing(
|
||||
filename,
|
||||
'B',
|
||||
f"{config.BATCH_ID}/processing/{filename}",
|
||||
@@ -97,7 +98,7 @@ def process_ac(item):
|
||||
|
||||
if os.path.exists(output_file):
|
||||
batch_tracker.update_file_progress(filename, 'AC', start_time, end_time)
|
||||
tracking.log_file_processing(
|
||||
tracking_utils.log_file_processing(
|
||||
filename,
|
||||
'AC',
|
||||
f"{config.BATCH_ID}/processing/{filename}",
|
||||
@@ -110,7 +111,7 @@ def process_ac(item):
|
||||
filename, 'AC', start_time, datetime.now(),
|
||||
status='Failed', error=error_msg
|
||||
)
|
||||
tracking.log_file_processing(
|
||||
tracking_utils.log_file_processing(
|
||||
filename,
|
||||
'AC',
|
||||
f"{config.BATCH_ID}/processing/{filename}",
|
||||
@@ -126,7 +127,7 @@ def process_ac(item):
|
||||
filename, 'AC', start_time, datetime.now(),
|
||||
status='Failed', error=error_msg
|
||||
)
|
||||
tracking.log_file_processing(
|
||||
tracking_utils.log_file_processing(
|
||||
filename,
|
||||
'AC',
|
||||
f"{config.BATCH_ID}/processing/{filename}",
|
||||
@@ -155,7 +156,7 @@ def process_results(result, filename, input_dict_b, stats):
|
||||
stats['completed_ac'] += 1
|
||||
stats['processed_files'].setdefault(proc_type, set()).add(filename)
|
||||
|
||||
tracking.log_file_processing(
|
||||
tracking_utils.log_file_processing(
|
||||
filename,
|
||||
proc_type,
|
||||
f"{config.BATCH_ID}/processing/{filename}",
|
||||
@@ -172,7 +173,7 @@ def process_remaining_files(input_dict, input_dict_b, run_timestamp, stats):
|
||||
print("\nEntering self-repair mode...")
|
||||
|
||||
|
||||
completed_ac, completed_b = tracking.get_processed_files(config.BATCH_ID)
|
||||
completed_ac, completed_b = tracking_utils.get_processed_files(config.BATCH_ID)
|
||||
|
||||
|
||||
stats['processed_files']['AC'] = completed_ac
|
||||
@@ -231,7 +232,7 @@ def main():
|
||||
# Filter B - files with reimbursement
|
||||
input_dict_b = {
|
||||
k: v for k, v in input_dict.items()
|
||||
if string_funcs.contains_reimbursement(str(v))
|
||||
if string_utils.contains_reimbursement(str(v))
|
||||
}
|
||||
|
||||
batch_tracker.set_input_file_counts(
|
||||
@@ -241,8 +242,8 @@ def main():
|
||||
batch_tracker.add_batch_run()
|
||||
|
||||
if config.FILTER_ALREADY_PROCESSED:
|
||||
# Get processed files using new tracking function
|
||||
processed_ac, processed_b = tracking.get_processed_files(config.BATCH_ID)
|
||||
# Get processed files using new tracking_utils function
|
||||
processed_ac, processed_b = tracking_utils.get_processed_files(config.BATCH_ID)
|
||||
|
||||
# Filter AC files
|
||||
input_dict_ac = {
|
||||
@@ -310,7 +311,7 @@ def main():
|
||||
stats = process_results(result, filename, input_dict_b, stats)
|
||||
|
||||
# Calculate and display progress
|
||||
ac_progress, b_progress = tracking.calculate_progress_stats(
|
||||
ac_progress, b_progress = tracking_utils.calculate_progress_stats(
|
||||
stats['completed_ac'], total_ac,
|
||||
stats['completed_b'], total_b
|
||||
)
|
||||
@@ -335,8 +336,8 @@ def main():
|
||||
print("Starting output consolidation and QA/QC...")
|
||||
try:
|
||||
ac_final_df, b_final_df, abc_final_df = consolidate_output.consolidate_output(run_timestamp)
|
||||
wb = qa_qc_helpers.perform_qc_qa(ac_final_df, b_final_df, abc_final_df, input_dict)
|
||||
qa_qc_helpers.save_qcqa_wb(wb, run_timestamp)
|
||||
wb = qa_qc_utils.perform_qc_qa(ac_final_df, b_final_df, abc_final_df, input_dict)
|
||||
qa_qc_utils.save_qcqa_wb(wb, run_timestamp)
|
||||
print("Consolidation and QA/QC complete")
|
||||
except Exception as e:
|
||||
print(f"Error during consolidation: {str(e)}")
|
||||
+15
-17
@@ -2,12 +2,10 @@
|
||||
|
||||
import re
|
||||
|
||||
import claude_funcs
|
||||
import config
|
||||
import postprocessing_funcs
|
||||
import prompts
|
||||
import string_funcs
|
||||
import valid
|
||||
import src.constants.valid as valid
|
||||
import src.utils.llm_utils as llm_utils
|
||||
import src.utils.string_utils as string_utils
|
||||
from src import config, postprocessing_funcs, prompts
|
||||
|
||||
|
||||
def run_bottom_up_primary(
|
||||
@@ -33,17 +31,17 @@ def run_bottom_up_primary(
|
||||
# if '%' in chunk_dict[page_number] or '$' in chunk_dict[page_number]:
|
||||
# prompt = prompts.BOTTOM_UP_PRIMARY(chunk_dict[page_number], config.CLIENT_NAME)
|
||||
for page_number in text_dict.keys():
|
||||
if string_funcs.contains_reimbursement(text_dict, page_number):
|
||||
if string_utils.contains_reimbursement(text_dict, page_number):
|
||||
# Run Primary
|
||||
prompt = prompts.BOTTOM_UP_PRIMARY(
|
||||
text_dict[page_number], config.CLIENT_NAME
|
||||
)
|
||||
answer = claude_funcs.invoke_claude(
|
||||
answer = llm_utils.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, filename
|
||||
)
|
||||
answer_strings[page_number] = answer
|
||||
|
||||
answer_dicts = string_funcs.primary_string_to_dict(answer_strings, filename)
|
||||
answer_dicts = string_utils.primary_string_to_dict(answer_strings, filename)
|
||||
|
||||
answer_dicts = postprocessing_funcs.consolidate_subheader(answer_dicts)
|
||||
answer_dicts = postprocessing_funcs.filter_service_column(
|
||||
@@ -89,10 +87,10 @@ def run_bottom_up_secondary(
|
||||
|
||||
# if '%' in d['FULL_METHODOLOGY'] or '$' in d['FULL_METHODOLOGY']:
|
||||
prompt = prompts.BOTTOM_UP_METHODOLOGY_BREAKOUT(d)
|
||||
answer = claude_funcs.invoke_claude(
|
||||
answer = llm_utils.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, max_tokens=512
|
||||
)
|
||||
answer_dict = string_funcs.secondary_string_to_dict(answer, filename)
|
||||
answer_dict = string_utils.secondary_string_to_dict(answer, filename)
|
||||
d.update(answer_dict)
|
||||
|
||||
# Ensure all bottom up methodology keys are present
|
||||
@@ -117,10 +115,10 @@ def run_bottom_up_secondary(
|
||||
|
||||
# Escalator
|
||||
prompt = prompts.BOTTOM_UP_ESCALATOR(d, text_dict[page_num])
|
||||
escalator_answer = claude_funcs.invoke_claude(
|
||||
escalator_answer = llm_utils.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, max_tokens=1024
|
||||
)
|
||||
escalator_dict = string_funcs.secondary_string_to_dict(
|
||||
escalator_dict = string_utils.secondary_string_to_dict(
|
||||
escalator_answer, filename
|
||||
)
|
||||
d.update(escalator_dict)
|
||||
@@ -188,10 +186,10 @@ def run_top_down_primary(text_dict, td_pages, filename):
|
||||
page_text = "\n".join([text_dict[page_num] for page_num in td_pages])
|
||||
|
||||
prompt = prompts.TOP_DOWN_PRIMARY(page_text)
|
||||
answer = claude_funcs.invoke_claude(
|
||||
answer = llm_utils.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, max_tokens=4096
|
||||
)
|
||||
answer_dict = string_funcs.secondary_string_to_dict(answer, filename)
|
||||
answer_dict = string_utils.secondary_string_to_dict(answer, filename)
|
||||
for field in prompts.TD_PRIMARY_FIELDS:
|
||||
if field not in answer_dict:
|
||||
answer_dict[field] = "N/A"
|
||||
@@ -219,7 +217,7 @@ def get_td_dict(text_dict, filename, PROMPT_FUNC, VALID_VALUES):
|
||||
and page_num.isdigit()
|
||||
):
|
||||
prompt = PROMPT_FUNC(page_text)
|
||||
answer = claude_funcs.invoke_claude(
|
||||
answer = llm_utils.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 1024
|
||||
)
|
||||
td_dict[page_num] = answer
|
||||
@@ -253,7 +251,7 @@ def add_medically_necessary(merged_dicts, medically_necessary_dict):
|
||||
d["MEDICAL_NECESSITY_LANGUAGE"] = medically_necessary_dict[
|
||||
str(closest_page)
|
||||
]
|
||||
if not string_funcs.is_empty(d["MEDICAL_NECESSITY_LANGUAGE"]):
|
||||
if not string_utils.is_empty(d["MEDICAL_NECESSITY_LANGUAGE"]):
|
||||
d["MEDICAL_NECESSITY_LANGUAGE_IND"] = "Y"
|
||||
else:
|
||||
d["MEDICAL_NECESSITY_LANGUAGE_IND"] = "N"
|
||||
@@ -1,5 +1,5 @@
|
||||
import postprocessing_funcs
|
||||
import valid
|
||||
import src.constants.valid as valid
|
||||
from src import postprocessing_funcs
|
||||
|
||||
|
||||
def b_postprocess(filename, df, pages):
|
||||
@@ -1,10 +1,7 @@
|
||||
import csv
|
||||
|
||||
import config
|
||||
import keywords
|
||||
import one_to_n_funcs
|
||||
import preprocessing_funcs
|
||||
import table_funcs
|
||||
import src.utils.table_utils as table_utils
|
||||
from src import config, keywords, preprocessing_funcs
|
||||
|
||||
|
||||
def preprocess(contract_text, filename, fields=config.FIELDS):
|
||||
@@ -24,7 +21,7 @@ def preprocess(contract_text, filename, fields=config.FIELDS):
|
||||
exhibit_pages = preprocessing_funcs.get_exhibit_pages(
|
||||
text_dict, filename
|
||||
) # All pages with exhibit headers
|
||||
text_dict = table_funcs.align_and_format_tables(text_dict, filename)
|
||||
text_dict = table_utils.align_and_format_tables(text_dict, filename)
|
||||
text_dict = preprocessing_funcs.chunk_consecutive(text_dict, exhibit_pages)
|
||||
# write text_dict to a csv file:
|
||||
with open("text_dict.csv", "w") as f:
|
||||
@@ -3,13 +3,11 @@ from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
|
||||
import config
|
||||
import costlog_funcs
|
||||
import tracking
|
||||
import valid
|
||||
from qa_qc_helpers import (perform_qc_qa, perform_qc_qa_tests_ac,
|
||||
perform_qc_qa_tests_b, perform_qc_qa_tests_both,
|
||||
save_qcqa_wb, table_stats_qaqc)
|
||||
import src.constants.valid as valid
|
||||
import src.tracking.costlog_utils as costlog_utils
|
||||
import src.tracking.tracking_utils as tracking_utils
|
||||
from src import config
|
||||
from src.qa_qc.qa_qc_utils import perform_qc_qa
|
||||
|
||||
|
||||
def upload_tracking_files(run_timestamp):
|
||||
@@ -31,7 +29,7 @@ def upload_tracking_files(run_timestamp):
|
||||
# Read and aggregate costlog
|
||||
if os.path.exists(config.COST_LOG_NAME):
|
||||
df_costlog = pd.read_csv(config.COST_LOG_NAME)
|
||||
df_caller, df_filename = costlog_funcs.aggregate_costlog(df_costlog)
|
||||
df_caller, df_filename = costlog_utils.aggregate_costlog(df_costlog)
|
||||
|
||||
# Write aggregated costlogs
|
||||
df_caller.to_csv(costlog_caller_fn, index=False)
|
||||
@@ -189,8 +187,8 @@ def upload_individual_outputs(run_timestamp):
|
||||
filename = basename + ".txt"
|
||||
|
||||
# If already fully processed and uploaded, skip
|
||||
if tracking.check_file_processed(config.BATCH_ID, filename, 'AC') and \
|
||||
tracking.check_file_processed(config.BATCH_ID, filename, 'B'):
|
||||
if tracking_utils.check_file_processed(config.BATCH_ID, filename, 'AC') and \
|
||||
tracking_utils.check_file_processed(config.BATCH_ID, filename, 'B'):
|
||||
print(f"Skipping {filename} - already processed")
|
||||
continue
|
||||
|
||||
@@ -202,8 +200,8 @@ def upload_individual_outputs(run_timestamp):
|
||||
s3_key = f"{individual_folder}/{config.AC_RESULTS_NAME}"
|
||||
try:
|
||||
config.S3_CLIENT.upload_file(ac_path, BUCKET_NAME, s3_key)
|
||||
tracking.log_file_processing(filename, 'AC', s3_key, 'Completed')
|
||||
tracking.update_master_batch_tracking(
|
||||
tracking_utils.log_file_processing(filename, 'AC', s3_key, 'Completed')
|
||||
tracking_utils.update_master_batch_tracking(
|
||||
config.BATCH_ID,
|
||||
run_timestamp,
|
||||
[filename],
|
||||
@@ -214,7 +212,7 @@ def upload_individual_outputs(run_timestamp):
|
||||
except Exception as e:
|
||||
error_msg = str(e)
|
||||
print(f"Error uploading AC file {basename}: {error_msg}")
|
||||
tracking.log_file_processing(filename, 'AC', s3_key, 'Failed', error_msg)
|
||||
tracking_utils.log_file_processing(filename, 'AC', s3_key, 'Failed', error_msg)
|
||||
|
||||
# Check for B results
|
||||
b_path = os.path.join(subdir, config.B_RESULTS_NAME)
|
||||
@@ -222,8 +220,8 @@ def upload_individual_outputs(run_timestamp):
|
||||
s3_key = f"{individual_folder}/{config.B_RESULTS_NAME}"
|
||||
try:
|
||||
config.S3_CLIENT.upload_file(b_path, BUCKET_NAME, s3_key)
|
||||
tracking.log_file_processing(filename, 'B', s3_key, 'Completed')
|
||||
tracking.update_master_batch_tracking(
|
||||
tracking_utils.log_file_processing(filename, 'B', s3_key, 'Completed')
|
||||
tracking_utils.update_master_batch_tracking(
|
||||
config.BATCH_ID,
|
||||
run_timestamp,
|
||||
[filename],
|
||||
@@ -234,7 +232,7 @@ def upload_individual_outputs(run_timestamp):
|
||||
except Exception as e:
|
||||
error_msg = str(e)
|
||||
print(f"Error uploading B file {basename}: {error_msg}")
|
||||
tracking.log_file_processing(filename, 'B', s3_key, 'Failed', error_msg)
|
||||
tracking_utils.log_file_processing(filename, 'B', s3_key, 'Failed', error_msg)
|
||||
|
||||
def consolidate_output(run_timestamp, ac_df = None, b_df = None):
|
||||
"""Consolidate and upload outputs"""
|
||||
@@ -276,8 +274,8 @@ def consolidate_output(run_timestamp, ac_df = None, b_df = None):
|
||||
s3_key = f"{config.BATCH_ID}/{run_timestamp}/consolidated/{config.BATCH_ID}-AC.csv"
|
||||
try:
|
||||
config.S3_CLIENT.upload_file(output_path, config.S3_OUTPUT_BUCKET, s3_key)
|
||||
tracking.log_file_processing(f"{config.BATCH_ID}-AC.csv", 'consolidated', s3_key, 'Completed')
|
||||
tracking.update_master_batch_tracking(
|
||||
tracking_utils.log_file_processing(f"{config.BATCH_ID}-AC.csv", 'consolidated', s3_key, 'Completed')
|
||||
tracking_utils.update_master_batch_tracking(
|
||||
config.BATCH_ID,
|
||||
run_timestamp,
|
||||
[f"{config.BATCH_ID}-AC.csv"],
|
||||
@@ -287,7 +285,7 @@ def consolidate_output(run_timestamp, ac_df = None, b_df = None):
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"Error uploading consolidated AC file: {str(e)}")
|
||||
tracking.log_file_processing(f"{config.BATCH_ID}-AC.csv", 'consolidated', s3_key, 'Failed', str(e))
|
||||
tracking_utils.log_file_processing(f"{config.BATCH_ID}-AC.csv", 'consolidated', s3_key, 'Failed', str(e))
|
||||
|
||||
# If B only
|
||||
if "b" in config.FIELDS and b_final_df is None:
|
||||
@@ -314,8 +312,8 @@ def consolidate_output(run_timestamp, ac_df = None, b_df = None):
|
||||
s3_key = f"{config.BATCH_ID}/{run_timestamp}/consolidated/{config.BATCH_ID}-B.csv"
|
||||
try:
|
||||
config.S3_CLIENT.upload_file(output_path, config.S3_OUTPUT_BUCKET, s3_key)
|
||||
tracking.log_file_processing(f"{config.BATCH_ID}-B.csv", 'consolidated', s3_key, 'Completed')
|
||||
tracking.update_master_batch_tracking(
|
||||
tracking_utils.log_file_processing(f"{config.BATCH_ID}-B.csv", 'consolidated', s3_key, 'Completed')
|
||||
tracking_utils.update_master_batch_tracking(
|
||||
config.BATCH_ID,
|
||||
run_timestamp,
|
||||
[f"{config.BATCH_ID}-B.csv"],
|
||||
@@ -325,7 +323,7 @@ def consolidate_output(run_timestamp, ac_df = None, b_df = None):
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"Error uploading consolidated B file: {str(e)}")
|
||||
tracking.log_file_processing(f"{config.BATCH_ID}-B.csv", 'consolidated', s3_key, 'Failed', str(e))
|
||||
tracking_utils.log_file_processing(f"{config.BATCH_ID}-B.csv", 'consolidated', s3_key, 'Failed', str(e))
|
||||
|
||||
# If ABC
|
||||
if "a" in config.FIELDS and "c" in config.FIELDS and "b" in config.FIELDS:
|
||||
@@ -354,11 +352,11 @@ def consolidate_output(run_timestamp, ac_df = None, b_df = None):
|
||||
s3_key = f"{config.BATCH_ID}/{run_timestamp}/consolidated/{config.BATCH_ID}-ABC.csv"
|
||||
try:
|
||||
config.S3_CLIENT.upload_file(abc_path, config.S3_OUTPUT_BUCKET, s3_key)
|
||||
tracking.log_file_processing(f"{config.BATCH_ID}-ABC.csv", 'consolidated', s3_key, 'Completed')
|
||||
tracking_utils.log_file_processing(f"{config.BATCH_ID}-ABC.csv", 'consolidated', s3_key, 'Completed')
|
||||
except Exception as e:
|
||||
error_msg = str(e)
|
||||
print(f"Error uploading ABC file: {error_msg}")
|
||||
tracking.log_file_processing(f"{config.BATCH_ID}-ABC.csv", 'consolidated', s3_key, 'Failed', error_msg)
|
||||
tracking_utils.log_file_processing(f"{config.BATCH_ID}-ABC.csv", 'consolidated', s3_key, 'Failed', error_msg)
|
||||
except Exception as e:
|
||||
print(f"Error creating ABC consolidated file: {str(e)}")
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import pandas as pd
|
||||
|
||||
import config
|
||||
from src import config
|
||||
|
||||
VALID_LOBS = [
|
||||
"MEDICARE",
|
||||
@@ -0,0 +1,104 @@
|
||||
import src.utils.llm_utils as llm_utils
|
||||
from src import config, prompts
|
||||
|
||||
|
||||
def run_conditional(combined_results, text_dict, filename):
|
||||
|
||||
dsh_dict, rbrvs_dict, sl_dict = {}, {}, {}
|
||||
|
||||
for d in combined_results:
|
||||
if d is not None:
|
||||
page_num = d["page_num"]
|
||||
|
||||
# LOB/Program Check
|
||||
if "," in d["CONTRACT_LOB"]:
|
||||
prompt = prompts.CONDITIONAL_LOB_CHECK(d, text_dict[page_num])
|
||||
lob_answer = llm_utils.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, max_tokens=128
|
||||
)
|
||||
d["CONTRACT_LOB"] = lob_answer
|
||||
|
||||
# Prov Type Level 2
|
||||
prompt = prompts.CONDITIONAL_PROV_2(d, text_dict[page_num])
|
||||
prov2_answer = llm_utils.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, max_tokens=128
|
||||
)
|
||||
d["PROV_TYPE_LEVEL_2"] = prov2_answer.strip()
|
||||
|
||||
# # Inclusion of essential RBRVS "Fee Source" Language (Y/N) - Top Down approach to be used for non-NY
|
||||
if "NY" not in config.STATE:
|
||||
prompt = prompts.CONDITIONAL_RBRVS(text_dict[page_num])
|
||||
rbrvs_answer = llm_utils.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE2, filename, max_tokens=16
|
||||
)
|
||||
|
||||
d['Inclusion of essential RBRVS "Fee Source" Language (Y/N)'] = (
|
||||
rbrvs_answer.strip()
|
||||
)
|
||||
rbrvs_dict[page_num] = rbrvs_answer
|
||||
else:
|
||||
if "RBRVS" in d["FULL_METHODOLOGY"]:
|
||||
d['Inclusion of essential RBRVS "Fee Source" Language (Y/N)'] = "Y"
|
||||
else:
|
||||
d['Inclusion of essential RBRVS "Fee Source" Language (Y/N)'] = "N"
|
||||
|
||||
# IP_OP
|
||||
if "FACILITY" in d["PROV_TYPE"].upper():
|
||||
prompt = prompts.CONDITIONAL_IP_OP_SMALL(
|
||||
d["FULL_SERVICE"], d["FULL_METHODOLOGY"]
|
||||
)
|
||||
ip_op_answer = llm_utils.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, max_tokens=16
|
||||
)
|
||||
if ip_op_answer.strip() in ["IP", "OP"]:
|
||||
d["IP_OP"] = ip_op_answer.strip()
|
||||
else:
|
||||
d["IP_OP"] = "N/A"
|
||||
|
||||
if "IP" in ip_op_answer:
|
||||
# IP - DSH/IME/UC, included (Y/N)
|
||||
if page_num in dsh_dict.keys():
|
||||
d["IP - DSH/IME/UC, included (Y/N)"] = dsh_dict[page_num]
|
||||
else:
|
||||
prompt = prompts.CONDITIONAL_DSH_IME_UC(text_dict[page_num])
|
||||
dsh_answer = llm_utils.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE2, filename, max_tokens=16
|
||||
)
|
||||
d["IP - DSH/IME/UC, included (Y/N)"] = dsh_answer.strip()
|
||||
dsh_dict[page_num] = dsh_answer
|
||||
|
||||
# Stop-Loss Catastrophic Threshold
|
||||
if page_num in sl_dict.keys():
|
||||
d["IP - Stoploss Catastrophic Threshold"] = sl_dict[page_num]
|
||||
else:
|
||||
if any(
|
||||
[
|
||||
i.upper() in text_dict[page_num].upper()
|
||||
for i in ["stop loss", "stop-loss", "stoploss"]
|
||||
]
|
||||
):
|
||||
prompt = prompts.CONDITIONAL_STOP_LOSS(text_dict[page_num])
|
||||
sl_answer = llm_utils.invoke_claude(
|
||||
prompt,
|
||||
config.MODEL_ID_CLAUDE3_HAIKU,
|
||||
filename,
|
||||
max_tokens=256,
|
||||
)
|
||||
else:
|
||||
sl_answer = "N/A"
|
||||
d["IP - Stoploss Catastrophic Threshold"] = sl_answer
|
||||
sl_dict[page_num] = sl_answer
|
||||
else:
|
||||
d["IP - Stoploss Catastrophic Threshold"] = "N/A"
|
||||
d["IP - DSH/IME/UC, included (Y/N)"] = "N/A"
|
||||
else:
|
||||
d["IP - Stoploss Catastrophic Threshold"] = "N/A"
|
||||
d["IP - DSH/IME/UC, included (Y/N)"] = "N/A"
|
||||
d["IP_OP"] = "N/A"
|
||||
else:
|
||||
d["IP_OP"] = "N/A"
|
||||
d["IP - DSH/IME/UC, included (Y/N)"] = "N/A"
|
||||
d['Inclusion of essential RBRVS "Fee Source" Language (Y/N)'] = "N/A"
|
||||
d["IP - Stoploss Catastrophic Threshold"] = "N/A"
|
||||
|
||||
return combined_results
|
||||
@@ -0,0 +1,148 @@
|
||||
import csv
|
||||
import os
|
||||
|
||||
import pandas as pd
|
||||
|
||||
import src.investment.conditional_funcs as conditional_funcs
|
||||
import src.investment.one_to_n_funcs as one_to_n_funcs
|
||||
import src.investment.postprocess as postprocess
|
||||
import src.investment.preprocess as preprocess
|
||||
from src import config, smart_chunking_funcs
|
||||
from src.regex.regex_utils import irs_hotfix, npi_hotfix
|
||||
|
||||
|
||||
def run_ac_prompts(file_object):
|
||||
|
||||
ac_answers_dict = {}
|
||||
log_data = []
|
||||
chunk_data = []
|
||||
|
||||
################## INITIATE PROCESSING ##################
|
||||
filename, contract_text = file_object
|
||||
print(f"Processing AC for {filename}...")
|
||||
|
||||
################## PREPROCESS - SMART CHUNK ##################
|
||||
|
||||
# Top Sheet Addition
|
||||
text_dict, top_sheet_dict, exhibit_pages, num_pages, ac_chunks = (
|
||||
preprocess.preprocess(contract_text, filename, fields="ac")
|
||||
)
|
||||
print(f"AC Preprocessing Complete - {filename}")
|
||||
|
||||
################## RUN REGEX FUNCTIONALITY ##################
|
||||
# no prompting for these fields
|
||||
|
||||
irs_answers = irs_hotfix(filename, text_dict, top_sheet_dict)
|
||||
ac_answers_dict.update(irs_answers)
|
||||
|
||||
npi_answers = npi_hotfix(text_dict, top_sheet_dict)
|
||||
ac_answers_dict.update(npi_answers)
|
||||
################## RUN SMART CHUNKED PROMPTS ##################
|
||||
ac_answers_dict, no_keyword_matches = (
|
||||
smart_chunking_funcs.run_smart_chunk_prompts(
|
||||
ac_answers_dict, ac_chunks, contract_text, filename, text_dict
|
||||
)
|
||||
)
|
||||
|
||||
################## GET FULL CONTEXT FIELDS ##################
|
||||
# Process 'full_context' fields together
|
||||
full_context_fields = smart_chunking_funcs.get_full_context_fields(
|
||||
ac_answers_dict, no_keyword_matches
|
||||
)
|
||||
|
||||
################## RUN FULL CONTEXT PROMPTS ##################
|
||||
ac_answers_dict = smart_chunking_funcs.run_full_context_prompts(
|
||||
ac_answers_dict, full_context_fields, contract_text, filename
|
||||
)
|
||||
|
||||
################## AC CONDITIONAL PROMPTS ##################
|
||||
ac_answers_dict = smart_chunking_funcs.run_ac_conditional_prompts(
|
||||
ac_answers_dict, contract_text, filename
|
||||
)
|
||||
|
||||
################## CREATE OUTPUT DIRECTORIES ##################
|
||||
base_filename = os.path.splitext(filename)[0].strip()
|
||||
output_dir = os.path.join(config.OUTPUT_DIRECTORY, base_filename)
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
################## POSTPROCESS ##################
|
||||
|
||||
ac_df = pd.DataFrame([ac_answers_dict])
|
||||
|
||||
# print(f"Before ac_postprocess Unique values: {ac_df['CONTRACT_EFFECTIVE_DT'].unique()}")
|
||||
|
||||
ac_df = postprocess.ac_postprocess(ac_df, text_dict, filename, num_pages)
|
||||
|
||||
# print(f"After ac_postprocess Unique values: {ac_df['Contract Effective Date'].unique()}")
|
||||
|
||||
################## WRITE TO OUTPUT ##################
|
||||
ac_df.to_csv(os.path.join(output_dir, config.AC_RESULTS_NAME), index=False)
|
||||
|
||||
chunk_file_path = "chunk_log.csv"
|
||||
with open(chunk_file_path, "a", newline="", encoding="utf-8") as chunk_file:
|
||||
fieldnames = [
|
||||
"Contract name",
|
||||
"Field group",
|
||||
"Field list",
|
||||
"Methodology",
|
||||
"Case sensitivity",
|
||||
"Page list",
|
||||
"Chunk size",
|
||||
"% Reduction",
|
||||
]
|
||||
writer = csv.DictWriter(chunk_file, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
writer.writerows(chunk_data)
|
||||
|
||||
print(f"Chunk log written to {chunk_file_path}")
|
||||
|
||||
return ac_answers_dict
|
||||
|
||||
|
||||
def run_b_prompts(file_object):
|
||||
################## INITIATE PROCESSING ##################
|
||||
filename, contract_text = file_object
|
||||
print(f"Processing B for {filename}...")
|
||||
|
||||
################## PREPROCESS ##################
|
||||
text_dict, top_sheet_dict, exhibit_pages, num_pages, ac_chunks = (
|
||||
preprocess.preprocess(contract_text, filename, fields="b")
|
||||
)
|
||||
print(f"B Preprocessing Complete - {filename}")
|
||||
|
||||
################## RUN BOTTOM UP PROMPTS ##################
|
||||
bu_results = one_to_n_funcs.run_bottom_up(
|
||||
filename, text_dict
|
||||
) # Returns list of dictionaries
|
||||
print(f"B Bottom Up Complete - {filename}")
|
||||
|
||||
################## RUN TOP DOWN PROMPTS ##################
|
||||
combined_results = one_to_n_funcs.run_top_down(
|
||||
filename, text_dict, bu_results, exhibit_pages
|
||||
) # Returns list of dictionaries
|
||||
print(f"B Top Down Complete - {filename}")
|
||||
|
||||
################## RUN CONDITIONAL PROMPTS ##################
|
||||
conditional_results = conditional_funcs.run_conditional(
|
||||
combined_results, text_dict, filename
|
||||
) # List of dictionaries
|
||||
print(f"B Conditional Prompts Complete - {filename}")
|
||||
|
||||
################## CREATE OUTPUT DIRECTORIES ##################
|
||||
base_filename = os.path.splitext(filename)[0].strip()
|
||||
output_dir = os.path.join(config.OUTPUT_DIRECTORY, base_filename)
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
################## WRITE UNPROCESSED OUTPUT ##################
|
||||
combined_df = pd.DataFrame(conditional_results)
|
||||
combined_df.to_csv(
|
||||
os.path.join(output_dir, config.UNPROCESSED_RESULTS_NAME), index=False
|
||||
)
|
||||
|
||||
################## RUN POSTPROCESSING ##################
|
||||
final_df = postprocess.b_postprocess(filename, combined_df, num_pages)
|
||||
print(f"B Postprocessing Complete - {filename} ")
|
||||
|
||||
################## WRITE FINAL ##################
|
||||
final_df.to_csv(os.path.join(output_dir, config.B_RESULTS_NAME), index=False)
|
||||
print(f"B Output Complete - {filename} ")
|
||||
@@ -0,0 +1,371 @@
|
||||
|
||||
import concurrent.futures
|
||||
import os
|
||||
import random
|
||||
import traceback
|
||||
from datetime import datetime
|
||||
|
||||
random.seed(42)
|
||||
|
||||
import src.investment.file_processing as file_processing
|
||||
import src.qa_qc.qa_qc_utils as qa_qc_utils
|
||||
import src.tracking.tracking_utils as tracking_utils
|
||||
import src.utils.io_utils as io_utils
|
||||
import src.utils.string_utils as string_utils
|
||||
# import sys
|
||||
# sys.path.append("src")
|
||||
from src import config, consolidate_output
|
||||
from src.tracking.batch_tracking import BatchTracker
|
||||
|
||||
|
||||
def process_b(item):
|
||||
filename, contract_text = item
|
||||
start_time = datetime.now()
|
||||
|
||||
if string_utils.contains_reimbursement(contract_text, 0):
|
||||
try:
|
||||
file_processing.run_b_prompts(item)
|
||||
end_time = datetime.now()
|
||||
|
||||
output_dir = os.path.join(config.OUTPUT_DIRECTORY, os.path.splitext(filename)[0])
|
||||
output_file = os.path.join(output_dir, config.B_RESULTS_NAME)
|
||||
|
||||
if os.path.exists(output_file):
|
||||
batch_tracker.update_file_progress(filename, 'B', start_time, end_time)
|
||||
tracking_utils.log_file_processing(
|
||||
filename,
|
||||
'B',
|
||||
f"{config.BATCH_ID}/processing/{filename}",
|
||||
'Completed'
|
||||
)
|
||||
return ('B', item)
|
||||
else:
|
||||
error_msg = 'B output file not created'
|
||||
batch_tracker.update_file_progress(
|
||||
filename, 'B', start_time, datetime.now(),
|
||||
status='Failed', error=error_msg
|
||||
)
|
||||
tracking_utils.log_file_processing(
|
||||
filename,
|
||||
'B',
|
||||
f"{config.BATCH_ID}/processing/{filename}",
|
||||
'Failed',
|
||||
error_msg
|
||||
)
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
error_msg = str(e)
|
||||
print(f"Error processing item {filename}: {error_msg}")
|
||||
traceback.print_exc()
|
||||
batch_tracker.update_file_progress(
|
||||
filename, 'B', start_time, datetime.now(),
|
||||
status='Failed', error=error_msg
|
||||
)
|
||||
tracking_utils.log_file_processing(
|
||||
filename,
|
||||
'B',
|
||||
f"{config.BATCH_ID}/processing/{filename}",
|
||||
'Failed',
|
||||
error_msg
|
||||
)
|
||||
return None
|
||||
else:
|
||||
print(f"No reimbursement information found in {filename}. Skipping processing.")
|
||||
batch_tracker.update_file_progress(
|
||||
filename, 'B', datetime.now(), datetime.now(),
|
||||
status='Skipped', error='No reimbursement information found'
|
||||
)
|
||||
tracking_utils.log_file_processing(
|
||||
filename,
|
||||
'B',
|
||||
f"{config.BATCH_ID}/processing/{filename}",
|
||||
'Skipped',
|
||||
'No reimbursement information found'
|
||||
)
|
||||
return None
|
||||
|
||||
def process_ac(item):
|
||||
filename, contract_text = item
|
||||
start_time = datetime.now()
|
||||
|
||||
try:
|
||||
file_processing.run_ac_prompts(item)
|
||||
end_time = datetime.now()
|
||||
|
||||
output_dir = os.path.join(config.OUTPUT_DIRECTORY, os.path.splitext(filename)[0])
|
||||
output_file = os.path.join(output_dir, config.AC_RESULTS_NAME)
|
||||
|
||||
if os.path.exists(output_file):
|
||||
batch_tracker.update_file_progress(filename, 'AC', start_time, end_time)
|
||||
tracking_utils.log_file_processing(
|
||||
filename,
|
||||
'AC',
|
||||
f"{config.BATCH_ID}/processing/{filename}",
|
||||
'Completed'
|
||||
)
|
||||
return ('AC', item)
|
||||
else:
|
||||
error_msg = 'AC output file not created'
|
||||
batch_tracker.update_file_progress(
|
||||
filename, 'AC', start_time, datetime.now(),
|
||||
status='Failed', error=error_msg
|
||||
)
|
||||
tracking_utils.log_file_processing(
|
||||
filename,
|
||||
'AC',
|
||||
f"{config.BATCH_ID}/processing/{filename}",
|
||||
'Failed',
|
||||
error_msg
|
||||
)
|
||||
return None
|
||||
except Exception as e:
|
||||
error_msg = str(e)
|
||||
print(f"Error processing item {filename}: {error_msg}")
|
||||
traceback.print_exc()
|
||||
batch_tracker.update_file_progress(
|
||||
filename, 'AC', start_time, datetime.now(),
|
||||
status='Failed', error=error_msg
|
||||
)
|
||||
tracking_utils.log_file_processing(
|
||||
filename,
|
||||
'AC',
|
||||
f"{config.BATCH_ID}/processing/{filename}",
|
||||
'Failed',
|
||||
error_msg
|
||||
)
|
||||
return None
|
||||
|
||||
def process_results(result, filename, input_dict_b, stats):
|
||||
"""Process individual results and update tracking stats"""
|
||||
if not result or not result[1]:
|
||||
return stats
|
||||
|
||||
proc_type, item = result
|
||||
filename = item[0]
|
||||
|
||||
output_dir = os.path.join(config.OUTPUT_DIRECTORY, os.path.splitext(filename)[0])
|
||||
output_file = os.path.join(output_dir,
|
||||
config.B_RESULTS_NAME if proc_type == 'B' else config.AC_RESULTS_NAME)
|
||||
|
||||
if os.path.exists(output_file):
|
||||
if filename not in stats['processed_files'].get(proc_type, set()):
|
||||
if proc_type == 'B':
|
||||
stats['completed_b'] += 1
|
||||
else:
|
||||
stats['completed_ac'] += 1
|
||||
stats['processed_files'].setdefault(proc_type, set()).add(filename)
|
||||
|
||||
tracking_utils.log_file_processing(
|
||||
filename,
|
||||
proc_type,
|
||||
f"{config.BATCH_ID}/processing/{filename}",
|
||||
'Completed'
|
||||
)
|
||||
|
||||
stats['processed_count'] += 1
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def process_remaining_files(input_dict, input_dict_b, run_timestamp, stats):
|
||||
"""Process any remaining files in self-repair mode"""
|
||||
print("\nEntering self-repair mode...")
|
||||
|
||||
|
||||
completed_ac, completed_b = tracking_utils.get_processed_files(config.BATCH_ID)
|
||||
|
||||
|
||||
stats['processed_files']['AC'] = completed_ac
|
||||
stats['processed_files']['B'] = completed_b
|
||||
stats['completed_ac'] = len(completed_ac)
|
||||
stats['completed_b'] = len(completed_b)
|
||||
|
||||
# Find truly remaining files
|
||||
remaining_ac = {k: v for k, v in input_dict.items()
|
||||
if k not in completed_ac}
|
||||
remaining_b = {k: v for k, v in input_dict_b.items()
|
||||
if k not in completed_b}
|
||||
|
||||
if not remaining_ac and not remaining_b:
|
||||
print("No files need repair processing.")
|
||||
return stats
|
||||
|
||||
print(f"Found {len(remaining_ac)} AC files and {len(remaining_b)} B files requiring processing")
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor:
|
||||
futures = []
|
||||
if remaining_b:
|
||||
print(f"Processing {len(remaining_b)} remaining B files...")
|
||||
futures.extend([executor.submit(process_b, item) for item in remaining_b.items()])
|
||||
if remaining_ac:
|
||||
print(f"Processing {len(remaining_ac)} remaining AC files...")
|
||||
futures.extend([executor.submit(process_ac, item) for item in remaining_ac.items()])
|
||||
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
try:
|
||||
result = future.result()
|
||||
if result:
|
||||
stats = process_results(result, result[1][0], input_dict_b, stats)
|
||||
print(f"Repair processing completed for: {result[1][0]}")
|
||||
except Exception as e:
|
||||
print(f"Error in repair processing: {e}")
|
||||
traceback.print_exc()
|
||||
|
||||
print("Self-repair processing complete")
|
||||
print("Reconsolidating outputs after repair...")
|
||||
consolidate_output.consolidate_output(run_timestamp)
|
||||
|
||||
return stats
|
||||
|
||||
def main():
|
||||
global batch_tracker
|
||||
batch_tracker = BatchTracker()
|
||||
|
||||
try:
|
||||
run_timestamp = datetime.now().strftime(f"run_%Y%m%d_%H:%M_{config.BATCH_ID}")
|
||||
|
||||
# Read and process input
|
||||
input_dict = io_utils.read_input()
|
||||
print(f"Total Input Files : {len(input_dict)}")
|
||||
|
||||
# Filter B - files with reimbursement
|
||||
input_dict_b = {
|
||||
k: v for k, v in input_dict.items()
|
||||
if string_utils.contains_reimbursement(str(v))
|
||||
}
|
||||
|
||||
batch_tracker.set_input_file_counts(
|
||||
total_files=len(input_dict),
|
||||
reimbursement_files=len(input_dict_b)
|
||||
)
|
||||
batch_tracker.add_batch_run()
|
||||
|
||||
if config.FILTER_ALREADY_PROCESSED:
|
||||
# Get processed files using new tracking_utils function
|
||||
processed_ac, processed_b = tracking_utils.get_processed_files(config.BATCH_ID)
|
||||
|
||||
# Filter AC files
|
||||
input_dict_ac = {
|
||||
k: v for k, v in input_dict.items()
|
||||
if k not in processed_ac
|
||||
}
|
||||
|
||||
# Filter B files
|
||||
input_dict_b = {
|
||||
k: v for k, v in input_dict_b.items()
|
||||
if k not in processed_b
|
||||
}
|
||||
print(f"Filtered out {len(processed_ac) + len(processed_b)} already processed files")
|
||||
else:
|
||||
input_dict_ac = input_dict
|
||||
|
||||
# Calculate totals
|
||||
total_ac = len(input_dict_ac) if "a" in config.FIELDS.lower() and "c" in config.FIELDS.lower() else 0
|
||||
total_b = len(input_dict_b) if "b" in config.FIELDS.lower() else 0
|
||||
|
||||
# Initialize tracking stats
|
||||
stats = {
|
||||
'completed_ac': 0,
|
||||
'completed_b': 0,
|
||||
'processed_count': 0,
|
||||
'processed_files': {'AC': set(), 'B': set()}
|
||||
}
|
||||
|
||||
# Setup processing types
|
||||
files_processing_types = {}
|
||||
if "b" in config.FIELDS.lower():
|
||||
for filename in input_dict_b:
|
||||
files_processing_types[filename] = files_processing_types.get(filename, []) + ['B']
|
||||
|
||||
if "a" in config.FIELDS.lower() and "c" in config.FIELDS.lower():
|
||||
for filename in input_dict_ac:
|
||||
files_processing_types[filename] = files_processing_types.get(filename, []) + ['AC']
|
||||
|
||||
total_processing_events = sum(len(types) for types in files_processing_types.values())
|
||||
print(f"Total unique files: {len(files_processing_types)}")
|
||||
print(f"Total AC files to process: {total_ac}")
|
||||
print(f"Total B files to process: {total_b}")
|
||||
|
||||
last_upload_count = 0
|
||||
UPLOAD_FREQUENCY = 10
|
||||
|
||||
# Process files
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor:
|
||||
futures = []
|
||||
|
||||
if "b" in config.FIELDS.lower() and len(input_dict_b) > 0:
|
||||
futures.extend([executor.submit(process_b, item) for item in input_dict_b.items()])
|
||||
|
||||
if "a" in config.FIELDS.lower() and "c" in config.FIELDS.lower() and len(input_dict_ac) > 0:
|
||||
futures.extend([executor.submit(process_ac, item) for item in input_dict_ac.items()])
|
||||
|
||||
if not futures:
|
||||
raise ValueError("no input files were staged for processing")
|
||||
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
try:
|
||||
result = future.result()
|
||||
if result:
|
||||
filename = result[1][0]
|
||||
stats = process_results(result, filename, input_dict_b, stats)
|
||||
|
||||
# Calculate and display progress
|
||||
ac_progress, b_progress = tracking_utils.calculate_progress_stats(
|
||||
stats['completed_ac'], total_ac,
|
||||
stats['completed_b'], total_b
|
||||
)
|
||||
print(f"\nProgress:")
|
||||
print(f"AC: {ac_progress:.2f}% ({stats['completed_ac']}/{total_ac})")
|
||||
print(f"B: {b_progress:.2f}% ({stats['completed_b']}/{total_b})")
|
||||
print(f"Overall: {(stats['processed_count'] / total_processing_events * 100):.2f}%")
|
||||
|
||||
if stats['processed_count'] - last_upload_count >= UPLOAD_FREQUENCY:
|
||||
try:
|
||||
consolidate_output.consolidate_output(run_timestamp)
|
||||
last_upload_count = stats['processed_count']
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not update tracking data: {str(e)}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error processing future: {e}")
|
||||
traceback.print_exc()
|
||||
|
||||
print("\nInitial Processing Complete")
|
||||
|
||||
print("Starting output consolidation and QA/QC...")
|
||||
try:
|
||||
ac_final_df, b_final_df, abc_final_df = consolidate_output.consolidate_output(run_timestamp)
|
||||
wb = qa_qc_utils.perform_qc_qa(ac_final_df, b_final_df, abc_final_df, input_dict)
|
||||
qa_qc_utils.save_qcqa_wb(wb, run_timestamp)
|
||||
print("Consolidation and QA/QC complete")
|
||||
except Exception as e:
|
||||
print(f"Error during consolidation: {str(e)}")
|
||||
|
||||
# Process remaining files and get updated stats
|
||||
stats = process_remaining_files(input_dict, input_dict_b, run_timestamp, stats)
|
||||
|
||||
batch_tracker.update_batch_run(end_time=datetime.now())
|
||||
|
||||
print(f"\nFinal Statistics:")
|
||||
print(f"Total input files: {batch_tracker.total_input_files}")
|
||||
print(f"Files with reimbursement: {batch_tracker.reimbursement_files}")
|
||||
print(f"AC files processed: {stats['completed_ac']}/{total_ac}")
|
||||
print(f"B files processed: {stats['completed_b']}/{total_b}")
|
||||
print(f"Total processing events completed: {stats['processed_count']}/{total_processing_events}")
|
||||
|
||||
if stats['processed_count'] > 0 and total_processing_events > 0:
|
||||
success_rate = (stats['processed_count'] / total_processing_events) * 100
|
||||
print(f"Overall success rate: {success_rate:.2f}%")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error in main processing: {e}")
|
||||
traceback.print_exc()
|
||||
try:
|
||||
batch_tracker.update_batch_run(end_time=datetime.now())
|
||||
except Exception as tracking_error:
|
||||
print(f"Error updating batch tracking: {tracking_error}")
|
||||
raise e
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,369 @@
|
||||
|
||||
|
||||
import re
|
||||
|
||||
import src.constants.valid as valid
|
||||
import src.utils.llm_utils as llm_utils
|
||||
import src.utils.string_utils as string_utils
|
||||
from src import config, postprocessing_funcs, prompts
|
||||
|
||||
|
||||
def run_bottom_up_primary(
|
||||
text_dict: dict, tokens: int, filename: str
|
||||
) -> dict[str, str]:
|
||||
"""Executes the primary Bottom Up processing for pages that contain reimbursement terms.
|
||||
|
||||
This function scans through a dictionary of page texts, identifies pages containing reimbursement terms,
|
||||
and processes those pages using Claude to extract relevant information.
|
||||
|
||||
Args:
|
||||
text_dict (dict): A dictionary containing text content of documents keyed by page numbers.
|
||||
tokens (int): The maximum number of tokens to use in the language model invocation for generating the response. Deprecated.
|
||||
filename (str): The filename of the contract in question. Used for logging purposes.
|
||||
|
||||
Returns:
|
||||
dict[str, str]: A dictionary where each key is a page number (string) and the value is the response from the language model.
|
||||
"""
|
||||
|
||||
# chunk_dict = preprocess.chunk_text(text_dict)
|
||||
answer_strings = {}
|
||||
# for page_number in chunk_dict.keys():
|
||||
# if '%' in chunk_dict[page_number] or '$' in chunk_dict[page_number]:
|
||||
# prompt = prompts.BOTTOM_UP_PRIMARY(chunk_dict[page_number], config.CLIENT_NAME)
|
||||
for page_number in text_dict.keys():
|
||||
if string_utils.contains_reimbursement(text_dict, page_number):
|
||||
# Run Primary
|
||||
prompt = prompts.BOTTOM_UP_PRIMARY(
|
||||
text_dict[page_number], config.CLIENT_NAME
|
||||
)
|
||||
answer = llm_utils.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, filename
|
||||
)
|
||||
answer_strings[page_number] = answer
|
||||
|
||||
answer_dicts = string_utils.primary_string_to_dict(answer_strings, filename)
|
||||
|
||||
answer_dicts = postprocessing_funcs.consolidate_subheader(answer_dicts)
|
||||
answer_dicts = postprocessing_funcs.filter_service_column(
|
||||
answer_dicts
|
||||
) # Filter on entire Service + Subheader
|
||||
answer_dicts = postprocessing_funcs.filter_methodology_column(answer_dicts)
|
||||
|
||||
return answer_dicts
|
||||
|
||||
|
||||
def get_short_methodology(s):
|
||||
pattern = r"\d+\.*\d*%"
|
||||
short_string = re.sub(pattern, "%", s)
|
||||
return short_string
|
||||
|
||||
|
||||
def run_bottom_up_secondary(
|
||||
answer_dicts: list[dict], text_dict: dict, tokens: int, filename: str
|
||||
) -> list[dict]:
|
||||
"""Executes the secondary Bottom Up processing phase on the results obtained from the primary Bottom Up analysis.
|
||||
|
||||
This function enhances the primary results with additional analyses based on configured conditions. Invokes
|
||||
Claude 3 with tailored prompts to generate structured information that complements the initial results.
|
||||
|
||||
Each piece of data processed possibly undergoes several rounds of checks and transformations, ensuring detailed and
|
||||
comprehensive output.
|
||||
|
||||
Args:
|
||||
answer_dicts (list[dict]): Initial processed data from the primary Bottom Up analysis.
|
||||
text_dict (dict): Dictionary containing the original document text keyed by page numbers.
|
||||
tokens (int): Token limit for language model invocations.
|
||||
filename (str): The filename of the contract in question. Used for logging purposes.
|
||||
|
||||
Returns:
|
||||
list[dict]: A list of dictionaries containing enriched and finalized structured data from both the primary and
|
||||
secondary analyses.
|
||||
"""
|
||||
# Add additional fields to each row
|
||||
final_dicts = []
|
||||
for d in answer_dicts:
|
||||
if d is not None:
|
||||
page_num = d["page_num"]
|
||||
|
||||
# if '%' in d['FULL_METHODOLOGY'] or '$' in d['FULL_METHODOLOGY']:
|
||||
prompt = prompts.BOTTOM_UP_METHODOLOGY_BREAKOUT(d)
|
||||
answer = llm_utils.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, max_tokens=512
|
||||
)
|
||||
answer_dict = string_utils.secondary_string_to_dict(answer, filename)
|
||||
d.update(answer_dict)
|
||||
|
||||
# Ensure all bottom up methodology keys are present
|
||||
for key in [
|
||||
"LESSER",
|
||||
"RATE_STANDARD",
|
||||
"RATE_SHORT",
|
||||
"FLAT_FEE_STANDARD",
|
||||
"LESSER_RATE",
|
||||
"NOT_TO_EXCEED",
|
||||
]:
|
||||
if key not in d.keys():
|
||||
d[key] = ""
|
||||
|
||||
# SHORT_METHODOLOGY
|
||||
if "%" in d["RATE_STANDARD"]:
|
||||
d["SHORT_METHODOLOGY"] = get_short_methodology(d["RATE_STANDARD"])
|
||||
elif "$" in d["FLAT_FEE_STANDARD"]:
|
||||
d["SHORT_METHODOLOGY"] = "Flat Fee"
|
||||
else:
|
||||
d["SHORT_METHODOLOGY"] = "N/A"
|
||||
|
||||
# Escalator
|
||||
prompt = prompts.BOTTOM_UP_ESCALATOR(d, text_dict[page_num])
|
||||
escalator_answer = llm_utils.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, max_tokens=1024
|
||||
)
|
||||
escalator_dict = string_utils.secondary_string_to_dict(
|
||||
escalator_answer, filename
|
||||
)
|
||||
d.update(escalator_dict)
|
||||
|
||||
final_dicts.append(d)
|
||||
|
||||
return final_dicts
|
||||
|
||||
|
||||
def run_bottom_up(filename: str, text_dict: dict) -> list[dict]:
|
||||
"""Processes the text of a document using a two-tiered Bottom Up approach to extract key financial and operational information.
|
||||
|
||||
This function first runs BOTTOM_UP_PRIMARY and BOTTOM_UP_SECONDARY prompts, then processes these initial results to further
|
||||
refine and structure them into a dictionary form.
|
||||
|
||||
Args:
|
||||
filename (str): The name of the file being processed, used to tag output data.
|
||||
text_dict (dict): A dictionary of text keyed by page numbers that contains the content to be analyzed.
|
||||
|
||||
Returns:
|
||||
list[dict]: A list of dictionaries with each dictionary containing refined and structured information
|
||||
from both primary and secondary Bottom Up analyses, all tagged with the filename.
|
||||
"""
|
||||
|
||||
# Bottom Up Primary - FULL_SERVICE, FULL_METHODOLOGY
|
||||
bu_primary_results = run_bottom_up_primary(text_dict, 8000, filename)
|
||||
|
||||
# Bottom Up Secondary
|
||||
bu_secondary_results = run_bottom_up_secondary(
|
||||
bu_primary_results, text_dict, 8000, filename
|
||||
)
|
||||
|
||||
for d in bu_secondary_results:
|
||||
d["Filename"] = filename
|
||||
return bu_secondary_results # List of dictionaries
|
||||
|
||||
|
||||
def get_exhibit_range(text_dict, exhibit_pages, bu_page):
|
||||
lower_bound = max(
|
||||
[int(page) for page in exhibit_pages if int(page) <= int(bu_page)], default=None
|
||||
)
|
||||
upper_bound = min(
|
||||
[int(page) for page in exhibit_pages if int(page) > int(bu_page)], default=None
|
||||
)
|
||||
if lower_bound is not None and upper_bound is not None:
|
||||
pages_between = [
|
||||
page
|
||||
for page in text_dict.keys()
|
||||
if int(lower_bound) <= int(page) <= int(upper_bound)
|
||||
]
|
||||
elif lower_bound is not None: # No upper_bound found
|
||||
pages_between = [
|
||||
page for page in text_dict.keys() if int(lower_bound) <= int(page)
|
||||
]
|
||||
elif upper_bound is not None: # No lower_bound found
|
||||
pages_between = [
|
||||
page for page in text_dict.keys() if int(page) <= int(upper_bound)
|
||||
]
|
||||
else:
|
||||
pages_between = []
|
||||
return pages_between
|
||||
|
||||
|
||||
def run_top_down_primary(text_dict, td_pages, filename):
|
||||
page_text = "\n".join([text_dict[page_num] for page_num in td_pages])
|
||||
|
||||
prompt = prompts.TOP_DOWN_PRIMARY(page_text)
|
||||
answer = llm_utils.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, max_tokens=4096
|
||||
)
|
||||
answer_dict = string_utils.secondary_string_to_dict(answer, filename)
|
||||
for field in prompts.TD_PRIMARY_FIELDS:
|
||||
if field not in answer_dict:
|
||||
answer_dict[field] = "N/A"
|
||||
|
||||
# Add indicators
|
||||
if "ADD_ON_REIMBURSEMENT_LANGUAGE" in answer_dict:
|
||||
if "N/A" in answer_dict["ADD_ON_REIMBURSEMENT_LANGUAGE"]:
|
||||
answer_dict["ADD_ON_REIMBURSEMENT_IND"] = "N"
|
||||
else:
|
||||
answer_dict["ADD_ON_REIMBURSEMENT_IND"] = "Y"
|
||||
else:
|
||||
answer_dict["ADD_ON_REIMBURSEMENT_IND"] = "N"
|
||||
answer_dict["ADD_ON_REIMBURSEMENT_LANGUAGE"] = "N/A"
|
||||
|
||||
if config.PROV_TYPE:
|
||||
answer_dict["PROV_TYPE"] = config.PROV_TYPE
|
||||
return answer_dict
|
||||
|
||||
|
||||
def get_td_dict(text_dict, filename, PROMPT_FUNC, VALID_VALUES):
|
||||
td_dict = {}
|
||||
for page_num, page_text in text_dict.items():
|
||||
if (
|
||||
any(i.upper() in page_text.upper() for i in VALID_VALUES)
|
||||
and page_num.isdigit()
|
||||
):
|
||||
prompt = PROMPT_FUNC(page_text)
|
||||
answer = llm_utils.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 1024
|
||||
)
|
||||
td_dict[page_num] = answer
|
||||
return td_dict
|
||||
|
||||
|
||||
def add_medically_necessary(merged_dicts, medically_necessary_dict):
|
||||
if not medically_necessary_dict:
|
||||
for d in merged_dicts:
|
||||
d["MEDICAL_NECESSITY_LANGUAGE"] = "N/A"
|
||||
d["MEDICAL_NECESSITY_LANGUAGE_IND"] = "N"
|
||||
return merged_dicts
|
||||
|
||||
sorted_keys = sorted(int(k) for k in medically_necessary_dict.keys())
|
||||
|
||||
# Find the closest page number that is less than or equal to the given page number
|
||||
def find_closest_page(page_num):
|
||||
closest_page = sorted_keys[0]
|
||||
for key in sorted_keys:
|
||||
if key <= page_num:
|
||||
closest_page = key
|
||||
else:
|
||||
break
|
||||
return closest_page
|
||||
|
||||
for d in merged_dicts:
|
||||
try:
|
||||
current_page_num = int(d["page_num"])
|
||||
# Find the closest preceding medically necessary page
|
||||
closest_page = find_closest_page(current_page_num)
|
||||
d["MEDICAL_NECESSITY_LANGUAGE"] = medically_necessary_dict[
|
||||
str(closest_page)
|
||||
]
|
||||
if not string_utils.is_empty(d["MEDICAL_NECESSITY_LANGUAGE"]):
|
||||
d["MEDICAL_NECESSITY_LANGUAGE_IND"] = "Y"
|
||||
else:
|
||||
d["MEDICAL_NECESSITY_LANGUAGE_IND"] = "N"
|
||||
except:
|
||||
d["MEDICAL_NECESSITY_LANGUAGE"] = "N/A"
|
||||
d["MEDICAL_NECESSITY_LANGUAGE_IND"] = "N"
|
||||
|
||||
return merged_dicts
|
||||
|
||||
|
||||
def add_exclusions(merged_dicts, exclusions_dict):
|
||||
if not exclusions_dict:
|
||||
for d in merged_dicts:
|
||||
d["EXCLUSIONS"] = "N/A"
|
||||
return merged_dicts
|
||||
|
||||
# Prepare by sorting exclusion keys and initializing an empty dictionary to track exclusions by EXHIBIT
|
||||
sorted_keys = sorted(int(k) for k in exclusions_dict.keys())
|
||||
exhibit_exclusions = {}
|
||||
|
||||
# First pass: assign exclusions based on page number
|
||||
for d in merged_dicts:
|
||||
page_num = str(d["page_num"])
|
||||
if page_num.isdigit():
|
||||
for exc_page in sorted_keys:
|
||||
if exc_page >= int(page_num) and exc_page <= int(page_num) + 2:
|
||||
new_exclusion = exclusions_dict[str(exc_page)]
|
||||
# Handle EXCLUSIONS as a list to avoid duplicates
|
||||
if "EXCLUSIONS" not in d:
|
||||
d["EXCLUSIONS"] = []
|
||||
if new_exclusion not in d["EXCLUSIONS"]:
|
||||
try:
|
||||
d["EXCLUSIONS"].append(new_exclusion)
|
||||
except:
|
||||
d["EXCLUSIONS"] = [new_exclusion]
|
||||
|
||||
# Map this exclusion to the EXHIBIT value
|
||||
if "EXHIBIT" in d:
|
||||
if d["EXHIBIT"] not in exhibit_exclusions:
|
||||
exhibit_exclusions[d["EXHIBIT"]] = []
|
||||
if new_exclusion not in exhibit_exclusions[d["EXHIBIT"]]:
|
||||
exhibit_exclusions[d["EXHIBIT"]].append(new_exclusion)
|
||||
break
|
||||
else:
|
||||
# If no matching exclusion found, check if 'EXHIBIT' maps to any existing exclusion
|
||||
if "EXHIBIT" in d and d["EXHIBIT"] in exhibit_exclusions:
|
||||
for exclusion in exhibit_exclusions[d["EXHIBIT"]]:
|
||||
if "EXCLUSIONS" not in d:
|
||||
d["EXCLUSIONS"] = []
|
||||
if exclusion not in d["EXCLUSIONS"]:
|
||||
d["EXCLUSIONS"].append(exclusion)
|
||||
else:
|
||||
d["EXCLUSIONS"] = ["N/A"]
|
||||
else:
|
||||
d["EXCLUSIONS"] = ["N/A"]
|
||||
|
||||
# Convert lists to semicolon-separated strings
|
||||
for d in merged_dicts:
|
||||
if isinstance(d["EXCLUSIONS"], list):
|
||||
d["EXCLUSIONS"] = "; ".join(d["EXCLUSIONS"])
|
||||
|
||||
return merged_dicts
|
||||
|
||||
|
||||
def run_top_down(filename, text_dict, bu_results, exhibit_pages):
|
||||
bu_pages = {bu_dict["page_num"] for bu_dict in bu_results}
|
||||
|
||||
td_page_dict = {}
|
||||
td_answer_dict = {}
|
||||
|
||||
for bu_page in bu_pages:
|
||||
td_pages = get_exhibit_range(text_dict, exhibit_pages, bu_page)
|
||||
if td_pages not in td_page_dict.values():
|
||||
answer_dict = run_top_down_primary(text_dict, td_pages, filename)
|
||||
td_page_dict[bu_page] = td_pages
|
||||
td_answer_dict[bu_page] = answer_dict
|
||||
else:
|
||||
td_page_dict[bu_page] = td_pages
|
||||
for key, value in td_page_dict.items():
|
||||
if value == td_pages:
|
||||
answer_dict = td_answer_dict[key]
|
||||
td_answer_dict[bu_page] = answer_dict
|
||||
break
|
||||
|
||||
merged_dicts = []
|
||||
for bu_dict in bu_results:
|
||||
answer_dict = td_answer_dict[bu_dict["page_num"]]
|
||||
bu_dict.update(answer_dict)
|
||||
merged_dicts.append(bu_dict)
|
||||
|
||||
# Run and Merge Exclusions
|
||||
exclusion_dict = get_td_dict(
|
||||
text_dict, filename, prompts.TOP_DOWN_EXCLUSIONS, valid.EXCLUSION_TERMS
|
||||
)
|
||||
final_dicts = add_exclusions(merged_dicts, exclusion_dict)
|
||||
|
||||
# Run and Merge Medically Necessary
|
||||
medically_necessary_dict = get_td_dict(
|
||||
text_dict,
|
||||
filename,
|
||||
prompts.TOP_DOWN_MEDICALLY_NECESSARY,
|
||||
valid.VALID_MEDICALLY_NECESSARY,
|
||||
)
|
||||
medically_necessary_dict = postprocessing_funcs.filter_dict(
|
||||
medically_necessary_dict,
|
||||
pattern=re.compile(
|
||||
"|".join(
|
||||
re.escape(keyword) for keyword in valid.INVALID_MEDICALLY_NECESSARY
|
||||
),
|
||||
re.IGNORECASE,
|
||||
),
|
||||
)
|
||||
final_dicts = add_medically_necessary(final_dicts, medically_necessary_dict)
|
||||
|
||||
return final_dicts
|
||||
@@ -0,0 +1,110 @@
|
||||
import src.constants.valid as valid
|
||||
from src import postprocessing_funcs
|
||||
|
||||
|
||||
def b_postprocess(filename, df, pages):
|
||||
|
||||
if df.shape[0] > 0:
|
||||
# Metadata fields
|
||||
df.drop("Filename", axis=1, inplace=True)
|
||||
|
||||
df["Contract Name"] = "Filename: " + filename
|
||||
df["Parent Agreement Code"] = postprocessing_funcs.get_parent_agreement_code(
|
||||
filename
|
||||
)
|
||||
df["Pages"] = pages
|
||||
|
||||
# Add Single Code, Multiple Rate
|
||||
df = postprocessing_funcs.add_scmr(df)
|
||||
|
||||
# Filter Add Ons
|
||||
df = postprocessing_funcs.filter_add_ons(df)
|
||||
|
||||
# Clean MSR
|
||||
df = postprocessing_funcs.clean_msr_lesser(df)
|
||||
|
||||
# Clean Default
|
||||
df = postprocessing_funcs.clean_default_term(df)
|
||||
|
||||
# Clean Prov 2
|
||||
df = postprocessing_funcs.clean_prov_2(df)
|
||||
|
||||
# Clean Lesser Of Rate
|
||||
df = postprocessing_funcs.clean_lesser_rate(df)
|
||||
|
||||
# Clean LOB
|
||||
df = postprocessing_funcs.clean_lob(df, filename)
|
||||
|
||||
df = postprocessing_funcs.methodology_short_fix(df)
|
||||
|
||||
df = postprocessing_funcs.yn_fixes(df)
|
||||
|
||||
# Rename and reorder
|
||||
df.rename(columns=valid.B_MAPPING, inplace=True)
|
||||
|
||||
column_order = [col for col in valid.B_MAPPING.values() if col in df.columns]
|
||||
final_df = df[column_order]
|
||||
|
||||
return final_df
|
||||
else:
|
||||
return df
|
||||
|
||||
|
||||
def ac_postprocess(df, text_dict, filename, pages):
|
||||
|
||||
if df.shape[0] > 0:
|
||||
|
||||
df["Contract Name"] = "Filename: " + filename
|
||||
|
||||
df["Pages"] = pages
|
||||
|
||||
df["Parent Agreement Code"] = postprocessing_funcs.get_parent_agreement_code(
|
||||
filename
|
||||
)
|
||||
|
||||
df = df.fillna("")
|
||||
|
||||
df = postprocessing_funcs.clean_term_clause(df)
|
||||
|
||||
df = postprocessing_funcs.clean_auto_renewal_ind(df)
|
||||
|
||||
df = postprocessing_funcs.get_tin_from_filename(df)
|
||||
|
||||
df = postprocessing_funcs.clean_tin(df)
|
||||
|
||||
df = postprocessing_funcs.clean_npi(df)
|
||||
|
||||
df = postprocessing_funcs.clean_network_access_fees(df)
|
||||
|
||||
df = postprocessing_funcs.clean_health_plan_state(df)
|
||||
|
||||
df = postprocessing_funcs.clean_health_plan_state_from_hotfix(df,filename,text_dict)
|
||||
|
||||
df = postprocessing_funcs.clean_notice_provider_name_and_address(df)
|
||||
|
||||
df = postprocessing_funcs.clean_policies_and_procedures(df)
|
||||
|
||||
df = postprocessing_funcs.clean_tin_npi_other(df)
|
||||
|
||||
df = postprocessing_funcs.icm_number_fix(df)
|
||||
|
||||
df = postprocessing_funcs.yn_fixes(df)
|
||||
|
||||
df = df.apply(lambda x: x.map(postprocessing_funcs.replace_null_terms))
|
||||
|
||||
df = df.apply(lambda x: x.map(postprocessing_funcs.replace_quotes))
|
||||
|
||||
# Derive Indicators
|
||||
df = df.apply(postprocessing_funcs.derive_indicators, axis=1)
|
||||
|
||||
# Rename and Reorder
|
||||
df = df.rename(columns=valid.AC_MAPPING)
|
||||
column_order = [col for col in valid.ABC_COLUMNS if col in df.columns] + [
|
||||
col for col in df.columns if col not in valid.ABC_COLUMNS
|
||||
]
|
||||
|
||||
final_df = df[column_order]
|
||||
|
||||
return final_df
|
||||
else:
|
||||
return df
|
||||
@@ -0,0 +1,37 @@
|
||||
import csv
|
||||
|
||||
import src.utils.table_utils as table_utils
|
||||
from src import config, keywords, preprocessing_funcs
|
||||
|
||||
|
||||
def preprocess(contract_text, filename, fields=config.FIELDS):
|
||||
|
||||
contract_text = preprocessing_funcs.clean_newlines(contract_text)
|
||||
contract_text = preprocessing_funcs.clean_law_symbols(contract_text)
|
||||
|
||||
text_dict = preprocessing_funcs.split_text(
|
||||
contract_text
|
||||
) # return a dictionary with keys - page_num (str), values as the page_text
|
||||
|
||||
# text_dict = {k: v for k, v in text_dict.items() if k.isdigit() and 42 <= int(k) <= 42}
|
||||
num_pages = len(text_dict.keys())
|
||||
text_dict, top_sheet_dict = preprocessing_funcs.filter_quick_review(text_dict)
|
||||
|
||||
if fields == "b":
|
||||
exhibit_pages = preprocessing_funcs.get_exhibit_pages(
|
||||
text_dict, filename
|
||||
) # All pages with exhibit headers
|
||||
text_dict = table_utils.align_and_format_tables(text_dict, filename)
|
||||
text_dict = preprocessing_funcs.chunk_consecutive(text_dict, exhibit_pages)
|
||||
# write text_dict to a csv file:
|
||||
with open("text_dict.csv", "w") as f:
|
||||
writer = csv.writer(f)
|
||||
for key, value in text_dict.items():
|
||||
writer.writerow([key, value])
|
||||
|
||||
ac_chunks, full_context = "", ""
|
||||
if fields == "ac":
|
||||
exhibit_pages = []
|
||||
ac_chunks = preprocessing_funcs.smart_chunk_ac(text_dict, contract_text, keywords.GROUPED_KEYWORD_MAPPINGS)
|
||||
|
||||
return text_dict, top_sheet_dict, exhibit_pages, num_pages, ac_chunks
|
||||
@@ -10,7 +10,7 @@
|
||||
# "irs_group / "Gray Group":"
|
||||
# 'Tax ID,TIN,IRS,Provider Name,National Provider Identifier,NPI,Other NPIs,To Provider At:,To Provider at:'
|
||||
|
||||
import config
|
||||
from src import config
|
||||
|
||||
GROUPED_KEYWORD_MAPPINGS = {
|
||||
"term_group": {
|
||||
|
||||
@@ -4,12 +4,12 @@ import re
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
import claude_funcs
|
||||
import config
|
||||
import prompts
|
||||
import valid
|
||||
from string_funcs import is_empty
|
||||
from valid import DERIVED_INDICATOR_FIELDS, STATE_MAP, keywords_to_detect_invalid_state_name
|
||||
import src.constants.valid as valid
|
||||
import src.utils.llm_utils as llm_utils
|
||||
from src import config, prompts
|
||||
from src.constants.valid import (DERIVED_INDICATOR_FIELDS, STATE_MAP,
|
||||
keywords_to_detect_invalid_state_name)
|
||||
from src.utils.string_utils import is_empty
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.ERROR, format="%(asctime)s - %(levelname)s - %(message)s"
|
||||
@@ -649,7 +649,7 @@ def clean_lob(df, filename):
|
||||
> 1
|
||||
):
|
||||
prompt = prompts.LOB_SWEEPER(row["EXHIBIT"])
|
||||
answer = claude_funcs.invoke_claude(
|
||||
answer = llm_utils.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 256
|
||||
)
|
||||
return answer # Return the value from 'EXHIBIT' if condition is met
|
||||
|
||||
@@ -2,15 +2,10 @@ import re
|
||||
from collections import defaultdict
|
||||
from itertools import chain, count, groupby
|
||||
|
||||
import claude_funcs
|
||||
import config
|
||||
import keywords
|
||||
import prompts
|
||||
import smart_chunking_funcs
|
||||
import string_funcs
|
||||
import valid
|
||||
from valid import DERIVED_INDICATOR_FIELDS
|
||||
from regex_patterns import PIPE_PATTERN
|
||||
import src.utils.llm_utils as llm_utils
|
||||
import src.utils.string_utils as string_utils
|
||||
from src import config, keywords, prompts, smart_chunking_funcs
|
||||
from src.regex.regex_patterns import PIPE_PATTERN
|
||||
|
||||
|
||||
def clean_newlines(contract_text: str) -> str:
|
||||
@@ -66,7 +61,7 @@ def clean_law_symbols(contract_text):
|
||||
def chunk_consecutive_og(text_dict, exhibit_pages):
|
||||
# If needed - extend page 1 to page 1 and 2, then cut chunking off after (edge case: compensation terms not found on first page of exhibit)
|
||||
|
||||
reimbursement_pages = [page_num for page_num in text_dict.keys() if string_funcs.contains_reimbursement(text_dict, page_num)]
|
||||
reimbursement_pages = [page_num for page_num in text_dict.keys() if string_utils.contains_reimbursement(text_dict, page_num)]
|
||||
|
||||
page_dict = {}
|
||||
current_exhibit = None
|
||||
@@ -106,7 +101,7 @@ def chunk_consecutive(text_dict, exhibit_pages):
|
||||
reimbursement_pages = {
|
||||
page_num
|
||||
for page_num in text_dict.keys()
|
||||
if string_funcs.contains_reimbursement(text_dict, page_num)
|
||||
if string_utils.contains_reimbursement(text_dict, page_num)
|
||||
}
|
||||
|
||||
page_dict = {}
|
||||
@@ -304,7 +299,7 @@ def get_exhibit_pages(text_dict, filename):
|
||||
exhibit_pages = []
|
||||
for page_num, page in text_dict.items():
|
||||
prompt = prompts.EXHIBIT_CHECK(page[0:100])
|
||||
claude_answer_raw = claude_funcs.invoke_claude(
|
||||
claude_answer_raw = llm_utils.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE3_HAIKU, filename, max_tokens=10
|
||||
)
|
||||
claude_answer_extracted = re.findall(pattern=PIPE_PATTERN, string=claude_answer_raw)[-1]
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import valid
|
||||
import src.constants.valid as valid
|
||||
|
||||
#####################################################################################
|
||||
##################################### BOTTOM UP #####################################
|
||||
|
||||
@@ -6,12 +6,11 @@ import pandas as pd
|
||||
from openpyxl import Workbook
|
||||
from openpyxl.utils.dataframe import dataframe_to_rows
|
||||
|
||||
import config
|
||||
import io_utils
|
||||
import preprocessing_funcs
|
||||
import string_funcs
|
||||
from valid import keywords_to_detect_invalid_state_name
|
||||
from valid import AC_IND_LANG_COLS, B_IND_LANG_COLS, STATE_MAP
|
||||
import src.utils.io_utils as io_utils
|
||||
import src.utils.string_utils as string_utils
|
||||
from src import config, preprocessing_funcs
|
||||
from src.constants.valid import (STATE_MAP,
|
||||
keywords_to_detect_invalid_state_name)
|
||||
|
||||
|
||||
def detect_date_anomalies(series, field_name, is_merged=False):
|
||||
@@ -164,7 +163,7 @@ def blanks_check(df: pd.DataFrame, threshold=config.BLANKS_THRESHOLD) -> pd.Data
|
||||
|
||||
highly_blank_cols = []
|
||||
for col in df.columns:
|
||||
if df[col].apply(lambda x: string_funcs.is_empty(x)).sum() / len(df[col]) > threshold:
|
||||
if df[col].apply(lambda x: string_utils.is_empty(x)).sum() / len(df[col]) > threshold:
|
||||
highly_blank_cols.append(col)
|
||||
|
||||
return '\n'.join(highly_blank_cols)
|
||||
@@ -177,7 +176,7 @@ def self_talk_check(df: pd.DataFrame):
|
||||
|
||||
for col in lang_cols:
|
||||
|
||||
df['selftalk'] = df[col].apply(lambda x: any([(kw in x) for kw in keywords_to_detect_invalid_state_name]) if not string_funcs.is_empty(x) else False)
|
||||
df['selftalk'] = df[col].apply(lambda x: any([(kw in x) for kw in keywords_to_detect_invalid_state_name]) if not string_utils.is_empty(x) else False)
|
||||
|
||||
if df['selftalk'].any():
|
||||
|
||||
@@ -240,18 +239,18 @@ def perform_qc_qa_tests_both(ac_df, b_df, merged_df, input_dict, processed_files
|
||||
"# of fields for B run" : len(b_df.columns),
|
||||
"Highly Blank Fields in merged DataFrame" : blanks_check(merged_df),
|
||||
"Potential 'Self-Talk' columns in merged DataFrame" : self_talk_check(merged_df),
|
||||
"# of incorrectly formatted Provider Group TINs" : ac_df["IRS #"].apply(lambda x: not tin_check(x)[1] if not string_funcs.is_empty(x) else False).sum(),
|
||||
"# of incorrectly formatted Provider Group NPIs" : ac_df["NPI (10-digits)"].apply(lambda x: not npi_check(x)[1] if not string_funcs.is_empty(x) else False).sum(),
|
||||
"# of incorrectly formatted Provider Group Signatory TINs": ac_df["PROV_GROUP_TIN_SIGNATORY"].apply(lambda x: not tin_check(x)[1] if not string_funcs.is_empty(x) else False).sum(),
|
||||
"# of incorrectly formatted Provider TINs (other)" : ac_df["PROV_TIN_OTHER"].apply(lambda x: not tin_check(x)[1] if not string_funcs.is_empty(x) else False).sum(),
|
||||
"# of incorrectly formatted Provider NPIs (other)": ac_df["PROV_NPI_OTHER"].apply(lambda x: not npi_check(x)[1] if not string_funcs.is_empty(x) else False).sum(),
|
||||
"# of incorrect Health Plan States" : ac_df['Health Plan State'].apply(lambda x: not state_check(x)[1] if not string_funcs.is_empty(x) else False).sum(),
|
||||
"# of rows with Pages greater than Page Num" : merged_df.apply(lambda row: pages_pagenum_check(row['Pages'], row['Page_Num']) if (not string_funcs.is_empty(row['Pages']) and string_funcs.is_empty(row['Page_Num'])) else False, axis=1).sum(),
|
||||
"# of Payer Names found in Notice to Provider Name" : ac_df.apply(lambda row: row['PAYER NAME'] in row['Notice to Provider Name'] if not string_funcs.is_empty(row['Notice to Provider Name']) and not string_funcs.is_empty(row['PAYER NAME']) else False, axis = 1).sum(),
|
||||
"# of incorrectly formatted Provider Group TINs" : ac_df["IRS #"].apply(lambda x: not tin_check(x)[1] if not string_utils.is_empty(x) else False).sum(),
|
||||
"# of incorrectly formatted Provider Group NPIs" : ac_df["NPI (10-digits)"].apply(lambda x: not npi_check(x)[1] if not string_utils.is_empty(x) else False).sum(),
|
||||
"# of incorrectly formatted Provider Group Signatory TINs": ac_df["PROV_GROUP_TIN_SIGNATORY"].apply(lambda x: not tin_check(x)[1] if not string_utils.is_empty(x) else False).sum(),
|
||||
"# of incorrectly formatted Provider TINs (other)" : ac_df["PROV_TIN_OTHER"].apply(lambda x: not tin_check(x)[1] if not string_utils.is_empty(x) else False).sum(),
|
||||
"# of incorrectly formatted Provider NPIs (other)": ac_df["PROV_NPI_OTHER"].apply(lambda x: not npi_check(x)[1] if not string_utils.is_empty(x) else False).sum(),
|
||||
"# of incorrect Health Plan States" : ac_df['Health Plan State'].apply(lambda x: not state_check(x)[1] if not string_utils.is_empty(x) else False).sum(),
|
||||
"# of rows with Pages greater than Page Num" : merged_df.apply(lambda row: pages_pagenum_check(row['Pages'], row['Page_Num']) if (not string_utils.is_empty(row['Pages']) and string_utils.is_empty(row['Page_Num'])) else False, axis=1).sum(),
|
||||
"# of Payer Names found in Notice to Provider Name" : ac_df.apply(lambda row: row['PAYER NAME'] in row['Notice to Provider Name'] if not string_utils.is_empty(row['Notice to Provider Name']) and not string_utils.is_empty(row['PAYER NAME']) else False, axis = 1).sum(),
|
||||
"# of invalid Y/N values for B run" : b_df[yn_cols_b].apply(lambda x: yn_check(x)[1], axis = 1).sum().sum(),
|
||||
"# of invalid Y/N values for AC run" : ac_df[yn_cols_ac].apply(lambda x: yn_check(x)[1], axis = 1).sum().sum(),
|
||||
"# of invalid Termination Dates" : ac_df['Termination Date'].apply(lambda x: not date_format_check(x) if not string_funcs.is_empty(x) else x).sum(),
|
||||
"# of invalid Conract Effective Dates" : ac_df['Contract Effective Date'].apply(lambda x: not date_format_check(x) if not string_funcs.is_empty(x) else x).sum()
|
||||
"# of invalid Termination Dates" : ac_df['Termination Date'].apply(lambda x: not date_format_check(x) if not string_utils.is_empty(x) else x).sum(),
|
||||
"# of invalid Conract Effective Dates" : ac_df['Contract Effective Date'].apply(lambda x: not date_format_check(x) if not string_utils.is_empty(x) else x).sum()
|
||||
}
|
||||
|
||||
ac_stats = {}
|
||||
@@ -319,16 +318,16 @@ def perform_qc_qa_tests_ac(ac_df, input_dict, processed_files):
|
||||
"# of fields for AC run" : len(ac_df.columns),
|
||||
"Highly Blank Fields in AC DataFrame" : blanks_check(ac_df),
|
||||
"Potential 'Self-Talk' columns in merged DataFrame" : self_talk_check(ac_df),
|
||||
"# of incorrectly formatted Provider Group TINs" : ac_df["IRS #"].apply(lambda x: not tin_check(x)[1] if not string_funcs.is_empty(x) else False).sum(),
|
||||
"# of incorrectly formatted Provider Group NPIs" : ac_df["NPI (10-digits)"].apply(lambda x: not npi_check(x)[1] if not string_funcs.is_empty(x) else False).sum(),
|
||||
"# of incorrectly formatted Provider Group Signatory TINs": ac_df["PROV_GROUP_TIN_SIGNATORY"].apply(lambda x: not tin_check(x)[1] if not string_funcs.is_empty(x) else False).sum(),
|
||||
"# of incorrectly formatted Provider TINs (other)" : ac_df["PROV_TIN_OTHER"].apply(lambda x: not tin_check(x)[1] if not string_funcs.is_empty(x) else False).sum(),
|
||||
"# of incorrectly formatted Provider NPIs (other)": ac_df["PROV_NPI_OTHER"].apply(lambda x: not npi_check(x)[1] if not string_funcs.is_empty(x) else False).sum(),
|
||||
"# of incorrect Health Plan States" : ac_df['Health Plan State'].apply(lambda x: not state_check(x)[1] if not string_funcs.is_empty(x) else False).sum(),
|
||||
"# of Payer Names found in Notice to Provider Name" : ac_df.apply(lambda row: row['PAYER NAME'] in row['Notice to Provider Name'] if not string_funcs.is_empty(row['Notice to Provider Name']) else False, axis = 1).sum(),
|
||||
"# of incorrectly formatted Provider Group TINs" : ac_df["IRS #"].apply(lambda x: not tin_check(x)[1] if not string_utils.is_empty(x) else False).sum(),
|
||||
"# of incorrectly formatted Provider Group NPIs" : ac_df["NPI (10-digits)"].apply(lambda x: not npi_check(x)[1] if not string_utils.is_empty(x) else False).sum(),
|
||||
"# of incorrectly formatted Provider Group Signatory TINs": ac_df["PROV_GROUP_TIN_SIGNATORY"].apply(lambda x: not tin_check(x)[1] if not string_utils.is_empty(x) else False).sum(),
|
||||
"# of incorrectly formatted Provider TINs (other)" : ac_df["PROV_TIN_OTHER"].apply(lambda x: not tin_check(x)[1] if not string_utils.is_empty(x) else False).sum(),
|
||||
"# of incorrectly formatted Provider NPIs (other)": ac_df["PROV_NPI_OTHER"].apply(lambda x: not npi_check(x)[1] if not string_utils.is_empty(x) else False).sum(),
|
||||
"# of incorrect Health Plan States" : ac_df['Health Plan State'].apply(lambda x: not state_check(x)[1] if not string_utils.is_empty(x) else False).sum(),
|
||||
"# of Payer Names found in Notice to Provider Name" : ac_df.apply(lambda row: row['PAYER NAME'] in row['Notice to Provider Name'] if not string_utils.is_empty(row['Notice to Provider Name']) else False, axis = 1).sum(),
|
||||
"# of invalid Y/N values for AC run" : ac_df[yn_cols_ac].apply(lambda x: yn_check(x)[1], axis = 1).sum().sum(),
|
||||
"# of invalid Termination Dates" : ac_df['Termination Date'].apply(lambda x: not date_format_check(x) if not string_funcs.is_empty(x) else x).sum(),
|
||||
"# of invalid Conract Effective Dates" : ac_df['Contract Effective Date'].apply(lambda x: not date_format_check(x) if not string_funcs.is_empty(x) else x).sum()
|
||||
"# of invalid Termination Dates" : ac_df['Termination Date'].apply(lambda x: not date_format_check(x) if not string_utils.is_empty(x) else x).sum(),
|
||||
"# of invalid Conract Effective Dates" : ac_df['Contract Effective Date'].apply(lambda x: not date_format_check(x) if not string_utils.is_empty(x) else x).sum()
|
||||
}
|
||||
|
||||
ac_stats = {}
|
||||
@@ -404,7 +403,7 @@ def get_table_stats(text_dict):
|
||||
table_count, num_table_pages, rate_count = 0, 0, 0
|
||||
table_pages, rate_pages, rates_more_than_10 = [], [], []
|
||||
for page_num, page_text in text_dict.items():
|
||||
if string_funcs.contains_reimbursement(page_text, page_num):
|
||||
if string_utils.contains_reimbursement(page_text, page_num):
|
||||
rate_pages.append(page_num)
|
||||
rates_on_page = sum(1 for char in page_text if char in {"$", "%"})
|
||||
rate_count += rates_on_page
|
||||
@@ -1,6 +1,6 @@
|
||||
import re
|
||||
|
||||
import postprocessing_funcs
|
||||
from src import postprocessing_funcs
|
||||
|
||||
|
||||
#################################################################################################################################################
|
||||
@@ -1,12 +1,9 @@
|
||||
import re
|
||||
|
||||
import claude_funcs
|
||||
import config
|
||||
import keywords
|
||||
import postprocessing_funcs
|
||||
import prompts
|
||||
import string_funcs
|
||||
from enums.delimiters import Delimiter
|
||||
import src.utils.llm_utils as llm_utils
|
||||
import src.utils.string_utils as string_utils
|
||||
from src import config, keywords, postprocessing_funcs, prompts
|
||||
from src.enums.delimiters import Delimiter
|
||||
|
||||
"""
|
||||
Date pattern regex:
|
||||
@@ -99,7 +96,7 @@ def run_smart_chunk_prompts(
|
||||
prompt_agreement = prompts.AC_SINGLE_FIELD_TEMPLATE(
|
||||
context_agreement, question
|
||||
)
|
||||
field_group_answer_raw = claude_funcs.invoke_claude(
|
||||
field_group_answer_raw = llm_utils.invoke_claude(
|
||||
prompt_agreement,
|
||||
config.MODEL_ID_CLAUDE35_SONNET,
|
||||
filename,
|
||||
@@ -108,18 +105,18 @@ def run_smart_chunk_prompts(
|
||||
|
||||
# if answer is N/A use the smart-chunked context
|
||||
if field_group_answer_raw == "N/A":
|
||||
field_group_answer_raw = claude_funcs.invoke_claude(
|
||||
field_group_answer_raw = llm_utils.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 8192
|
||||
)
|
||||
else:
|
||||
field_group_answer_raw = claude_funcs.invoke_claude(
|
||||
field_group_answer_raw = llm_utils.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 8192
|
||||
)
|
||||
|
||||
if len(fields) == 1:
|
||||
if fields[0] == "CONTRACT_EFFECTIVE_DT":
|
||||
# print(f'Effective Date Answer obtained from LLM: {field_group_answer_raw}')
|
||||
field_group_answer_raw = string_funcs.extract_text_from_delimiters(
|
||||
field_group_answer_raw = string_utils.extract_text_from_delimiters(
|
||||
field_group_answer_raw, Delimiter.PIPE
|
||||
)
|
||||
# print(f'Parsed Effective Date Answer (extract_text_from_delimiters): {field_group_answer_raw}')
|
||||
@@ -138,7 +135,7 @@ def run_smart_chunk_prompts(
|
||||
field_group_answer_raw = field_group_answer_raw.replace(
|
||||
"_DATE", "_DT"
|
||||
)
|
||||
field_group_answer_dict = string_funcs.json_parsing_search(
|
||||
field_group_answer_dict = string_utils.json_parsing_search(
|
||||
field_group_answer_raw, fields
|
||||
)
|
||||
|
||||
@@ -207,7 +204,7 @@ def run_full_context_prompts(
|
||||
)
|
||||
|
||||
try:
|
||||
full_context_answers = claude_funcs.get_ac_answer(
|
||||
full_context_answers = llm_utils.get_ac_answer(
|
||||
full_context_prompt,
|
||||
filename,
|
||||
fields=list(full_context_questions.keys()),
|
||||
@@ -237,7 +234,7 @@ def run_ac_conditional_prompts(
|
||||
nrd_prompt = prompts.AC_SINGLE_FIELD_TEMPLATE(
|
||||
ac_answers_dict["NON_RENEWAL_LANGUAGE"], prompts.AC_DICT["NON_RENEWAL_DAYS"]
|
||||
)
|
||||
nrd_answer = claude_funcs.invoke_claude(
|
||||
nrd_answer = llm_utils.invoke_claude(
|
||||
nrd_prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 8192
|
||||
)
|
||||
ac_answers_dict["NON_RENEWAL_DAYS"] = nrd_answer
|
||||
@@ -246,13 +243,13 @@ def run_ac_conditional_prompts(
|
||||
|
||||
# Clean Up Contract Effective Date
|
||||
if (
|
||||
string_funcs.is_empty(ac_answers_dict["CONTRACT_EFFECTIVE_DT"])
|
||||
string_utils.is_empty(ac_answers_dict["CONTRACT_EFFECTIVE_DT"])
|
||||
and "meridian" in ac_answers_dict["PAYER_NAME"].lower()
|
||||
):
|
||||
date_prompt = prompts.AC_EFFECTIVE_DATE_CLEANUP(
|
||||
contract_text[0 : min(100000, len(contract_text)) - 1]
|
||||
)
|
||||
date_answer = claude_funcs.invoke_claude(
|
||||
date_answer = llm_utils.invoke_claude(
|
||||
date_prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 124
|
||||
)
|
||||
ac_answers_dict["CONTRACT_EFFECTIVE_DT"] = date_answer
|
||||
|
||||
+1
-1
@@ -7,7 +7,7 @@ from pathlib import Path
|
||||
import boto3
|
||||
import pandas as pd
|
||||
|
||||
import config
|
||||
from src import config
|
||||
|
||||
|
||||
class BatchTracker:
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
import pandas as pd
|
||||
|
||||
import config
|
||||
from src import config
|
||||
|
||||
|
||||
def aggregate_costlog(df: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame]:
|
||||
@@ -6,7 +6,7 @@ from datetime import datetime
|
||||
|
||||
import pandas as pd
|
||||
|
||||
import config
|
||||
from src import config
|
||||
|
||||
|
||||
def write_stats_to_csv(filename=config.TRACKING_FILE):
|
||||
@@ -5,8 +5,8 @@ import pandas as pd
|
||||
from botocore.exceptions import ClientError
|
||||
from pyxlsb import open_workbook
|
||||
|
||||
import config
|
||||
import tracking
|
||||
import src.tracking.tracking_utils as tracking_utils
|
||||
from src import config
|
||||
|
||||
|
||||
def read_local(file_path): # io_utils.py
|
||||
@@ -137,7 +137,7 @@ def filter_already_processed(input_dict, input_dict_b): # io_utils.py
|
||||
"""Filter out already processed files based on either local files or S3 master tracking"""
|
||||
if config.WRITE_TO_S3:
|
||||
# Get processed files from S3 master tracking for this specific batch
|
||||
already_processed_ac, already_processed_b = tracking.get_already_processed_files(config.BATCH_ID)
|
||||
already_processed_ac, already_processed_b = tracking_utils.get_already_processed_files(config.BATCH_ID)
|
||||
|
||||
# Keep files that still need either AC or B processing
|
||||
input_dict_ac = {
|
||||
@@ -8,8 +8,8 @@ import time
|
||||
import anthropic
|
||||
from botocore.exceptions import ClientError, ReadTimeoutError
|
||||
|
||||
import config
|
||||
import string_funcs
|
||||
import src.utils.string_utils as string_utils
|
||||
from src import config
|
||||
|
||||
|
||||
def update_stats(filename, tokens_sent, tokens_received, elapsed_time, model_type):
|
||||
@@ -434,5 +434,5 @@ def get_ac_answer(prompt, filename, fields):
|
||||
)
|
||||
response_text = response_text.replace("_DATE", "_DT")
|
||||
# response_dict = dict_operations.secondary_string_to_dict(response_text, filename)
|
||||
response_dict = string_funcs.json_parsing_search(response_text, fields)
|
||||
response_dict = string_utils.json_parsing_search(response_text, fields)
|
||||
return response_dict
|
||||
@@ -5,12 +5,11 @@ import warnings
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
import claude_funcs
|
||||
import config
|
||||
import prompts
|
||||
from enums.delimiters import Delimiter
|
||||
from regex_patterns import (BACKTICK_PATTERN, PIPE_PATTERN,
|
||||
TRIPLE_BACKTICK_PATTERN)
|
||||
import src.utils.llm_utils as llm_utils
|
||||
from src import config, prompts
|
||||
from src.enums.delimiters import Delimiter
|
||||
from src.regex.regex_patterns import (BACKTICK_PATTERN, PIPE_PATTERN,
|
||||
TRIPLE_BACKTICK_PATTERN)
|
||||
|
||||
|
||||
def extract_text_from_delimiters(
|
||||
@@ -162,7 +161,7 @@ def secondary_string_to_dict(dict_string, filename):
|
||||
except:
|
||||
try:
|
||||
prompt = prompts.FIX_JSON(dict_substring)
|
||||
dict_substring = claude_funcs.invoke_claude(
|
||||
dict_substring = llm_utils.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE3_HAIKU, filename
|
||||
)
|
||||
result_dict = json.loads(dict_substring)
|
||||
@@ -1,14 +1,13 @@
|
||||
import claude_funcs
|
||||
import config
|
||||
import prompts
|
||||
import string_funcs
|
||||
import src.utils.llm_utils as llm_utils
|
||||
import src.utils.string_utils as string_utils
|
||||
from src import config, prompts
|
||||
|
||||
|
||||
def align_and_format_tables(text_dict, filename):
|
||||
for page_num, page_text in text_dict.items():
|
||||
if "Table Start" in page_text and string_funcs.contains_reimbursement(page_text):
|
||||
if "Table Start" in page_text and string_utils.contains_reimbursement(page_text):
|
||||
prompt = prompts.ALIGN_AND_FORMAT_TABLES(page_text)
|
||||
aligned_page = claude_funcs.invoke_claude(
|
||||
aligned_page = llm_utils.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 8192
|
||||
)
|
||||
text_dict[page_num] = aligned_page
|
||||
@@ -1,7 +1,8 @@
|
||||
import pytest
|
||||
import re
|
||||
|
||||
from smart_chunking_funcs import DATE_PATTERN
|
||||
import pytest
|
||||
|
||||
from src.smart_chunking_funcs import DATE_PATTERN
|
||||
|
||||
legal_traditional_dates = [
|
||||
"This 12th day of December, 2024",
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from tracking import calculate_progress_stats
|
||||
|
||||
from src.tracking.tracking_utils import calculate_progress_stats
|
||||
|
||||
|
||||
class TestCalculateProgressStats(unittest.TestCase):
|
||||
def test_normal_case(self):
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
from claude_funcs import count_tokens
|
||||
|
||||
def test_count_tokens():
|
||||
assert count_tokens("") == 0
|
||||
assert count_tokens("a") == 1
|
||||
@@ -1,6 +1,7 @@
|
||||
import pytest
|
||||
import math
|
||||
|
||||
import pandas as pd
|
||||
import pytest
|
||||
from pandas.testing import assert_frame_equal
|
||||
|
||||
# from scripts.cnc_hotfixes.hotfix_helper_funcs import state_check
|
||||
@@ -203,4 +204,4 @@ from pandas.testing import assert_frame_equal
|
||||
# df = pd.DataFrame({'Health Plan State': ['NY', 'CA']})
|
||||
|
||||
# with pytest.raises(ValueError):
|
||||
# apply_state_check_to_df(df,is_dataframe=False)
|
||||
# apply_state_check_to_df(df,is_dataframe=False)
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import pytest
|
||||
from costlog_funcs import aggregate_costlog
|
||||
import pandas as pd
|
||||
import pytest
|
||||
|
||||
from src.tracking.costlog_utils import aggregate_costlog
|
||||
|
||||
|
||||
def test_missing_columns():
|
||||
with pytest.raises(ValueError):
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import pytest
|
||||
|
||||
# from hotfix_helper_funcs import apply_date_formatting_fix
|
||||
|
||||
# def test_change_date_formatting():
|
||||
@@ -18,4 +19,4 @@ import pytest
|
||||
# with pytest.raises(ValueError):
|
||||
# apply_date_formatting_fix("7/20/1992")
|
||||
# with pytest.raises(ValueError):
|
||||
# apply_date_formatting_fix("some string")
|
||||
# apply_date_formatting_fix("some string")
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
from src.utils.llm_utils import count_tokens
|
||||
|
||||
|
||||
def test_count_tokens():
|
||||
assert count_tokens("") == 0
|
||||
assert count_tokens("a") == 1
|
||||
@@ -1,6 +1,7 @@
|
||||
import pandas as pd
|
||||
import os
|
||||
|
||||
import pandas as pd
|
||||
|
||||
fn_batch_1_clean_abc = 'reference_files/CNC-Batch 1 Output_File1.xlsx'
|
||||
df_batch_1_clean_abc = pd.read_excel(fn_batch_1_clean_abc)
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import pytest
|
||||
from postprocessing_funcs import convert_to_us_date_format, InvalidDateException
|
||||
|
||||
from src.postprocessing_funcs import (InvalidDateException,
|
||||
convert_to_us_date_format)
|
||||
|
||||
|
||||
class TestConvertToUSDateFormat:
|
||||
|
||||
+3
-4
@@ -1,8 +1,7 @@
|
||||
import pytest
|
||||
import pandas as pd
|
||||
import sys
|
||||
sys.path.append('src')
|
||||
from qa_qc_helpers import blanks_check
|
||||
import pytest
|
||||
|
||||
from src.qa_qc.qa_qc_utils import blanks_check
|
||||
|
||||
|
||||
def test_blanks_check_no_highly_blank_columns():
|
||||
+2
-4
@@ -1,8 +1,6 @@
|
||||
import pytest
|
||||
from datetime import datetime
|
||||
import sys
|
||||
sys.path.append('src')
|
||||
from qa_qc_helpers import date_format_check
|
||||
from src.qa_qc.qa_qc_utils import date_format_check
|
||||
|
||||
|
||||
def test_valid_date_formats():
|
||||
"""Test valid date formats are correctly validated"""
|
||||
@@ -1,8 +1,6 @@
|
||||
import pytest
|
||||
import sys
|
||||
sys.path.append('src')
|
||||
from qa_qc_helpers import npi_check
|
||||
import re
|
||||
import pytest
|
||||
from src.qa_qc.qa_qc_utils import npi_check
|
||||
|
||||
|
||||
def test_valid_npi_format():
|
||||
"""Test that a correctly formatted NPI returns True"""
|
||||
+3
-3
@@ -1,7 +1,7 @@
|
||||
import pytest
|
||||
import sys
|
||||
sys.path.append('src')
|
||||
from qa_qc_helpers import pages_pagenum_check
|
||||
|
||||
from src.qa_qc.qa_qc_utils import pages_pagenum_check
|
||||
|
||||
|
||||
def test_pages_pagenum_check_valid_inputs():
|
||||
# Test with valid string inputs
|
||||
@@ -0,0 +1,44 @@
|
||||
### These tests (`selftalk_test.py`) were previously saved as just `selftalk_test` (no extension). As such, they were previously not running; when I added the extension they started failing. I will leave these commented until a later date.
|
||||
### - Alex 2025-01-08
|
||||
|
||||
# import pytest
|
||||
# import pandas as pd
|
||||
# import sys
|
||||
# sys.path.append('src')
|
||||
# from qa_qc.qa_qc_utils import self_talk_check
|
||||
|
||||
|
||||
# def test_self_talk_check_no_self_talk():
|
||||
# # Create a sample DataFrame with no self-talk columns
|
||||
# df = pd.DataFrame({
|
||||
# "Name": ["John", "Jane", "Bob", "Alice", "Tom"],
|
||||
# "Age": [25, 30, 35, 40, 45],
|
||||
# "City": ["New York", "Los Angeles", "Chicago", "Houston", "Miami"]
|
||||
# })
|
||||
|
||||
# result = self_talk_check(df)
|
||||
# assert result == ""
|
||||
|
||||
# def test_self_talk_check_with_self_talk():
|
||||
# # Create a sample DataFrame with self-talk columns
|
||||
# df = pd.DataFrame({
|
||||
# "Name": ["John", "Jane", "Bob", "Alice", "Tom"],
|
||||
# "Language": ["English", "Spanish", "The Contract Text suggests Armenian", "French", "German"],
|
||||
# "Clause": ["", "this suggests", "Mike refers to xyz", "The Contract Text says", "My Answer is this: "]
|
||||
# })
|
||||
|
||||
# result = self_talk_check(df)
|
||||
# expected_output = "Language - 20.0%\nClause - 80.0%"
|
||||
# assert result == expected_output
|
||||
|
||||
# def test_self_talk_check_all_self_talk():
|
||||
# # Create a sample DataFrame where all language columns have self-talk
|
||||
# df = pd.DataFrame({
|
||||
# "Name": ["John", "Jane", "Bob", "Alice", "Tom"],
|
||||
# "Language": ["Statewide", "Multiple sources imply", "The contract text says", "For this text, it would seem", "This suggests that"],
|
||||
# "Clause": ["The Contract Text contains this", "My Answer Is This: ", "It would seem that multiple clauses say", "I conclude that the exhibit suggests", "For each clause, this is the case"]
|
||||
# })
|
||||
|
||||
# result = self_talk_check(df)
|
||||
# expected_output = "Language - 100.0%\nClause - 100.0%"
|
||||
# assert result == expected_output
|
||||
+2
-3
@@ -1,7 +1,6 @@
|
||||
import pytest
|
||||
import sys
|
||||
sys.path.append('src')
|
||||
from qa_qc_helpers import state_check
|
||||
|
||||
from src.qa_qc.qa_qc_utils import state_check
|
||||
|
||||
|
||||
def test_state_check_valid_abbreviations():
|
||||
@@ -1,8 +1,7 @@
|
||||
import pytest
|
||||
import sys
|
||||
sys.path.append('src')
|
||||
from qa_qc_helpers import tin_check
|
||||
import re
|
||||
import pytest
|
||||
|
||||
from src.qa_qc.qa_qc_utils import tin_check
|
||||
|
||||
|
||||
def test_valid_tin_format():
|
||||
"""Test that a correctly formatted TIN returns True"""
|
||||
@@ -1,7 +1,7 @@
|
||||
import pytest
|
||||
import sys
|
||||
sys.path.append('src')
|
||||
from qa_qc_helpers import yn_check
|
||||
|
||||
from src.qa_qc.qa_qc_utils import yn_check
|
||||
|
||||
|
||||
def test_yn_check_with_valid_string_input():
|
||||
# Test with valid string inputs
|
||||
@@ -1,42 +0,0 @@
|
||||
import pytest
|
||||
import pandas as pd
|
||||
import sys
|
||||
sys.path.append('src')
|
||||
from qa_qc_helpers import self_talk_check
|
||||
from hotfix_helper_funcs import keywords_to_detect_invalid_state_name
|
||||
|
||||
|
||||
def test_self_talk_check_no_self_talk():
|
||||
# Create a sample DataFrame with no self-talk columns
|
||||
df = pd.DataFrame({
|
||||
"Name": ["John", "Jane", "Bob", "Alice", "Tom"],
|
||||
"Age": [25, 30, 35, 40, 45],
|
||||
"City": ["New York", "Los Angeles", "Chicago", "Houston", "Miami"]
|
||||
})
|
||||
|
||||
result = self_talk_check(df)
|
||||
assert result == ""
|
||||
|
||||
def test_self_talk_check_with_self_talk():
|
||||
# Create a sample DataFrame with self-talk columns
|
||||
df = pd.DataFrame({
|
||||
"Name": ["John", "Jane", "Bob", "Alice", "Tom"],
|
||||
"Language": ["English", "Spanish", "The Contract Text suggests Armenian", "French", "German"],
|
||||
"Clause": ["", "this suggests", "Mike refers to xyz", "The Contract Text says", "My Answer is this: "]
|
||||
})
|
||||
|
||||
result = self_talk_check(df)
|
||||
expected_output = "Language - 20.0%\nClause - 80.0%"
|
||||
assert result == expected_output
|
||||
|
||||
def test_self_talk_check_all_self_talk():
|
||||
# Create a sample DataFrame where all language columns have self-talk
|
||||
df = pd.DataFrame({
|
||||
"Name": ["John", "Jane", "Bob", "Alice", "Tom"],
|
||||
"Language": ["Statewide", "Multiple sources imply", "The contract text says", "For this text, it would seem", "This suggests that"],
|
||||
"Clause": ["The Contract Text contains this", "My Answer Is This: ", "It would seem that multiple clauses say", "I conclude that the exhibit suggests", "For each clause, this is the case"]
|
||||
})
|
||||
|
||||
result = self_talk_check(df)
|
||||
expected_output = "Language - 100.0%\nClause - 100.0%"
|
||||
assert result == expected_output
|
||||
+3
-2
@@ -1,6 +1,7 @@
|
||||
import pytest
|
||||
from enums.delimiters import Delimiter
|
||||
from string_funcs import extract_text_from_delimiters
|
||||
|
||||
from src.enums.delimiters import Delimiter
|
||||
from src.utils.string_utils import extract_text_from_delimiters
|
||||
|
||||
|
||||
class TestExtractTextFromDelimiters:
|
||||
@@ -1,6 +1,7 @@
|
||||
|
||||
import pytest
|
||||
from preprocessing_funcs import smart_chunk_ac
|
||||
|
||||
from src.preprocessing_funcs import smart_chunk_ac
|
||||
|
||||
test_chunk_dict_term = {
|
||||
"1": "NA",
|
||||
|
||||
Reference in New Issue
Block a user