diff --git a/fieldExtraction/src/investment/code_funcs.py b/fieldExtraction/src/investment/code_funcs.py index ca02266..58fa3d8 100644 --- a/fieldExtraction/src/investment/code_funcs.py +++ b/fieldExtraction/src/investment/code_funcs.py @@ -381,8 +381,13 @@ def get_code_description(code, code_mapping): description = code_mapping.mapping.get(expanded_code_range, "mismap") # If mismap, try to handle numeric codes if string_utils.is_empty(description) or description == "mismap": - x_range_code_list = [int(f"{base_code}{i}") for i in range(10)] - description = get_code_description(x_range_code_list, code_mapping) + try: + x_range_code_list = [int(f"{base_code}{i}") for i in range(10)] + description = get_code_description(x_range_code_list, code_mapping) + except Exception as e: + print(f"Error message: {e}") + description = "N/A" + elif "-" in code_str and all(part.isdigit() for part in code_str.split("-")): try: diff --git a/fieldExtraction/src/investment/file_processing.py b/fieldExtraction/src/investment/file_processing.py index c270b19..8dcf213 100644 --- a/fieldExtraction/src/investment/file_processing.py +++ b/fieldExtraction/src/investment/file_processing.py @@ -95,6 +95,7 @@ def run_one_to_n_prompts(filename, text_dict, exhibit_pages, exhibit_chunk_mappi ################## RUN PROMPTS ################## one_to_n_results = [] + # print(f"Total number of exhibits in {filename}: {len(exhibit_pages)}") for exhibit_page in exhibit_pages: exhibit_chunk = string_utils.get_exhibit_chunk(text_dict, exhibit_chunk_mapping, exhibit_page) if string_utils.contains_reimbursement(exhibit_chunk): diff --git a/fieldExtraction/src/investment/one_to_n_funcs.py b/fieldExtraction/src/investment/one_to_n_funcs.py index 1d621fe..f8d6bd4 100644 --- a/fieldExtraction/src/investment/one_to_n_funcs.py +++ b/fieldExtraction/src/investment/one_to_n_funcs.py @@ -15,6 +15,7 @@ import src.investment.carveout_funcs as carveout_funcs import src.investment.special_case_funcs as special_case_funcs from src.config import FIELD_JSON_PATH from src.regex.regex_utils import find_regex_matches +from src.utils.string_utils import datetime_str def get_exhibit_header(exhibit_chunk, filename): """ @@ -227,7 +228,7 @@ def get_methodology_breakout(reimbursement_primary_answers: list[dict], filename try: methodology_breakout_answer = string_utils.universal_json_load(llm_response) except ValueError as e: - print(f"universal_string_load failed for input {llm_response} with error {e}\nassigning methodology_breakout_answer to []") + # print(f"universal_string_load failed for input {llm_response} with error {e}\nassigning methodology_breakout_answer to []") methodology_breakout_answer = [] for methodology_breakout_dict in methodology_breakout_answer: # Fee Schedule Breakout @@ -360,14 +361,18 @@ def reimbursement_level(exhibit_text, filename, reimbursement_level_fields, all_ dict: The parsed LLM response as a list of dictionaries with unique identifiers """ reimbursement_primary_answers = get_reimbursement_primary(reimbursement_level_fields, exhibit_text, filename) # Returns list of dicts - + + # print(f"{datetime_str()} reimbursement_primary_answers complete for exhibit page {exhibit_page} of {filename}...") if reimbursement_primary_answers: # If any reimbursements found carveout_answers = get_special_cases(reimbursement_primary_answers, filename) + # print(f"{datetime_str()} carveout_answers complete for exhibit page {exhibit_page} of {filename}...") # Later, we will only run methodology breakout for those without certain carveout indicators. We will run carveout-specific breakouts for the others methodology_breakout_answers = get_methodology_breakout(carveout_answers, filename) - # service_breakout_answers = get_service_breakout(methodology_breakout_answers, filename) # Not using as of now - keep temporarily while field initialization is finishing. + # print(f"{datetime_str()} methodology_breakout_answers complete for exhibit page {exhibit_page} of {filename}...") + code_breakout_answers = code_funcs.get_code_breakout(methodology_breakout_answers, filename, all_dataset) + # print(f"{datetime_str()} code_breakout_answers complete for exhibit page {exhibit_page} of {filename}...") # Add unique identifiers to each row composed of filename, exhibit page, service code, and index for idx, row in enumerate(code_breakout_answers): diff --git a/fieldExtraction/src/investment/table_funcs.py b/fieldExtraction/src/investment/table_funcs.py index dd5836a..574e64d 100644 --- a/fieldExtraction/src/investment/table_funcs.py +++ b/fieldExtraction/src/investment/table_funcs.py @@ -305,7 +305,7 @@ def split_long_tables(text_dict, table_dict, row_limit): subpage_index = 0 for table in tables: - if len(table.table[list(table.table.keys())[0]]) > row_limit: + if table.table and len(table.table[list(table.table.keys())[0]]) > row_limit: split_tables = table.split(row_limit) for split_table in split_tables: new_page_num = f"{page_num}.{subpage_index}" diff --git a/fieldExtraction/src/preprocessing_funcs.py b/fieldExtraction/src/preprocessing_funcs.py index 1b05009..cd567d8 100644 --- a/fieldExtraction/src/preprocessing_funcs.py +++ b/fieldExtraction/src/preprocessing_funcs.py @@ -22,7 +22,10 @@ def remove_page_indicators(contract_text: str) -> str: str: cleaned text with newlines and page number indicators removed """ - cleaned_text = re.sub(r"Page [0-9]+ of [0-9]+\n\n", " ", contract_text) # TODO: fix in main branch + if contract_text: + cleaned_text = re.sub(r"Page [0-9]+ of [0-9]+\n\n", " ", contract_text) + else: + cleaned_text = contract_text return cleaned_text diff --git a/fieldExtraction/src/utils/string_utils.py b/fieldExtraction/src/utils/string_utils.py index 391963b..29b242a 100644 --- a/fieldExtraction/src/utils/string_utils.py +++ b/fieldExtraction/src/utils/string_utils.py @@ -226,13 +226,15 @@ def primary_string_to_dict(string_dict, filename): def universal_json_load(string_dict): """Matches json dicts and list of dicts - NOT simple lists""" - match = re.search(r'\{.*\}|\[\{.*\}\]', string_dict, re.DOTALL) + match = re.search(r'\{.*\}|\[\s*?\{.*\}\s*?\]', string_dict, re.DOTALL) # Updated regex if match: try: return json.loads(match.group()) except json.JSONDecodeError: pass - raise ValueError("No valid JSON object found in the input string") + raise ValueError("No valid JSON object found in the input string") + else: + return {} reimbursement_strings = [ # These are for `method='keyword'` "%",