From 47a01e13f0ce3d833ae108f9f4ba47752e123e54 Mon Sep 17 00:00:00 2001 From: Alex Galarce Date: Thu, 5 Dec 2024 22:46:41 +0000 Subject: [PATCH] Merged in chore/contract_effective_date (pull request #307) New Prompt Method (starting with contract effective date) * add todos for Siddhant * moved effective date prompts to prompts.py and added docstring and type hints * moved effective date prompts to prompts.py and added docstring and type hints * fixed conflict Co-authored-by: Alex Galarce * modified contract effective date logic in existing smart chunking * Merged main into chore/contract_effective_date * updated extract_text_from_delimiters function and unit tests for it * Merge remote-tracking branch 'origin/main' into chore/contract_effective_date * added comments to preprocessing_funcs * preprocessing_funcs.py edited online with Bitbucket * utils.py edited online with Bitbucket * add debug line * add NA handling into fix_date_formatting() * add debug print statements * worked on review comments * fixed date checking * fixed date checking and added unit tests * Merged main into chore/contract_effective_date * added logic to handle case when smart chunking returns empty context for effective date * added comments to file_processing.py * optimized imports and removed a TODO * Merged main into chore/contract_effective_date * made prompt changes for effective date * keywords.py edited online with Bitbucket * Removed excess print statements Approved-by: Katon Minhas --- fieldExtraction/src/ac_funcs.py | 61 +++++++++++++++++-- fieldExtraction/src/batch_tracking.py | 1 - fieldExtraction/src/cnc_hotfix.py | 25 +++++--- .../src/cnc_hotfix_effective_date_utils.py | 41 +------------ fieldExtraction/src/detect_complex.py | 1 - fieldExtraction/src/enums/__init__.py | 0 fieldExtraction/src/enums/delimiters.py | 6 ++ fieldExtraction/src/file_processing.py | 40 ++++++++++-- fieldExtraction/src/keywords.py | 8 +-- fieldExtraction/src/preprocessing_funcs.py | 21 ++++++- fieldExtraction/src/prompts.py | 49 ++++++++++++++- fieldExtraction/src/regex_patterns.py | 3 + fieldExtraction/src/table_funcs.py | 3 - fieldExtraction/src/tin_pull.py | 2 - fieldExtraction/src/utils.py | 51 +++++++++++++++- fieldExtraction/tests/ac_funcs_test.py | 52 ++++++++++++++++ fieldExtraction/tests/utils_test.py | 43 +++++++++++++ 17 files changed, 332 insertions(+), 75 deletions(-) create mode 100644 fieldExtraction/src/enums/__init__.py create mode 100644 fieldExtraction/src/enums/delimiters.py create mode 100644 fieldExtraction/src/regex_patterns.py create mode 100644 fieldExtraction/tests/ac_funcs_test.py create mode 100644 fieldExtraction/tests/utils_test.py diff --git a/fieldExtraction/src/ac_funcs.py b/fieldExtraction/src/ac_funcs.py index f6f39ea..356db66 100644 --- a/fieldExtraction/src/ac_funcs.py +++ b/fieldExtraction/src/ac_funcs.py @@ -13,12 +13,15 @@ from botocore.config import Config import traceback import time import sys - +import warnings +from enums.delimiters import Delimiter + import claude_funcs import dict_operations import config import utils import prompts +from regex_patterns import PIPE_PATTERN, BACKTICK_PATTERN, TRIPLE_BACKTICK_PATTERN def get_ac_answer(prompt, filename, fields): @@ -101,9 +104,9 @@ def select_context(text_dict, field): keywords = ["Provider Manual", "Policies and Procedures"] if field == "ARBITRATION_AND_DISPUTES": keywords = ["Informal Dispute Resolution", "Arbitration"] - print(field) + page_list = [] - print(keywords) + for keyword in keywords: for page in text_dict: if keyword.lower() in text_dict[page].lower(): @@ -135,7 +138,6 @@ def create_prompt(context, question): def json_parsing(response_text, context): - print("Parsing JSON response.") try: if response_text.split("{", 1)[1].strip()[0] == '"': response_text = "{" + response_text.split("{", 1)[1] @@ -249,10 +251,59 @@ def json_parsing(response_text, context): ] snippet_l = [snippet.strip() for snippet in snippet_l] - print(f"Parsed fields: {field_l}") return field_l, answer_l, page_no_l, location_l, snippet_l +def extract_text_from_delimiters(raw_output: str, delimiter: Delimiter, match_index: int=-1) -> str: + """ + Extracts a match enclosed in specified delimiters from the raw output. + This function searches for text enclosed by a specified delimiter in the given raw output string. + It returns the match specified by the `which_match` index. If no match is found, it returns "N/A". + + - raw_output (str): The raw output string to search within. + - which_match (int, optional): The index of the match to return. Defaults to -1, which returns the last match. If the index is out of range, the last match is returned. + - str: The extracted text or "N/A" if no match is found. + + Raises: + - TypeError: If `raw_output` is not a string or `delimiter` is not a Delimiter enum value. + - ValueError: If an unsupported delimiter is provided. + + Returns: + - str: The extracted answer or "N/A" if no match is found. + """ + if not isinstance(raw_output, str): + raise TypeError("Expected a string for raw_output, got {0}.".format(type(raw_output).__name__)) + + if not isinstance(delimiter, Delimiter): + raise TypeError("Expected a Delimiter enum value, got {0}.".format(type(delimiter).__name__)) + + # Define a pattern based on the delimiter type + if delimiter == Delimiter.PIPE: + pattern = PIPE_PATTERN + elif delimiter == Delimiter.BACKTICK: + pattern = BACKTICK_PATTERN + elif delimiter == Delimiter.TRIPLE_BACKTICK: + pattern = TRIPLE_BACKTICK_PATTERN + else: + raise ValueError("Unsupported delimiter. Use one of the Delimiter enum values.") + + # Find all matches based on the pattern + matches = re.findall(pattern=pattern, string=raw_output) + + if len(matches) == 0: + return "N/A" + + # Adjust for negative indices + if match_index < 0: + match_index += len(matches) + + if not 0 <= match_index < len(matches): + warnings.warn("Index out of range. Returning the last match by default.") + match_index = -1 + + return matches[match_index] + + def json_parsing_search(response_text, field_list): try: if response_text.split("{", 1)[1].strip()[0] == '"': diff --git a/fieldExtraction/src/batch_tracking.py b/fieldExtraction/src/batch_tracking.py index e760af3..39ed452 100644 --- a/fieldExtraction/src/batch_tracking.py +++ b/fieldExtraction/src/batch_tracking.py @@ -156,7 +156,6 @@ class BatchTracker: current_batch_entries = df[df['batch_id'] == config.BATCH_ID] if current_batch_entries.empty: - print("No entry found for current batch. Creating new entry.") self.add_batch_run() return diff --git a/fieldExtraction/src/cnc_hotfix.py b/fieldExtraction/src/cnc_hotfix.py index 6a55fb8..393b287 100644 --- a/fieldExtraction/src/cnc_hotfix.py +++ b/fieldExtraction/src/cnc_hotfix.py @@ -452,12 +452,12 @@ def clean_irs(abc_df, filename, text_dict ): def contract_effective_date_fix(contract_name: str, text_dict: dict[str, str],is_meridian: bool=False) -> str: # perform smart chunking page_list = ac_smart_chunking.chunk_with_include_exclude_keywords(text_dict, - include_keywords = cnc_hotfix_effective_date_utils.included_keyword, - exclude_keywords = cnc_hotfix_effective_date_utils.excluded_keyword + include_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["included_keywords"], + exclude_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["excluded_keywords"], ) context = "\n".join(text_dict[page] for page in page_list) - prompt = cnc_hotfix_effective_date_utils.get_effective_date_prompt(context) # get effective date - this is for both meridian and non-meridian + prompt = prompts.get_effective_date_prompt(context) # get effective date - this is for both meridian and non-meridian try: claude_answer_raw = claude_funcs.invoke_claude( @@ -471,9 +471,16 @@ def contract_effective_date_fix(contract_name: str, text_dict: dict[str, str],is if is_meridian: if claude_answer_extracted == "N/A": # perform smart chunking with signature date pages included + + included_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["included_keywords"] + + included_keywords = list(itertools.chain(included_keywords, ["Signature"])) + + excluded_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["excluded_keywords"] + page_list = ac_smart_chunking.chunk_with_include_exclude_keywords(text_dict, - include_keywords = cnc_hotfix_effective_date_utils.included_keyword + ["Signature"], - exclude_keywords = cnc_hotfix_effective_date_utils.excluded_keyword + include_keywords = included_keywords, + exclude_keywords = excluded_keywords ) context = "\n".join(text_dict[page] for page in page_list) @@ -534,7 +541,6 @@ def clean_contract_effective_date( try: answer = contract_effective_date_fix(file_name, text_dict,is_meridian) except Exception as e: - # KATON: Do we want to give out "answer='Error'" in this case? print(f"Error processing {file_name}, got error: {e}") else: answer = contract_effective_date @@ -542,7 +548,12 @@ def clean_contract_effective_date( if answer == "N/A": df["Contract Effective Date_corrected"] = answer else: - df["Contract Effective Date_corrected"] = pd.to_datetime(answer, dayfirst=False).strftime('%m/%d/%Y') + try: + corrected_date = utils.convert_to_us_date_format(answer) # apply date formatting fix + except utils.InvalidDateException as e: + corrected_date = "N/A" + + df["Contract Effective Date_corrected"] = corrected_date return df diff --git a/fieldExtraction/src/cnc_hotfix_effective_date_utils.py b/fieldExtraction/src/cnc_hotfix_effective_date_utils.py index e967dab..c3678e9 100644 --- a/fieldExtraction/src/cnc_hotfix_effective_date_utils.py +++ b/fieldExtraction/src/cnc_hotfix_effective_date_utils.py @@ -1,45 +1,6 @@ -included_keyword = ["Effective Date", "entered into", "shall be effective"] -excluded_keyword = ["%", "$","Permit","Certificat","Registration","Department of Public Health"] regex_backticks = r"\|([^|]+)\|" -def get_effective_date_prompt(context): - prompt_template = f"""" - Please analyze the following contract and extract ONLY the effective date. Follow these precise rules in order: - - 1. Look for dates that appear after phrases matching these exact patterns: - - "Effective Date:" followed by a date - - "FOR HEALTH PLAN USE ONLY" section containing "Effective Date:" - - "To be completed by Health Plan only:" section containing "Effective Date:" - - 2. If multiple matches are found: - - Prioritize dates within "FOR HEALTH PLAN USE ONLY" or "To be completed by Health Plan only" sections - - Then prioritize dates that directly follow "Effective Date:" - - 3. Do not use the date found in the "Signature Date" section. - - If there is nothing next to "Effective Date:", return N/A. - - If there is only one date in the document, don't extract this date unless it is explicitly labeled "effective date". It is okay to return N/A. - - 4. If partial dates are found, return N/A. - - 5. Return the date in YYYY-MM-DD format or N/A if date not found. Enclose just your final answer in |pipes|. - - Here is the contract to analyze: - - ## START CONTRACT TEXT ## - - {context} - - ## END CONTRACT TEXT ## - - Ensure that you do not use the date found in the "Signature Date" section. - - Before returning the output, VERIFY the date format is YYYY-MM-DD. If the date is not found, return N/A. - - Finally state "Answer: |YYYY-MM-DD| or |N/A|" - """ - - return prompt_template - +#TODO: leave this one in here; don't move it to prompts until we get confirmation that we want to handle meridian differently. def get_effective_date_meridian_prompt(context): prompt_template = f"""" Please analyze the following contract and extract ONLY the Signature Date. Follow these precise rules in order: diff --git a/fieldExtraction/src/detect_complex.py b/fieldExtraction/src/detect_complex.py index b334efc..01b600f 100644 --- a/fieldExtraction/src/detect_complex.py +++ b/fieldExtraction/src/detect_complex.py @@ -72,7 +72,6 @@ for page in pages: file_list.append(obj["Key"]) contract_list = sorted(file_list) -print(len(contract_list)) files = {} with concurrent.futures.ThreadPoolExecutor( diff --git a/fieldExtraction/src/enums/__init__.py b/fieldExtraction/src/enums/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fieldExtraction/src/enums/delimiters.py b/fieldExtraction/src/enums/delimiters.py new file mode 100644 index 0000000..02db9f1 --- /dev/null +++ b/fieldExtraction/src/enums/delimiters.py @@ -0,0 +1,6 @@ +from enum import Enum + +class Delimiter(Enum): + PIPE = '|' + BACKTICK = '`' + TRIPLE_BACKTICK = '```' diff --git a/fieldExtraction/src/file_processing.py b/fieldExtraction/src/file_processing.py index 40f90ae..066eb1c 100644 --- a/fieldExtraction/src/file_processing.py +++ b/fieldExtraction/src/file_processing.py @@ -19,6 +19,8 @@ import ac_funcs import prompts import valid import claude_funcs +from enums.delimiters import Delimiter + from ac_regex_chunking import npi_hotfix, irs_hotfix def run_ac_prompts(file_object): @@ -60,17 +62,18 @@ def run_ac_prompts(file_object): no_keyword_matches = [] for field_group in field_groups: - keyword_dict = keywords.GROUPED_KEYWORD_MAPPINGS[field_group]["keywords"] + keyword_dict = keywords.GROUPED_KEYWORD_MAPPINGS[field_group].get("keywords",[]) fields = keywords.GROUPED_KEYWORD_MAPPINGS[field_group]["fields"] fields.sort() questions = {} for field in fields: - questions[field] = prompts.AC_DICT[field] + questions[field] = prompts.AC_DICT.get(field,"") # Use chunk if available and different from full context, otherwise use full context ############# Important Note ############# # the above logic is changed # New Logic - chunk if available and different from full context, otherwise add the field to full context list + if ( field_group in ac_chunks and ac_chunks[field_group].strip() @@ -85,7 +88,10 @@ def run_ac_prompts(file_object): question = questions[ fields[0] ] # Take the one question for the single field included - prompt = prompts.AC_SINGLE_FIELD_TEMPLATE(context, question) + if fields[0] == "CONTRACT_EFFECTIVE_DT": + prompt = prompts.get_effective_date_prompt(context) + else: + prompt = prompts.AC_SINGLE_FIELD_TEMPLATE(context, question) elif len(fields) > 1: prompt = prompts.AC_MULTI_FIELD_TEMPLATE(context, questions) else: @@ -114,7 +120,19 @@ def run_ac_prompts(file_object): ) if len(fields) == 1: - field_group_answer_dict = {field: field_group_answer_raw} + if fields[0] == "CONTRACT_EFFECTIVE_DT": + # print(f'Effective Date Answer obtained from LLM: {field_group_answer_raw}') + field_group_answer_raw = ac_funcs.extract_text_from_delimiters(field_group_answer_raw, Delimiter.PIPE) + # print(f'Parsed Effective Date Answer (extract_text_from_delimiters): {field_group_answer_raw}') + try: + field_group_answer_raw = utils.convert_to_us_date_format(field_group_answer_raw) # Fix date formatting + # print(f'Effective Date Answer (convert_to_us_date_format): {field_group_answer_raw}') + except utils.InvalidDateException as e: + field_group_answer_raw = "N/A" + # print(f'Got Invalid Date Exception, using contract effective date as N/A for contract {filename}') + + field_group_answer_dict = {fields[0]: field_group_answer_raw} + elif len(fields) > 1: field_group_answer_raw = field_group_answer_raw.replace("_DATE", "_DT") field_group_answer_dict = ac_funcs.json_parsing_search(field_group_answer_raw, fields) @@ -169,8 +187,12 @@ def run_ac_prompts(file_object): "% Reduction": f"{100*(1-len(context)/len(contract_text)):0.2f}%", } ) - else: - no_keyword_matches.extend(fields) + else: # If no chunk available + if field_group == "CONTRACT_EFFECTIVE_DT": # Special case for effective date + ac_answers_dict["CONTRACT_EFFECTIVE_DT"] = "N/A" + # print('Got no chunk for effective date, so setting it to N/A', ac_answers_dict) + else: # Add to no_keyword_matches list to run using full context + no_keyword_matches.extend(fields) ################## RUN FULL CONTEXT PROMPTS ################## # Process 'full_context' fields together @@ -267,6 +289,7 @@ def run_ac_prompts(file_object): date_answer = claude_funcs.invoke_claude( date_prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 124 ) + ac_answers_dict["CONTRACT_EFFECTIVE_DT"] = date_answer ################## CREATE OUTPUT DIRECTORIES ################## @@ -277,8 +300,13 @@ def run_ac_prompts(file_object): ################## POSTPROCESS ################## ac_df = pd.DataFrame([ac_answers_dict]) + + # print(f"Before ac_postprocess Unique values: {ac_df['CONTRACT_EFFECTIVE_DT'].unique()}") + ac_df = postprocess.ac_postprocess(ac_df, text_dict, filename, num_pages) + # print(f"After ac_postprocess Unique values: {ac_df['Contract Effective Date'].unique()}") + ################## WRITE TO OUTPUT ################## ac_df.to_csv(os.path.join(output_dir, config.AC_RESULTS_NAME), index=False) diff --git a/fieldExtraction/src/keywords.py b/fieldExtraction/src/keywords.py index d69be2b..0695ce0 100644 --- a/fieldExtraction/src/keywords.py +++ b/fieldExtraction/src/keywords.py @@ -97,11 +97,9 @@ GROUPED_KEYWORD_MAPPINGS = { }, "CONTRACT_EFFECTIVE_DT": { "fields": ["CONTRACT_EFFECTIVE_DT"], - "methodology": "or", - "keywords": [ - "effective date", - "effective", - ], # TODO Switch methodology to regex for date formats + 'or' methodology is case-insensitive -remove duplicates + "methodology": "include_exclude", + "included_keywords" : ["Effective Date", "entered into", "shall be effective", "will become effective"], + "excluded_keywords" : ["%", "$","Permit","Certificat","Registration","Department of Public Health"], "case_sensitive": False, }, "TERMINATION_UPON_NOTICE": { diff --git a/fieldExtraction/src/preprocessing_funcs.py b/fieldExtraction/src/preprocessing_funcs.py index e242666..0f368dc 100644 --- a/fieldExtraction/src/preprocessing_funcs.py +++ b/fieldExtraction/src/preprocessing_funcs.py @@ -1,5 +1,5 @@ -from itertools import groupby, count -import re +from itertools import groupby, count, chain +import re from collections import defaultdict import utils import valid @@ -240,6 +240,23 @@ def smart_chunk_ac( keyword_page_dict = ac_smart_chunking.keyword_search( text_dict, key_dict["keywords"], key_dict["case_sensitive"], True ) + elif key_dict["methodology"] == "include_exclude": + # Chunk pages based on included and excluded keywords + page_list = ac_smart_chunking.chunk_with_include_exclude_keywords( + text_dict, key_dict["included_keywords"], key_dict["excluded_keywords"] + ) + + # Create a copy of keywords list by combining included and excluded keywords - this is required for keyword search + included_keywords = key_dict["included_keywords"] + excluded_keywords = key_dict["excluded_keywords"] + + keywords = list(chain(included_keywords,excluded_keywords)) + + # Perform keyword search on the text dictionary - this returns a dictionary keyed by keyword and values are list of pages containing that keyword + # This is not currently used but is available for debugging purposes (or future use cases) + keyword_page_dict = ac_smart_chunking.keyword_search( + text_dict, keywords, key_dict["case_sensitive"] + ) else: page_list = list(text_dict.keys()) keyword_page_dict = {"All pages": list(text_dict.keys())} diff --git a/fieldExtraction/src/prompts.py b/fieldExtraction/src/prompts.py index 8074c62..b60dbdf 100644 --- a/fieldExtraction/src/prompts.py +++ b/fieldExtraction/src/prompts.py @@ -395,9 +395,55 @@ If for any question, no correct answer is found, return 'N/A'. {questions}. """ +def get_effective_date_prompt(context: str) -> str: + """ + Generate a prompt template to extract the effective date from a contract. + + Args: + context (str): The contract text to analyze. + + Returns: + str: The prompt template for extracting the effective date. + """ + prompt_template = f""" + Please analyze the following contract and extract ONLY the effective date of the contract or amendment. Follow these precise rules in order: + + 1. Look for dates that appear after phrases matching these exact patterns: + - "Effective Date:" followed by a date + - "FOR HEALTH PLAN USE ONLY" section containing "Effective Date:" + - "To be completed by Health Plan only:" section containing "Effective Date:" + - "is entered into as of" followed by a date + + 2. If multiple matches are found: + - Prioritize dates within "FOR HEALTH PLAN USE ONLY" or "To be completed by Health Plan only" sections + - Then prioritize dates that directly follow "Effective Date:" + + 3. Do not use the date found in the "Signature Date" section. + - If there is nothing next to "Effective Date:", return N/A. + - If there is only one date in the document, don't extract this date unless it is explicitly labeled "effective date". It is okay to return N/A. + + 4. If partial dates are found, return N/A. + + 5. Return the date in YYYY-MM-DD format or N/A if date not found. Enclose just your final answer in |pipes|. + + Here is the contract to analyze: + + ## START CONTRACT TEXT ## + + {context} + + ## END CONTRACT TEXT ## + + Ensure that you do not use the date found in the "Signature Date" section. + + Before returning the output, VERIFY the date format is YYYY-MM-DD. If the date is not found, return N/A. + + Finally state "Answer: |YYYY-MM-DD| or |N/A|" + """ + + return prompt_template AC_DICT = { - "CONTRACT_EFFECTIVE_DT": "Extract the contract effective date. This may be mentioned in one of the following locations: the signatory section, the preamble of the agreement, or the start of the amendment. Look for phrases such as /'This amendment is effective/'. Return this date in YYYY-MM-DD format.", "CONTRACT_TITLE": "What is the complete title of the contract or the document? It is generally written in CAPITAL LETTERS and is found at the beginning of the document before a paragraph containing provider and payer name. It typically contains the words AGREEMENT, AMENDMENT, CONTRACT, ADDENDUM, MEMORANDUM, LETTER, PROVIDER or REQUEST. In case of agreement, typical structure is the provider name, followed by the phrase with agreement, followed potentially by a number signifying a version of the contract. In case of amendment, typical structure is word amendment followed by number, followed by the phrase with agreement. Extract the complete title of the contract or document as accurately as possible. Ensure: All leading characters are included for each word (e.g. 'NETWORK' instead of 'ETWORK'). Spaces between words are preserved in phrases (e.g., 'PROVIDER COLLABORATION' instead of 'PROVIDERCOLLABORATION'). If text appears incomplete or lacks spacing, use logical patterns like capitalization or common formatting rules to infer and correct the output. Replace new lines and line breaks with space.", "PAYER_NAME": "What is the name of the payer that is a party to the contract as stated in the Preamble?", "PROV_GROUP_NAME": "What is the name of the Group provider that is a party to the contract? This is generally found near keyword provider. Only return the name of the Group provider along with dba name.", @@ -487,7 +533,6 @@ AC_DICT = { "RELATIONSHIP_OF_PARTIES_LANGUAGE": "Extract the entire Relationship of Parties passage present in the contract.", } - def AC_EFFECTIVE_DATE_CLEANUP(contract_text): return f"""### Contract Start ### {contract_text} ### Contract End ### Above is a contract. What is the contract effective date mentioned in any of the following locations: the signatory section, the preamble of the agreement, or the start of the amendment? Look for phrases such as /'This amendment is effective/'. diff --git a/fieldExtraction/src/regex_patterns.py b/fieldExtraction/src/regex_patterns.py new file mode 100644 index 0000000..df45ab5 --- /dev/null +++ b/fieldExtraction/src/regex_patterns.py @@ -0,0 +1,3 @@ +PIPE_PATTERN = r'\|([^|]+)\|' +BACKTICK_PATTERN = r'`([^`]+)`' +TRIPLE_BACKTICK_PATTERN = r'```([^`]+)```' \ No newline at end of file diff --git a/fieldExtraction/src/table_funcs.py b/fieldExtraction/src/table_funcs.py index 20adac5..2e0537d 100644 --- a/fieldExtraction/src/table_funcs.py +++ b/fieldExtraction/src/table_funcs.py @@ -110,7 +110,6 @@ def convert_to_dict(table_text): tuple: A dictionary with keys mapping to lists of values, and an integer representing the number of elements in each list. """ table_text = clean_tables(table_text) - # print(table_text, '\n') table_text_list = table_text.split("]") table_text_list = [item for item in table_text_list if len(item) > 0] @@ -120,14 +119,12 @@ def convert_to_dict(table_text): key = key_value_text.split(":")[0].strip(", '") try: value = ":".join(key_value_text.split(":")[1:]).strip() + "]" - # print(value, '\n') if len(value) > 1: value_list = ast.literal_eval(value) final_dict[key] = value_list num_elements = len(value_list) except: value = ":".join(key_value_text.split(":")[1:]).strip() + "']" - # print(value, '\n') if len(value) > 1: value_list = ast.literal_eval(value) final_dict[key] = value_list diff --git a/fieldExtraction/src/tin_pull.py b/fieldExtraction/src/tin_pull.py index ff469cd..c3b41c9 100644 --- a/fieldExtraction/src/tin_pull.py +++ b/fieldExtraction/src/tin_pull.py @@ -29,7 +29,6 @@ def find_tin_numbers(text): def invoke_llm(context, llm_selected=LLM_SELECTED): - print("Invoking LLM.") prompt_data = f""" Human: Use the following pieces of context to provide a concise answer to the questions at the end. You must answer in python list format. @@ -91,7 +90,6 @@ def invoke_llm(context, llm_selected=LLM_SELECTED): response_text = response_body["completion"] else: print("Unsupported LLM selected") - # print(f"LLM response: {response_text}") except Exception as e: print(f"Error invoking LLM: {e}") response_text = "failed" diff --git a/fieldExtraction/src/utils.py b/fieldExtraction/src/utils.py index 1d2777f..409f192 100644 --- a/fieldExtraction/src/utils.py +++ b/fieldExtraction/src/utils.py @@ -9,6 +9,10 @@ from collections import defaultdict import config +class InvalidDateException(Exception): + pass + + def read_local(file_path): # Check if the file is a text file if os.path.isfile(file_path) and file_path.endswith(".txt"): @@ -354,4 +358,49 @@ def add_hyphen_if_needed(input_str): if not bool(re.fullmatch(r"\b\d{2}-\d{7}\b", input_str)): input_str = None - return input_str \ No newline at end of file + return input_str + +def convert_to_us_date_format(date_str: str) -> str: + """ + Input date string is expected to be in the format YYYY-MM-DD + + Ensure the date is in MM/DD/YYYY format. If the month is greater than 12, swap the month and day. + If the input is `N/A` or some variant (defined by is_empty), InvalidDateException will be raised. + Args: + date_str (str): The date string to be formatted. We expect it to come in as YYYY-MM-DD format (note the hyphens as well) + + Returns: + str: The date string formatted as MM/DD/YYYY. + + Raises: + InvalidDateException: If the date string is not in the expected format. + """ + + if not isinstance(date_str, str): + raise TypeError(f"Date must be a string, got: {type(date_str)}") + + if is_empty(date_str) or re.fullmatch(r"\d{4}-\d{1,2}-\d{1,2}", date_str) is None: + raise InvalidDateException(f"Invalid date format: {date_str}") + + # Split the date string into year, month, and day + year, month, day = date_str.split("-") + + year_int = int(year) + month_int = int(month) + day_int = int(day) + + if year_int == 0 or month_int == 0 or day_int == 0: + raise InvalidDateException(f"Invalid date format: {date_str}") + + if month_int > 12 and 0 < day_int <= 31: + # Swap the month and day if the month is greater than 12 + month_int, day_int = day_int, month_int + + if 0 < month_int <= 12 and 0 < day_int <= 31: + # Return the formatted date in MM/DD/YYYY format + year, month, day = str(year_int).zfill(4), str(month_int).zfill(2), str(day_int).zfill(2) + return "/".join([month, day, year]) + + else: + # Raise an exception if the month or day is out of range + raise InvalidDateException(f"Invalid date format: {date_str}") diff --git a/fieldExtraction/tests/ac_funcs_test.py b/fieldExtraction/tests/ac_funcs_test.py new file mode 100644 index 0000000..122e80d --- /dev/null +++ b/fieldExtraction/tests/ac_funcs_test.py @@ -0,0 +1,52 @@ +import pytest +from enums.delimiters import Delimiter +from ac_funcs import extract_text_from_delimiters + + +class TestExtractTextFromDelimiters: + # Test case for valid inputs + + def test_pipe_delimiter_match_first_position(self): + raw_text = "Here is the answer |this is a pipe answer| and |more text.|" + result = extract_text_from_delimiters(raw_text,delimiter=Delimiter.PIPE,match_index=0) + assert result == "this is a pipe answer" + + def test_pipe_delimiter_match_last_position(self): + raw_text = "Here is the answer |this is a pipe answer| and |more text.|" + result = extract_text_from_delimiters(raw_text,delimiter=Delimiter.PIPE,match_index=-1) + assert result == "more text." + + def test_pipe_delimiter_match_any_position(self): + raw_text = "|Here| is the answer |this is a pipe answer| and |more text.|" + result = extract_text_from_delimiters(raw_text,delimiter=Delimiter.PIPE,match_index=1) + assert result == "this is a pipe answer" + + def test_backtick_delimiter_match_first_position(self): + raw_text = "Here is the answer `this is a backtick answer` and `more text.`" + result = extract_text_from_delimiters(raw_text, delimiter=Delimiter.BACKTICK, match_index=0) + assert result == "this is a backtick answer" + + def test_backtick_delimiter_match_last_position(self): + raw_text = "Here is the answer `this is a backtick answer` and `more text.`" + result = extract_text_from_delimiters(raw_text, delimiter=Delimiter.BACKTICK, match_index=-1) + assert result == "more text." + + def test_backtick_delimiter_match_any_position(self): + raw_text = "`Here` is the answer `this` is a `backtick answer` and `more text.`" + result = extract_text_from_delimiters(raw_text, delimiter=Delimiter.BACKTICK, match_index=2) + assert result == "backtick answer" + + def test_triple_backtick_delimiter_match_first_position(self): + raw_text = "Here is the answer ```this is a triple backtick answer``` and more text." + result = extract_text_from_delimiters(raw_text, delimiter=Delimiter.TRIPLE_BACKTICK, match_index=0) + assert result == "this is a triple backtick answer" + + def test_triple_backtick_delimiter_match_last_position(self): + raw_text = "Here is the answer ```this``` is a ```triple``` backtick ```answer``` and more text." + result = extract_text_from_delimiters(raw_text, delimiter=Delimiter.TRIPLE_BACKTICK, match_index=-1) + assert result == "answer" + + def test_triple_backtick_delimiter_match_any_position(self): + raw_text = "```Here``` is the answer ```this is a triple backtick answer``` and ```more text.```" + result = extract_text_from_delimiters(raw_text, delimiter=Delimiter.TRIPLE_BACKTICK, match_index=1) + assert result == "this is a triple backtick answer" \ No newline at end of file diff --git a/fieldExtraction/tests/utils_test.py b/fieldExtraction/tests/utils_test.py new file mode 100644 index 0000000..28ffd3d --- /dev/null +++ b/fieldExtraction/tests/utils_test.py @@ -0,0 +1,43 @@ +import pytest +from utils import convert_to_us_date_format, InvalidDateException, is_empty + + +class TestConvertToUSDateFormat: + def test_convert_to_us_date_format_valid(self): + assert convert_to_us_date_format("2023-12-01") == "12/01/2023" + assert convert_to_us_date_format("2023-01-12") == "01/12/2023" + assert convert_to_us_date_format("2023-13-01") == "01/13/2023" + + def test_convert_to_us_date_format_invalid(self): + with pytest.raises(InvalidDateException): + convert_to_us_date_format("2023-13-32") + with pytest.raises(InvalidDateException): + convert_to_us_date_format("2023-00-01") + with pytest.raises(InvalidDateException): + convert_to_us_date_format("2023-12-00") + with pytest.raises(InvalidDateException): + convert_to_us_date_format("2023-12-32") + with pytest.raises(InvalidDateException): + convert_to_us_date_format("2023-13-13") + + def test_convert_to_us_date_format_empty(self): + with pytest.raises(InvalidDateException): + convert_to_us_date_format("") + with pytest.raises(InvalidDateException): + convert_to_us_date_format("N/A") + with pytest.raises(InvalidDateException): + convert_to_us_date_format("null") + with pytest.raises(InvalidDateException): + convert_to_us_date_format("none") + + def test_convert_to_us_date_format_invalid_format(self): + with pytest.raises(InvalidDateException): + convert_to_us_date_format("2023/12/01") + with pytest.raises(InvalidDateException): + convert_to_us_date_format("12-01-2023") + with pytest.raises(InvalidDateException): + convert_to_us_date_format("2023.12.01") + + def test_convert_to_us_date_format_invalid_type(self): + with pytest.raises(TypeError): + convert_to_us_date_format(None)