Merged in chore/contract_effective_date (pull request #307)
New Prompt Method (starting with contract effective date) * add todos for Siddhant * moved effective date prompts to prompts.py and added docstring and type hints * moved effective date prompts to prompts.py and added docstring and type hints * fixed conflict Co-authored-by: Alex Galarce <agalarce@aarete.com> * modified contract effective date logic in existing smart chunking * Merged main into chore/contract_effective_date * updated extract_text_from_delimiters function and unit tests for it * Merge remote-tracking branch 'origin/main' into chore/contract_effective_date * added comments to preprocessing_funcs * preprocessing_funcs.py edited online with Bitbucket * utils.py edited online with Bitbucket * add debug line * add NA handling into fix_date_formatting() * add debug print statements * worked on review comments * fixed date checking * fixed date checking and added unit tests * Merged main into chore/contract_effective_date * added logic to handle case when smart chunking returns empty context for effective date * added comments to file_processing.py * optimized imports and removed a TODO * Merged main into chore/contract_effective_date * made prompt changes for effective date * keywords.py edited online with Bitbucket * Removed excess print statements Approved-by: Katon Minhas
This commit is contained in:
Alex Galarce
committed by
Katon Minhas
Katon Minhas
parent
b2a8e77cca
commit
47a01e13f0
@@ -13,12 +13,15 @@ from botocore.config import Config
|
||||
import traceback
|
||||
import time
|
||||
import sys
|
||||
|
||||
import warnings
|
||||
from enums.delimiters import Delimiter
|
||||
|
||||
import claude_funcs
|
||||
import dict_operations
|
||||
import config
|
||||
import utils
|
||||
import prompts
|
||||
from regex_patterns import PIPE_PATTERN, BACKTICK_PATTERN, TRIPLE_BACKTICK_PATTERN
|
||||
|
||||
|
||||
def get_ac_answer(prompt, filename, fields):
|
||||
@@ -101,9 +104,9 @@ def select_context(text_dict, field):
|
||||
keywords = ["Provider Manual", "Policies and Procedures"]
|
||||
if field == "ARBITRATION_AND_DISPUTES":
|
||||
keywords = ["Informal Dispute Resolution", "Arbitration"]
|
||||
print(field)
|
||||
|
||||
page_list = []
|
||||
print(keywords)
|
||||
|
||||
for keyword in keywords:
|
||||
for page in text_dict:
|
||||
if keyword.lower() in text_dict[page].lower():
|
||||
@@ -135,7 +138,6 @@ def create_prompt(context, question):
|
||||
|
||||
|
||||
def json_parsing(response_text, context):
|
||||
print("Parsing JSON response.")
|
||||
try:
|
||||
if response_text.split("{", 1)[1].strip()[0] == '"':
|
||||
response_text = "{" + response_text.split("{", 1)[1]
|
||||
@@ -249,10 +251,59 @@ def json_parsing(response_text, context):
|
||||
]
|
||||
snippet_l = [snippet.strip() for snippet in snippet_l]
|
||||
|
||||
print(f"Parsed fields: {field_l}")
|
||||
return field_l, answer_l, page_no_l, location_l, snippet_l
|
||||
|
||||
|
||||
def extract_text_from_delimiters(raw_output: str, delimiter: Delimiter, match_index: int=-1) -> str:
|
||||
"""
|
||||
Extracts a match enclosed in specified delimiters from the raw output.
|
||||
This function searches for text enclosed by a specified delimiter in the given raw output string.
|
||||
It returns the match specified by the `which_match` index. If no match is found, it returns "N/A".
|
||||
|
||||
- raw_output (str): The raw output string to search within.
|
||||
- which_match (int, optional): The index of the match to return. Defaults to -1, which returns the last match. If the index is out of range, the last match is returned.
|
||||
- str: The extracted text or "N/A" if no match is found.
|
||||
|
||||
Raises:
|
||||
- TypeError: If `raw_output` is not a string or `delimiter` is not a Delimiter enum value.
|
||||
- ValueError: If an unsupported delimiter is provided.
|
||||
|
||||
Returns:
|
||||
- str: The extracted answer or "N/A" if no match is found.
|
||||
"""
|
||||
if not isinstance(raw_output, str):
|
||||
raise TypeError("Expected a string for raw_output, got {0}.".format(type(raw_output).__name__))
|
||||
|
||||
if not isinstance(delimiter, Delimiter):
|
||||
raise TypeError("Expected a Delimiter enum value, got {0}.".format(type(delimiter).__name__))
|
||||
|
||||
# Define a pattern based on the delimiter type
|
||||
if delimiter == Delimiter.PIPE:
|
||||
pattern = PIPE_PATTERN
|
||||
elif delimiter == Delimiter.BACKTICK:
|
||||
pattern = BACKTICK_PATTERN
|
||||
elif delimiter == Delimiter.TRIPLE_BACKTICK:
|
||||
pattern = TRIPLE_BACKTICK_PATTERN
|
||||
else:
|
||||
raise ValueError("Unsupported delimiter. Use one of the Delimiter enum values.")
|
||||
|
||||
# Find all matches based on the pattern
|
||||
matches = re.findall(pattern=pattern, string=raw_output)
|
||||
|
||||
if len(matches) == 0:
|
||||
return "N/A"
|
||||
|
||||
# Adjust for negative indices
|
||||
if match_index < 0:
|
||||
match_index += len(matches)
|
||||
|
||||
if not 0 <= match_index < len(matches):
|
||||
warnings.warn("Index out of range. Returning the last match by default.")
|
||||
match_index = -1
|
||||
|
||||
return matches[match_index]
|
||||
|
||||
|
||||
def json_parsing_search(response_text, field_list):
|
||||
try:
|
||||
if response_text.split("{", 1)[1].strip()[0] == '"':
|
||||
|
||||
@@ -156,7 +156,6 @@ class BatchTracker:
|
||||
current_batch_entries = df[df['batch_id'] == config.BATCH_ID]
|
||||
|
||||
if current_batch_entries.empty:
|
||||
print("No entry found for current batch. Creating new entry.")
|
||||
self.add_batch_run()
|
||||
return
|
||||
|
||||
|
||||
@@ -452,12 +452,12 @@ def clean_irs(abc_df, filename, text_dict ):
|
||||
def contract_effective_date_fix(contract_name: str, text_dict: dict[str, str],is_meridian: bool=False) -> str:
|
||||
# perform smart chunking
|
||||
page_list = ac_smart_chunking.chunk_with_include_exclude_keywords(text_dict,
|
||||
include_keywords = cnc_hotfix_effective_date_utils.included_keyword,
|
||||
exclude_keywords = cnc_hotfix_effective_date_utils.excluded_keyword
|
||||
include_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["included_keywords"],
|
||||
exclude_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["excluded_keywords"],
|
||||
)
|
||||
|
||||
context = "\n".join(text_dict[page] for page in page_list)
|
||||
prompt = cnc_hotfix_effective_date_utils.get_effective_date_prompt(context) # get effective date - this is for both meridian and non-meridian
|
||||
prompt = prompts.get_effective_date_prompt(context) # get effective date - this is for both meridian and non-meridian
|
||||
|
||||
try:
|
||||
claude_answer_raw = claude_funcs.invoke_claude(
|
||||
@@ -471,9 +471,16 @@ def contract_effective_date_fix(contract_name: str, text_dict: dict[str, str],is
|
||||
if is_meridian:
|
||||
if claude_answer_extracted == "N/A":
|
||||
# perform smart chunking with signature date pages included
|
||||
|
||||
included_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["included_keywords"]
|
||||
|
||||
included_keywords = list(itertools.chain(included_keywords, ["Signature"]))
|
||||
|
||||
excluded_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["excluded_keywords"]
|
||||
|
||||
page_list = ac_smart_chunking.chunk_with_include_exclude_keywords(text_dict,
|
||||
include_keywords = cnc_hotfix_effective_date_utils.included_keyword + ["Signature"],
|
||||
exclude_keywords = cnc_hotfix_effective_date_utils.excluded_keyword
|
||||
include_keywords = included_keywords,
|
||||
exclude_keywords = excluded_keywords
|
||||
)
|
||||
|
||||
context = "\n".join(text_dict[page] for page in page_list)
|
||||
@@ -534,7 +541,6 @@ def clean_contract_effective_date(
|
||||
try:
|
||||
answer = contract_effective_date_fix(file_name, text_dict,is_meridian)
|
||||
except Exception as e:
|
||||
# KATON: Do we want to give out "answer='Error'" in this case?
|
||||
print(f"Error processing {file_name}, got error: {e}")
|
||||
else:
|
||||
answer = contract_effective_date
|
||||
@@ -542,7 +548,12 @@ def clean_contract_effective_date(
|
||||
if answer == "N/A":
|
||||
df["Contract Effective Date_corrected"] = answer
|
||||
else:
|
||||
df["Contract Effective Date_corrected"] = pd.to_datetime(answer, dayfirst=False).strftime('%m/%d/%Y')
|
||||
try:
|
||||
corrected_date = utils.convert_to_us_date_format(answer) # apply date formatting fix
|
||||
except utils.InvalidDateException as e:
|
||||
corrected_date = "N/A"
|
||||
|
||||
df["Contract Effective Date_corrected"] = corrected_date
|
||||
|
||||
return df
|
||||
|
||||
|
||||
@@ -1,45 +1,6 @@
|
||||
included_keyword = ["Effective Date", "entered into", "shall be effective"]
|
||||
excluded_keyword = ["%", "$","Permit","Certificat","Registration","Department of Public Health"]
|
||||
regex_backticks = r"\|([^|]+)\|"
|
||||
|
||||
def get_effective_date_prompt(context):
|
||||
prompt_template = f""""
|
||||
Please analyze the following contract and extract ONLY the effective date. Follow these precise rules in order:
|
||||
|
||||
1. Look for dates that appear after phrases matching these exact patterns:
|
||||
- "Effective Date:" followed by a date
|
||||
- "FOR HEALTH PLAN USE ONLY" section containing "Effective Date:"
|
||||
- "To be completed by Health Plan only:" section containing "Effective Date:"
|
||||
|
||||
2. If multiple matches are found:
|
||||
- Prioritize dates within "FOR HEALTH PLAN USE ONLY" or "To be completed by Health Plan only" sections
|
||||
- Then prioritize dates that directly follow "Effective Date:"
|
||||
|
||||
3. Do not use the date found in the "Signature Date" section.
|
||||
- If there is nothing next to "Effective Date:", return N/A.
|
||||
- If there is only one date in the document, don't extract this date unless it is explicitly labeled "effective date". It is okay to return N/A.
|
||||
|
||||
4. If partial dates are found, return N/A.
|
||||
|
||||
5. Return the date in YYYY-MM-DD format or N/A if date not found. Enclose just your final answer in |pipes|.
|
||||
|
||||
Here is the contract to analyze:
|
||||
|
||||
## START CONTRACT TEXT ##
|
||||
|
||||
{context}
|
||||
|
||||
## END CONTRACT TEXT ##
|
||||
|
||||
Ensure that you do not use the date found in the "Signature Date" section.
|
||||
|
||||
Before returning the output, VERIFY the date format is YYYY-MM-DD. If the date is not found, return N/A.
|
||||
|
||||
Finally state "Answer: |YYYY-MM-DD| or |N/A|"
|
||||
"""
|
||||
|
||||
return prompt_template
|
||||
|
||||
#TODO: leave this one in here; don't move it to prompts until we get confirmation that we want to handle meridian differently.
|
||||
def get_effective_date_meridian_prompt(context):
|
||||
prompt_template = f""""
|
||||
Please analyze the following contract and extract ONLY the Signature Date. Follow these precise rules in order:
|
||||
|
||||
@@ -72,7 +72,6 @@ for page in pages:
|
||||
file_list.append(obj["Key"])
|
||||
|
||||
contract_list = sorted(file_list)
|
||||
print(len(contract_list))
|
||||
files = {}
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
from enum import Enum
|
||||
|
||||
class Delimiter(Enum):
|
||||
PIPE = '|'
|
||||
BACKTICK = '`'
|
||||
TRIPLE_BACKTICK = '```'
|
||||
@@ -19,6 +19,8 @@ import ac_funcs
|
||||
import prompts
|
||||
import valid
|
||||
import claude_funcs
|
||||
from enums.delimiters import Delimiter
|
||||
|
||||
from ac_regex_chunking import npi_hotfix, irs_hotfix
|
||||
|
||||
def run_ac_prompts(file_object):
|
||||
@@ -60,17 +62,18 @@ def run_ac_prompts(file_object):
|
||||
no_keyword_matches = []
|
||||
|
||||
for field_group in field_groups:
|
||||
keyword_dict = keywords.GROUPED_KEYWORD_MAPPINGS[field_group]["keywords"]
|
||||
keyword_dict = keywords.GROUPED_KEYWORD_MAPPINGS[field_group].get("keywords",[])
|
||||
fields = keywords.GROUPED_KEYWORD_MAPPINGS[field_group]["fields"]
|
||||
fields.sort()
|
||||
questions = {}
|
||||
for field in fields:
|
||||
questions[field] = prompts.AC_DICT[field]
|
||||
questions[field] = prompts.AC_DICT.get(field,"")
|
||||
|
||||
# Use chunk if available and different from full context, otherwise use full context
|
||||
############# Important Note #############
|
||||
# the above logic is changed
|
||||
# New Logic - chunk if available and different from full context, otherwise add the field to full context list
|
||||
|
||||
if (
|
||||
field_group in ac_chunks
|
||||
and ac_chunks[field_group].strip()
|
||||
@@ -85,7 +88,10 @@ def run_ac_prompts(file_object):
|
||||
question = questions[
|
||||
fields[0]
|
||||
] # Take the one question for the single field included
|
||||
prompt = prompts.AC_SINGLE_FIELD_TEMPLATE(context, question)
|
||||
if fields[0] == "CONTRACT_EFFECTIVE_DT":
|
||||
prompt = prompts.get_effective_date_prompt(context)
|
||||
else:
|
||||
prompt = prompts.AC_SINGLE_FIELD_TEMPLATE(context, question)
|
||||
elif len(fields) > 1:
|
||||
prompt = prompts.AC_MULTI_FIELD_TEMPLATE(context, questions)
|
||||
else:
|
||||
@@ -114,7 +120,19 @@ def run_ac_prompts(file_object):
|
||||
)
|
||||
|
||||
if len(fields) == 1:
|
||||
field_group_answer_dict = {field: field_group_answer_raw}
|
||||
if fields[0] == "CONTRACT_EFFECTIVE_DT":
|
||||
# print(f'Effective Date Answer obtained from LLM: {field_group_answer_raw}')
|
||||
field_group_answer_raw = ac_funcs.extract_text_from_delimiters(field_group_answer_raw, Delimiter.PIPE)
|
||||
# print(f'Parsed Effective Date Answer (extract_text_from_delimiters): {field_group_answer_raw}')
|
||||
try:
|
||||
field_group_answer_raw = utils.convert_to_us_date_format(field_group_answer_raw) # Fix date formatting
|
||||
# print(f'Effective Date Answer (convert_to_us_date_format): {field_group_answer_raw}')
|
||||
except utils.InvalidDateException as e:
|
||||
field_group_answer_raw = "N/A"
|
||||
# print(f'Got Invalid Date Exception, using contract effective date as N/A for contract {filename}')
|
||||
|
||||
field_group_answer_dict = {fields[0]: field_group_answer_raw}
|
||||
|
||||
elif len(fields) > 1:
|
||||
field_group_answer_raw = field_group_answer_raw.replace("_DATE", "_DT")
|
||||
field_group_answer_dict = ac_funcs.json_parsing_search(field_group_answer_raw, fields)
|
||||
@@ -169,8 +187,12 @@ def run_ac_prompts(file_object):
|
||||
"% Reduction": f"{100*(1-len(context)/len(contract_text)):0.2f}%",
|
||||
}
|
||||
)
|
||||
else:
|
||||
no_keyword_matches.extend(fields)
|
||||
else: # If no chunk available
|
||||
if field_group == "CONTRACT_EFFECTIVE_DT": # Special case for effective date
|
||||
ac_answers_dict["CONTRACT_EFFECTIVE_DT"] = "N/A"
|
||||
# print('Got no chunk for effective date, so setting it to N/A', ac_answers_dict)
|
||||
else: # Add to no_keyword_matches list to run using full context
|
||||
no_keyword_matches.extend(fields)
|
||||
|
||||
################## RUN FULL CONTEXT PROMPTS ##################
|
||||
# Process 'full_context' fields together
|
||||
@@ -267,6 +289,7 @@ def run_ac_prompts(file_object):
|
||||
date_answer = claude_funcs.invoke_claude(
|
||||
date_prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 124
|
||||
)
|
||||
|
||||
ac_answers_dict["CONTRACT_EFFECTIVE_DT"] = date_answer
|
||||
|
||||
################## CREATE OUTPUT DIRECTORIES ##################
|
||||
@@ -277,8 +300,13 @@ def run_ac_prompts(file_object):
|
||||
################## POSTPROCESS ##################
|
||||
|
||||
ac_df = pd.DataFrame([ac_answers_dict])
|
||||
|
||||
# print(f"Before ac_postprocess Unique values: {ac_df['CONTRACT_EFFECTIVE_DT'].unique()}")
|
||||
|
||||
ac_df = postprocess.ac_postprocess(ac_df, text_dict, filename, num_pages)
|
||||
|
||||
# print(f"After ac_postprocess Unique values: {ac_df['Contract Effective Date'].unique()}")
|
||||
|
||||
################## WRITE TO OUTPUT ##################
|
||||
ac_df.to_csv(os.path.join(output_dir, config.AC_RESULTS_NAME), index=False)
|
||||
|
||||
|
||||
@@ -97,11 +97,9 @@ GROUPED_KEYWORD_MAPPINGS = {
|
||||
},
|
||||
"CONTRACT_EFFECTIVE_DT": {
|
||||
"fields": ["CONTRACT_EFFECTIVE_DT"],
|
||||
"methodology": "or",
|
||||
"keywords": [
|
||||
"effective date",
|
||||
"effective",
|
||||
], # TODO Switch methodology to regex for date formats + 'or' methodology is case-insensitive -remove duplicates
|
||||
"methodology": "include_exclude",
|
||||
"included_keywords" : ["Effective Date", "entered into", "shall be effective", "will become effective"],
|
||||
"excluded_keywords" : ["%", "$","Permit","Certificat","Registration","Department of Public Health"],
|
||||
"case_sensitive": False,
|
||||
},
|
||||
"TERMINATION_UPON_NOTICE": {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from itertools import groupby, count
|
||||
import re
|
||||
from itertools import groupby, count, chain
|
||||
import re
|
||||
from collections import defaultdict
|
||||
import utils
|
||||
import valid
|
||||
@@ -240,6 +240,23 @@ def smart_chunk_ac(
|
||||
keyword_page_dict = ac_smart_chunking.keyword_search(
|
||||
text_dict, key_dict["keywords"], key_dict["case_sensitive"], True
|
||||
)
|
||||
elif key_dict["methodology"] == "include_exclude":
|
||||
# Chunk pages based on included and excluded keywords
|
||||
page_list = ac_smart_chunking.chunk_with_include_exclude_keywords(
|
||||
text_dict, key_dict["included_keywords"], key_dict["excluded_keywords"]
|
||||
)
|
||||
|
||||
# Create a copy of keywords list by combining included and excluded keywords - this is required for keyword search
|
||||
included_keywords = key_dict["included_keywords"]
|
||||
excluded_keywords = key_dict["excluded_keywords"]
|
||||
|
||||
keywords = list(chain(included_keywords,excluded_keywords))
|
||||
|
||||
# Perform keyword search on the text dictionary - this returns a dictionary keyed by keyword and values are list of pages containing that keyword
|
||||
# This is not currently used but is available for debugging purposes (or future use cases)
|
||||
keyword_page_dict = ac_smart_chunking.keyword_search(
|
||||
text_dict, keywords, key_dict["case_sensitive"]
|
||||
)
|
||||
else:
|
||||
page_list = list(text_dict.keys())
|
||||
keyword_page_dict = {"All pages": list(text_dict.keys())}
|
||||
|
||||
@@ -395,9 +395,55 @@ If for any question, no correct answer is found, return 'N/A'.
|
||||
{questions}.
|
||||
"""
|
||||
|
||||
def get_effective_date_prompt(context: str) -> str:
|
||||
"""
|
||||
Generate a prompt template to extract the effective date from a contract.
|
||||
|
||||
Args:
|
||||
context (str): The contract text to analyze.
|
||||
|
||||
Returns:
|
||||
str: The prompt template for extracting the effective date.
|
||||
"""
|
||||
prompt_template = f"""
|
||||
Please analyze the following contract and extract ONLY the effective date of the contract or amendment. Follow these precise rules in order:
|
||||
|
||||
1. Look for dates that appear after phrases matching these exact patterns:
|
||||
- "Effective Date:" followed by a date
|
||||
- "FOR HEALTH PLAN USE ONLY" section containing "Effective Date:"
|
||||
- "To be completed by Health Plan only:" section containing "Effective Date:"
|
||||
- "is entered into as of" followed by a date
|
||||
|
||||
2. If multiple matches are found:
|
||||
- Prioritize dates within "FOR HEALTH PLAN USE ONLY" or "To be completed by Health Plan only" sections
|
||||
- Then prioritize dates that directly follow "Effective Date:"
|
||||
|
||||
3. Do not use the date found in the "Signature Date" section.
|
||||
- If there is nothing next to "Effective Date:", return N/A.
|
||||
- If there is only one date in the document, don't extract this date unless it is explicitly labeled "effective date". It is okay to return N/A.
|
||||
|
||||
4. If partial dates are found, return N/A.
|
||||
|
||||
5. Return the date in YYYY-MM-DD format or N/A if date not found. Enclose just your final answer in |pipes|.
|
||||
|
||||
Here is the contract to analyze:
|
||||
|
||||
## START CONTRACT TEXT ##
|
||||
|
||||
{context}
|
||||
|
||||
## END CONTRACT TEXT ##
|
||||
|
||||
Ensure that you do not use the date found in the "Signature Date" section.
|
||||
|
||||
Before returning the output, VERIFY the date format is YYYY-MM-DD. If the date is not found, return N/A.
|
||||
|
||||
Finally state "Answer: |YYYY-MM-DD| or |N/A|"
|
||||
"""
|
||||
|
||||
return prompt_template
|
||||
|
||||
AC_DICT = {
|
||||
"CONTRACT_EFFECTIVE_DT": "Extract the contract effective date. This may be mentioned in one of the following locations: the signatory section, the preamble of the agreement, or the start of the amendment. Look for phrases such as /'This amendment is effective/'. Return this date in YYYY-MM-DD format.",
|
||||
"CONTRACT_TITLE": "What is the complete title of the contract or the document? It is generally written in CAPITAL LETTERS and is found at the beginning of the document before a paragraph containing provider and payer name. It typically contains the words AGREEMENT, AMENDMENT, CONTRACT, ADDENDUM, MEMORANDUM, LETTER, PROVIDER or REQUEST. In case of agreement, typical structure is the provider name, followed by the phrase with agreement, followed potentially by a number signifying a version of the contract. In case of amendment, typical structure is word amendment followed by number, followed by the phrase with agreement. Extract the complete title of the contract or document as accurately as possible. Ensure: All leading characters are included for each word (e.g. 'NETWORK' instead of 'ETWORK'). Spaces between words are preserved in phrases (e.g., 'PROVIDER COLLABORATION' instead of 'PROVIDERCOLLABORATION'). If text appears incomplete or lacks spacing, use logical patterns like capitalization or common formatting rules to infer and correct the output. Replace new lines and line breaks with space.",
|
||||
"PAYER_NAME": "What is the name of the payer that is a party to the contract as stated in the Preamble?",
|
||||
"PROV_GROUP_NAME": "What is the name of the Group provider that is a party to the contract? This is generally found near keyword provider. Only return the name of the Group provider along with dba name.",
|
||||
@@ -487,7 +533,6 @@ AC_DICT = {
|
||||
"RELATIONSHIP_OF_PARTIES_LANGUAGE": "Extract the entire Relationship of Parties passage present in the contract.",
|
||||
}
|
||||
|
||||
|
||||
def AC_EFFECTIVE_DATE_CLEANUP(contract_text):
|
||||
return f"""### Contract Start ### {contract_text} ### Contract End ###
|
||||
Above is a contract. What is the contract effective date mentioned in any of the following locations: the signatory section, the preamble of the agreement, or the start of the amendment? Look for phrases such as /'This amendment is effective/'.
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
PIPE_PATTERN = r'\|([^|]+)\|'
|
||||
BACKTICK_PATTERN = r'`([^`]+)`'
|
||||
TRIPLE_BACKTICK_PATTERN = r'```([^`]+)```'
|
||||
@@ -110,7 +110,6 @@ def convert_to_dict(table_text):
|
||||
tuple: A dictionary with keys mapping to lists of values, and an integer representing the number of elements in each list.
|
||||
"""
|
||||
table_text = clean_tables(table_text)
|
||||
# print(table_text, '\n')
|
||||
table_text_list = table_text.split("]")
|
||||
table_text_list = [item for item in table_text_list if len(item) > 0]
|
||||
|
||||
@@ -120,14 +119,12 @@ def convert_to_dict(table_text):
|
||||
key = key_value_text.split(":")[0].strip(", '")
|
||||
try:
|
||||
value = ":".join(key_value_text.split(":")[1:]).strip() + "]"
|
||||
# print(value, '\n')
|
||||
if len(value) > 1:
|
||||
value_list = ast.literal_eval(value)
|
||||
final_dict[key] = value_list
|
||||
num_elements = len(value_list)
|
||||
except:
|
||||
value = ":".join(key_value_text.split(":")[1:]).strip() + "']"
|
||||
# print(value, '\n')
|
||||
if len(value) > 1:
|
||||
value_list = ast.literal_eval(value)
|
||||
final_dict[key] = value_list
|
||||
|
||||
@@ -29,7 +29,6 @@ def find_tin_numbers(text):
|
||||
|
||||
|
||||
def invoke_llm(context, llm_selected=LLM_SELECTED):
|
||||
print("Invoking LLM.")
|
||||
prompt_data = f"""
|
||||
|
||||
Human: Use the following pieces of context to provide a concise answer to the questions at the end. You must answer in python list format.
|
||||
@@ -91,7 +90,6 @@ def invoke_llm(context, llm_selected=LLM_SELECTED):
|
||||
response_text = response_body["completion"]
|
||||
else:
|
||||
print("Unsupported LLM selected")
|
||||
# print(f"LLM response: {response_text}")
|
||||
except Exception as e:
|
||||
print(f"Error invoking LLM: {e}")
|
||||
response_text = "failed"
|
||||
|
||||
@@ -9,6 +9,10 @@ from collections import defaultdict
|
||||
import config
|
||||
|
||||
|
||||
class InvalidDateException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def read_local(file_path):
|
||||
# Check if the file is a text file
|
||||
if os.path.isfile(file_path) and file_path.endswith(".txt"):
|
||||
@@ -354,4 +358,49 @@ def add_hyphen_if_needed(input_str):
|
||||
if not bool(re.fullmatch(r"\b\d{2}-\d{7}\b", input_str)):
|
||||
input_str = None
|
||||
|
||||
return input_str
|
||||
return input_str
|
||||
|
||||
def convert_to_us_date_format(date_str: str) -> str:
|
||||
"""
|
||||
Input date string is expected to be in the format YYYY-MM-DD
|
||||
|
||||
Ensure the date is in MM/DD/YYYY format. If the month is greater than 12, swap the month and day.
|
||||
If the input is `N/A` or some variant (defined by is_empty), InvalidDateException will be raised.
|
||||
Args:
|
||||
date_str (str): The date string to be formatted. We expect it to come in as YYYY-MM-DD format (note the hyphens as well)
|
||||
|
||||
Returns:
|
||||
str: The date string formatted as MM/DD/YYYY.
|
||||
|
||||
Raises:
|
||||
InvalidDateException: If the date string is not in the expected format.
|
||||
"""
|
||||
|
||||
if not isinstance(date_str, str):
|
||||
raise TypeError(f"Date must be a string, got: {type(date_str)}")
|
||||
|
||||
if is_empty(date_str) or re.fullmatch(r"\d{4}-\d{1,2}-\d{1,2}", date_str) is None:
|
||||
raise InvalidDateException(f"Invalid date format: {date_str}")
|
||||
|
||||
# Split the date string into year, month, and day
|
||||
year, month, day = date_str.split("-")
|
||||
|
||||
year_int = int(year)
|
||||
month_int = int(month)
|
||||
day_int = int(day)
|
||||
|
||||
if year_int == 0 or month_int == 0 or day_int == 0:
|
||||
raise InvalidDateException(f"Invalid date format: {date_str}")
|
||||
|
||||
if month_int > 12 and 0 < day_int <= 31:
|
||||
# Swap the month and day if the month is greater than 12
|
||||
month_int, day_int = day_int, month_int
|
||||
|
||||
if 0 < month_int <= 12 and 0 < day_int <= 31:
|
||||
# Return the formatted date in MM/DD/YYYY format
|
||||
year, month, day = str(year_int).zfill(4), str(month_int).zfill(2), str(day_int).zfill(2)
|
||||
return "/".join([month, day, year])
|
||||
|
||||
else:
|
||||
# Raise an exception if the month or day is out of range
|
||||
raise InvalidDateException(f"Invalid date format: {date_str}")
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import pytest
|
||||
from enums.delimiters import Delimiter
|
||||
from ac_funcs import extract_text_from_delimiters
|
||||
|
||||
|
||||
class TestExtractTextFromDelimiters:
|
||||
# Test case for valid inputs
|
||||
|
||||
def test_pipe_delimiter_match_first_position(self):
|
||||
raw_text = "Here is the answer |this is a pipe answer| and |more text.|"
|
||||
result = extract_text_from_delimiters(raw_text,delimiter=Delimiter.PIPE,match_index=0)
|
||||
assert result == "this is a pipe answer"
|
||||
|
||||
def test_pipe_delimiter_match_last_position(self):
|
||||
raw_text = "Here is the answer |this is a pipe answer| and |more text.|"
|
||||
result = extract_text_from_delimiters(raw_text,delimiter=Delimiter.PIPE,match_index=-1)
|
||||
assert result == "more text."
|
||||
|
||||
def test_pipe_delimiter_match_any_position(self):
|
||||
raw_text = "|Here| is the answer |this is a pipe answer| and |more text.|"
|
||||
result = extract_text_from_delimiters(raw_text,delimiter=Delimiter.PIPE,match_index=1)
|
||||
assert result == "this is a pipe answer"
|
||||
|
||||
def test_backtick_delimiter_match_first_position(self):
|
||||
raw_text = "Here is the answer `this is a backtick answer` and `more text.`"
|
||||
result = extract_text_from_delimiters(raw_text, delimiter=Delimiter.BACKTICK, match_index=0)
|
||||
assert result == "this is a backtick answer"
|
||||
|
||||
def test_backtick_delimiter_match_last_position(self):
|
||||
raw_text = "Here is the answer `this is a backtick answer` and `more text.`"
|
||||
result = extract_text_from_delimiters(raw_text, delimiter=Delimiter.BACKTICK, match_index=-1)
|
||||
assert result == "more text."
|
||||
|
||||
def test_backtick_delimiter_match_any_position(self):
|
||||
raw_text = "`Here` is the answer `this` is a `backtick answer` and `more text.`"
|
||||
result = extract_text_from_delimiters(raw_text, delimiter=Delimiter.BACKTICK, match_index=2)
|
||||
assert result == "backtick answer"
|
||||
|
||||
def test_triple_backtick_delimiter_match_first_position(self):
|
||||
raw_text = "Here is the answer ```this is a triple backtick answer``` and more text."
|
||||
result = extract_text_from_delimiters(raw_text, delimiter=Delimiter.TRIPLE_BACKTICK, match_index=0)
|
||||
assert result == "this is a triple backtick answer"
|
||||
|
||||
def test_triple_backtick_delimiter_match_last_position(self):
|
||||
raw_text = "Here is the answer ```this``` is a ```triple``` backtick ```answer``` and more text."
|
||||
result = extract_text_from_delimiters(raw_text, delimiter=Delimiter.TRIPLE_BACKTICK, match_index=-1)
|
||||
assert result == "answer"
|
||||
|
||||
def test_triple_backtick_delimiter_match_any_position(self):
|
||||
raw_text = "```Here``` is the answer ```this is a triple backtick answer``` and ```more text.```"
|
||||
result = extract_text_from_delimiters(raw_text, delimiter=Delimiter.TRIPLE_BACKTICK, match_index=1)
|
||||
assert result == "this is a triple backtick answer"
|
||||
@@ -0,0 +1,43 @@
|
||||
import pytest
|
||||
from utils import convert_to_us_date_format, InvalidDateException, is_empty
|
||||
|
||||
|
||||
class TestConvertToUSDateFormat:
|
||||
def test_convert_to_us_date_format_valid(self):
|
||||
assert convert_to_us_date_format("2023-12-01") == "12/01/2023"
|
||||
assert convert_to_us_date_format("2023-01-12") == "01/12/2023"
|
||||
assert convert_to_us_date_format("2023-13-01") == "01/13/2023"
|
||||
|
||||
def test_convert_to_us_date_format_invalid(self):
|
||||
with pytest.raises(InvalidDateException):
|
||||
convert_to_us_date_format("2023-13-32")
|
||||
with pytest.raises(InvalidDateException):
|
||||
convert_to_us_date_format("2023-00-01")
|
||||
with pytest.raises(InvalidDateException):
|
||||
convert_to_us_date_format("2023-12-00")
|
||||
with pytest.raises(InvalidDateException):
|
||||
convert_to_us_date_format("2023-12-32")
|
||||
with pytest.raises(InvalidDateException):
|
||||
convert_to_us_date_format("2023-13-13")
|
||||
|
||||
def test_convert_to_us_date_format_empty(self):
|
||||
with pytest.raises(InvalidDateException):
|
||||
convert_to_us_date_format("")
|
||||
with pytest.raises(InvalidDateException):
|
||||
convert_to_us_date_format("N/A")
|
||||
with pytest.raises(InvalidDateException):
|
||||
convert_to_us_date_format("null")
|
||||
with pytest.raises(InvalidDateException):
|
||||
convert_to_us_date_format("none")
|
||||
|
||||
def test_convert_to_us_date_format_invalid_format(self):
|
||||
with pytest.raises(InvalidDateException):
|
||||
convert_to_us_date_format("2023/12/01")
|
||||
with pytest.raises(InvalidDateException):
|
||||
convert_to_us_date_format("12-01-2023")
|
||||
with pytest.raises(InvalidDateException):
|
||||
convert_to_us_date_format("2023.12.01")
|
||||
|
||||
def test_convert_to_us_date_format_invalid_type(self):
|
||||
with pytest.raises(TypeError):
|
||||
convert_to_us_date_format(None)
|
||||
Reference in New Issue
Block a user