Michael McGuinness
166 lines
6.8 KiB
Python
166 lines
6.8 KiB
Python
from itertools import groupby, count
|
|
import re
|
|
|
|
|
|
def clean_newlines(contract_text):
|
|
"""
|
|
Cleans up isolated newlines in a contract text, converting them into spaces to ensure text continuity.
|
|
|
|
This function uses a regular expression to identify standalone newlines (those not part of a paragraph break, which
|
|
typically consists of consecutive newlines) and replaces them with a single space. This process helps in preserving
|
|
the flow of text and making it more readable and processable, particularly for text extraction and analysis tasks that
|
|
might be sensitive to abrupt breaks in the text.
|
|
|
|
Parameters:
|
|
contract_text (str): The text of the contract that may contain sporadic newlines.
|
|
|
|
Returns:
|
|
str: The cleaned contract text with isolated newlines replaced by spaces.
|
|
"""
|
|
cleaned_text = re.sub(r"(?<!\n)\n(?!\n)", " ", contract_text)
|
|
return cleaned_text
|
|
|
|
|
|
def split_text(text):
|
|
"""
|
|
Splits a contract text into a dictionary of pages based on a specific delimiter indicating the start of a page.
|
|
|
|
This function uses the delimiter 'Start of Page No. = ' to segment the input text into different pages.
|
|
Each page is stored in a dictionary where the key is the page number (extracted as the first word after the delimiter)
|
|
and the value is the corresponding page text. This facilitates easier access and processing of individual pages in the contract.
|
|
|
|
Parameters:
|
|
text (str): The contract text containing multiple pages separated by the delimiter.
|
|
|
|
Returns:
|
|
dict: A dictionary with page numbers as keys and the respective page texts as values.
|
|
"""
|
|
text_list = text.split("Start of Page No. = ")
|
|
text_dict = {page.split()[0]: page for page in text_list}
|
|
return text_dict
|
|
|
|
|
|
def highlight_rates(text_dict):
|
|
"""
|
|
Highlights rate-related terms (percentages and dollar amounts) in the text of each page within a dictionary.
|
|
|
|
This function processes each page's text in the provided dictionary, identifying words that contain percentage signs ('%')
|
|
or dollar signs ('$'). It wraps these words with '>>>' and '<<<' to highlight them. The modified text for each page
|
|
is then updated in the dictionary, allowing for easier identification and processing of rate-related terms in subsequent analyses.
|
|
|
|
Parameters:
|
|
text_dict (dict): A dictionary where keys are page numbers and values are the text content of those pages.
|
|
|
|
Returns:
|
|
dict: The updated dictionary with rate-related terms highlighted in the text of each page.
|
|
"""
|
|
for page, text in text_dict.items():
|
|
words = text.split()
|
|
highlighted_words = []
|
|
for word in words:
|
|
if "%" in word or "$" in word:
|
|
word = f">>>{word}<<<"
|
|
highlighted_words.append(word)
|
|
text_dict[page] = " ".join(highlighted_words)
|
|
return text_dict
|
|
|
|
|
|
def chunk_text(text_dict):
|
|
"""
|
|
Creates text chunks from a dictionary of page texts, focusing on pages with special characters (percentages and dollar amounts).
|
|
|
|
This function identifies pages that contain '%' or '$' signs and includes those pages along with their immediate neighbors
|
|
(previous and next pages) to form chunks. The chunks are then grouped and concatenated into single text blocks for easier
|
|
processing. Each chunk is stored in a new dictionary where the keys represent the range of pages included in the chunk.
|
|
|
|
Parameters:
|
|
text_dict (dict): A dictionary where keys are page numbers (as strings) and values are the text content of those pages.
|
|
|
|
Returns:
|
|
dict: A new dictionary where each key is a string representing the range of pages in a chunk, and each value is the concatenated text of those pages.
|
|
"""
|
|
special_pages = {
|
|
int(page): text
|
|
for page, text in text_dict.items()
|
|
if (("%" in text) or ("$" in text) or ("percent" in text))
|
|
}
|
|
page_numbers = sorted(special_pages.keys())
|
|
chunk_page_numbers = []
|
|
for page in page_numbers:
|
|
chunk_page_numbers.append(page - 1)
|
|
chunk_page_numbers.append(page)
|
|
chunk_page_numbers.append(page + 1)
|
|
chunk_page_numbers = list(set(chunk_page_numbers))
|
|
chunk_page_numbers = [
|
|
page for page in chunk_page_numbers if str(page) in text_dict.keys()
|
|
]
|
|
chunks = [
|
|
list(group)
|
|
for key, group in groupby(chunk_page_numbers, lambda x, c=count(): x - next(c))
|
|
]
|
|
chunk_dict = {}
|
|
for item in chunks:
|
|
dict_key = f"{min(item)}-{max(item)}"
|
|
text = "".join([text_dict[str(page)] for page in item])
|
|
chunk_dict[dict_key] = text
|
|
return chunk_dict
|
|
|
|
|
|
def clean_billed_charges(contract_text):
|
|
"""
|
|
Cleans occurrences of 'billed charges' in a text by replacing them with '100% of billed charges' unless preceded by a percentage.
|
|
|
|
This function uses a regular expression to find all instances of the phrase 'billed charges'. For each match, it checks the preceding
|
|
text (up to 15 characters before the match) for a percentage sign ('%'). If a percentage is found, it leaves the match unchanged.
|
|
Otherwise, it replaces 'billed charges' with '100% of billed charges'.
|
|
|
|
Parameters:
|
|
text (str): The input text to be cleaned.
|
|
|
|
Returns:
|
|
str: The cleaned text with appropriate replacements made for 'billed charges'.
|
|
"""
|
|
|
|
substrings = [
|
|
"Physician's Billed Charges",
|
|
"Provider's Billed Charges",
|
|
"Allowable Billed Charges",
|
|
"Hospital's Billed Charges",
|
|
"Physician's Charges",
|
|
"Provider's Charges",
|
|
"Allowable Charges",
|
|
"Hospital's Charges",
|
|
"Billed Charges",
|
|
"the rates set forth in this Exhibit",
|
|
"the rutes set forth in this Exhibit",
|
|
]
|
|
max_substring_length = max([len(s) for s in substrings])
|
|
|
|
def find_substring_indices(contract_text, s):
|
|
indices = []
|
|
lower_contract_text = contract_text.lower()
|
|
lower_s = s.lower()
|
|
index = lower_contract_text.find(lower_s)
|
|
while index != -1:
|
|
indices.append(index)
|
|
index = lower_contract_text.find(lower_s, index + 1)
|
|
return indices
|
|
|
|
for s in substrings:
|
|
indices = find_substring_indices(contract_text, s)
|
|
index_adder = 0
|
|
if indices:
|
|
for i in indices:
|
|
index = i + index_adder
|
|
end_index = index + max_substring_length
|
|
match_part = contract_text[index:end_index]
|
|
previous = contract_text[max(0, index - 30) : index]
|
|
if "%" not in previous:
|
|
contract_text = (
|
|
contract_text[0:index]
|
|
+ f" 100% of {match_part}"
|
|
+ contract_text[end_index:]
|
|
)
|
|
index_adder += 9
|
|
return contract_text.replace(" ", " ")
|