doczyai-pipelines/archive/scripts/cnc_hotfixes/cnc_hotfix_effective_date_utils.py

import re

import hybrid_smart_chunking_funcs

regex_backticks = r"\|([^|]+)\|"


# TODO: leave this one in here; don't move it to prompts until we get confirmation that we want to handle meridian differently.
def get_effective_date_meridian_prompt(context):
    prompt_template = f""""
        Please analyze the following contract and extract ONLY the Signature Date. Follow these precise rules in order:

        1. Look for Signature Date
        
        2. If multiple matches are found:
        - Prioritize the latest full date
        
        3. If only partial dates are found, return N/A.

        4. Return the date in YYYY-MM-DD format or N/A if date not found. Enclose just your final answer in |pipes|.

        Here is the contract to analyze:

        ## START CONTRACT TEXT ##

        {context}
        
        ## END CONTRACT TEXT ##

        Before returning the output, VERIFY the date format is YYYY-MM-DD. If the date is not found, return N/A.

        Finally state "Answer: |YYYY-MM-DD| or |N/A|"
    """

    return prompt_template


def chunk_with_include_exclude_keywords(
    text_dict: dict[str, str],
    include_keywords: list[str],
    exclude_keywords: list[str],
    case_sensitive: bool = False,
    apply_date_filtering: bool = True,
):
    """Look for pages that include any of the include_keywords and exclude any pages that contain any of the exclude_keywords.

    Args:
        text_dict (dict[str, str]): A dictionary keyed by string page number and valued
            by string page contents
        include_keywords (list[str]): A list of string keywords to include
        exclude_keywords (list[str]): A list of string keywords to exclude
        case_sensitive (bool): A flag for converting all keywords and text to lower case,
            in order to ignore case altogether.
        apply_date_filtering (bool): A flag to filter out the pages not containing a date.

    Returns:
        list[str]: The sorted list of string page numbers that include all include_keywords and exclude any exclude_keywords
    """

    if not case_sensitive:
        include_keywords = [keyword.lower() for keyword in include_keywords]
        exclude_keywords = [keyword.lower() for keyword in exclude_keywords]
        text_dict = {key: value.lower() for key, value in text_dict.items()}

    if apply_date_filtering:
        pages_containing_date = set()

        for page_num, page_content in text_dict.items():
            dates_list = re.findall(hybrid_smart_chunking_funcs.DATE_PATTERN, page_content)

            if len(dates_list) > 0:
                pages_containing_date.add(page_num)

        pages_containing_date = sorted(pages_containing_date, key=int)

        # debug lines for inclusion and exclusion keywords found on each page

        # print(f"Pages containing date: {pages_containing_date}")

        d = list()  # to record details

        for page_num, page_content in text_dict.items():
            if page_num in pages_containing_date:
                inclusion_keywords = set()
                exclusion_keywords = set()

                contains_inclusion_keyword = False

                for keyword in include_keywords:
                    contains_inclusion_keyword = True
                    if keyword in page_content:
                        inclusion_keywords.add(keyword)

                for keyword in exclude_keywords:
                    if keyword in page_content:
                        exclusion_keywords.add(keyword)

                # print(f"Page #: {page_num} contains inclusion keywords: {inclusion_keywords} and contain exclusion keywords: {exclusion_keywords} | Inclusion Flag: {contains_inclusion_keyword}")

                if contains_inclusion_keyword:
                    d.append(
                        {
                            "Page Number": page_num,
                            "Inclusion Keywords": inclusion_keywords,
                            "Exclusion Keywords": exclusion_keywords,
                        }
                    )

                # print(f"Page #: {page_num} contains inclusion keywords: {inclusion_keywords} and contain exclusion keywords: {exclusion_keywords}")

    page_list = []

    for page in text_dict.keys():
        if not apply_date_filtering or (
            apply_date_filtering and page in pages_containing_date
        ):
            if any(
                keyword in text_dict[page] for keyword in include_keywords
            ) and not any(keyword in text_dict[page] for keyword in exclude_keywords):
                # if str(int(page) - 1) in text_dict.keys():
                #     page_list.append(str(int(page) - 1))
                #     print(f"Taking page #: {str(int(page) - 1)} (as buffer for following page)")

                page_list.append(page)
                # print(f"Taking page #: {page}")

                # if str(int(page) + 1) in text_dict.keys():
                #     page_list.append(str(int(page) + 1))
                #     # print(f"Taking page #: {str(int(page) + 1)} (as buffer for previous page)")

    page_list_sorted = sorted([int(page_str) for page_str in set(page_list)])
    page_list_sorted_str = list(map(str, page_list_sorted))

    # print(", ".join([page for page in page_list_sorted_str]))

    # print(f"Obtained total pages: {len(page_list_sorted_str)}")

    return page_list_sorted_str, d