From e34262fc86f5b87c52f8227b909767d7f5f6bd07 Mon Sep 17 00:00:00 2001 From: Alex Galarce Date: Mon, 3 Feb 2025 22:49:16 +0000 Subject: [PATCH] Merged in test/check-doc-dates-against-test-bed (pull request #380) Test/check doc dates against test bed * first commit * Enhance prompts and processing for contract termination dates - Updated prompt to include 'year to year' as a valid example output. - Implemented case-insensitive processing for contract text and field keywords for smart chunking. - Clarified prompt for derived termination date to specify it is exclusive of renewals * Merge remote-tracking branch 'origin/main' into test/check-doc-dates-against-test-bed * Update termination date prompt tests * Refine keywords for contract termination date prompt in investment prompts Approved-by: Katon Minhas --- .../investment_field_testing/testing.ipynb | 714 ++++++++++++++++++ fieldExtraction/src/investment/main.py | 23 +- .../src/investment/smart_chunking_funcs.py | 6 + .../src/prompts/investment_prompts.json | 4 +- .../src/prompts/investment_prompts.py | 2 +- .../tests/test_investment_prompts.py | 8 +- 6 files changed, 743 insertions(+), 14 deletions(-) create mode 100644 fieldExtraction/scripts/investment_field_testing/testing.ipynb diff --git a/fieldExtraction/scripts/investment_field_testing/testing.ipynb b/fieldExtraction/scripts/investment_field_testing/testing.ipynb new file mode 100644 index 0000000..888a343 --- /dev/null +++ b/fieldExtraction/scripts/investment_field_testing/testing.ipynb @@ -0,0 +1,714 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Manual Test Bed\n", + "Runs contracts through from `input_data`, but cannot yet automatically check them against the answer key (that workbook is not yet clean enough)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "os.chdir(\"../../\") # go to FieldExtraction folder" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(990, 1410)\n" + ] + } + ], + "source": [ + "# Load target data from `scripts/investment_field_testing/input_data` directory\n", + "TEST_DATA_INPUT_DIR = \"scripts/investment_field_testing/input_data\"\n", + "df_targets = pd.read_excel(TEST_DATA_INPUT_DIR + \"/test_bed_investment.xlsx\", sheet_name=\"test_bed_investment\")\n", + "\n", + "print(df_targets.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Sr no.ClientContract NameContract TypeCONTRACT_TITLECONTRACT_HEADERCONTRACT_NUMCONTRACT_BASE_TITLECONTRACT_BASE_EFFECTIVE_DTCONTRACT_SIGNATORY_IND...NEURO_DAY_LIMIT_CARVEOUT_EXCEPTIONNEURO_PD_CARVEOUT_INDNEURO_PD_CARVEOUT_IND_PGNEURO_PD_CARVEOUT_CODESNEURO_PD_CARVEOUT_TYPE_OF_CODENEURO_PD_CARVEOUT_REIMBURSEMENT_METHODOLOGYNEURO_PD_CARVEOUT_SCHEDULE_NAMENEURO_PD_CARVEOUT_SCHEDULE_VERSIONNEURO_PD_CARVEOUT_REIMBURSEMENT_RATENEURO_PD_CARVEOUT_EXCEPTION
01Molina Healthcare of TXCustom_HSA_AMD 2_EFF08012023_Children's Medica...Custom - FacilityAmendment Number Six to Molina Healthcare of T...Amendment6Hospital Services Agreement2012-03-01 00:00:00Y...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
12Molina Healthcare of TXCustom_HSA_AMD 2_EFF08012023_Children's Medica...Custom - FacilityAmendment Number Six to Molina Healthcare of T...Amendment6Hospital Services Agreement2012-03-01 00:00:00Y...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
23Molina Healthcare of TXCustom_HSA_AMD 2_EFF08012023_Children's Medica...Custom - FacilityAmendment Number Six to Molina Healthcare of T...Amendment6Hospital Services Agreement2012-03-01 00:00:00Y...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
34Molina Healthcare of TXCustom_HSA_AMD 2_EFF08012023_Children's Medica...Custom - FacilityAmendment Number Six to Molina Healthcare of T...Amendment6Hospital Services Agreement2012-03-01 00:00:00Y...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
45HealthNet of Cali2017-07-01 St. Agnes Medical Center AMD MU.txtCustom - FacilityAmendment to the Hospital Provider Services Ag...AmendmentNaNHospital Provider Services Agreement2009-09-01 00:00:00Y...NaNNaNNaNNaNNaNNaNNaNY6,733206-33208, 33214, 33222, 33233-33235
\n", + "

5 rows × 1410 columns

\n", + "
" + ], + "text/plain": [ + " Sr no. Client \\\n", + "0 1 Molina Healthcare of TX \n", + "1 2 Molina Healthcare of TX \n", + "2 3 Molina Healthcare of TX \n", + "3 4 Molina Healthcare of TX \n", + "4 5 HealthNet of Cali \n", + "\n", + " Contract Name Contract Type \\\n", + "0 Custom_HSA_AMD 2_EFF08012023_Children's Medica... Custom - Facility \n", + "1 Custom_HSA_AMD 2_EFF08012023_Children's Medica... Custom - Facility \n", + "2 Custom_HSA_AMD 2_EFF08012023_Children's Medica... Custom - Facility \n", + "3 Custom_HSA_AMD 2_EFF08012023_Children's Medica... Custom - Facility \n", + "4 2017-07-01 St. Agnes Medical Center AMD MU.txt Custom - Facility \n", + "\n", + " CONTRACT_TITLE CONTRACT_HEADER \\\n", + "0 Amendment Number Six to Molina Healthcare of T... Amendment \n", + "1 Amendment Number Six to Molina Healthcare of T... Amendment \n", + "2 Amendment Number Six to Molina Healthcare of T... Amendment \n", + "3 Amendment Number Six to Molina Healthcare of T... Amendment \n", + "4 Amendment to the Hospital Provider Services Ag... Amendment \n", + "\n", + " CONTRACT_NUM CONTRACT_BASE_TITLE \\\n", + "0 6 Hospital Services Agreement \n", + "1 6 Hospital Services Agreement \n", + "2 6 Hospital Services Agreement \n", + "3 6 Hospital Services Agreement \n", + "4 NaN Hospital Provider Services Agreement \n", + "\n", + " CONTRACT_BASE_EFFECTIVE_DT CONTRACT_SIGNATORY_IND ... \\\n", + "0 2012-03-01 00:00:00 Y ... \n", + "1 2012-03-01 00:00:00 Y ... \n", + "2 2012-03-01 00:00:00 Y ... \n", + "3 2012-03-01 00:00:00 Y ... \n", + "4 2009-09-01 00:00:00 Y ... \n", + "\n", + " NEURO_DAY_LIMIT_CARVEOUT_EXCEPTION NEURO_PD_CARVEOUT_IND \\\n", + "0 NaN NaN \n", + "1 NaN NaN \n", + "2 NaN NaN \n", + "3 NaN NaN \n", + "4 NaN NaN \n", + "\n", + " NEURO_PD_CARVEOUT_IND_PG NEURO_PD_CARVEOUT_CODES \\\n", + "0 NaN NaN \n", + "1 NaN NaN \n", + "2 NaN NaN \n", + "3 NaN NaN \n", + "4 NaN NaN \n", + "\n", + " NEURO_PD_CARVEOUT_TYPE_OF_CODE NEURO_PD_CARVEOUT_REIMBURSEMENT_METHODOLOGY \\\n", + "0 NaN NaN \n", + "1 NaN NaN \n", + "2 NaN NaN \n", + "3 NaN NaN \n", + "4 NaN NaN \n", + "\n", + " NEURO_PD_CARVEOUT_SCHEDULE_NAME NEURO_PD_CARVEOUT_SCHEDULE_VERSION \\\n", + "0 NaN NaN \n", + "1 NaN NaN \n", + "2 NaN NaN \n", + "3 NaN NaN \n", + "4 NaN Y \n", + "\n", + " NEURO_PD_CARVEOUT_REIMBURSEMENT_RATE NEURO_PD_CARVEOUT_EXCEPTION \n", + "0 NaN NaN \n", + "1 NaN NaN \n", + "2 NaN NaN \n", + "3 NaN NaN \n", + "4 6,7 33206-33208, 33214, 33222, 33233-33235 \n", + "\n", + "[5 rows x 1410 columns]" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_targets.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "# Also change config.RUN_MODE and config.READ_MODE to 'local'\n", + "test_params = {}\n", + "test_params['local_input_dir'] = TEST_DATA_INPUT_DIR + \"/investment_228\"\n", + "test_params['max_workers'] = 1" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "doczy-output exists\n" + ] + } + ], + "source": [ + "from src.investment.main import main as investment_main" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total Input Files : 1\n", + "Processing Adena Health System National Template Agreement C19460229AA.txt...\n", + "Preprocessing Complete - Adena Health System National Template Agreement C19460229AA.txt\n", + "Running regex fields...\n", + "Running smart chunked fields...\n", + "field_group_answer_raw: I'll analyze the contract for each question and provide my reasoning, then give the final answers in JSON format.\n", + "\n", + "CONTRACT_EFFECTIVE_DT:\n", + "The effective date is clearly stated on the signature page: \"Effective Date: 04/01/2021\". This meets the criteria for an explicitly labeled effective date.\n", + "\n", + "CONTRACT_TITLE:\n", + "The title of the contract appears at the beginning of the document: \"CARESOURCE PROVIDER AGREEMENT\". This is in capital letters and contains the word \"AGREEMENT\", which matches the criteria given.\n", + "\n", + "CONTRACT_TERMINATION_DT:\n", + "The contract does not specify a specific termination date. Instead, it states in section 8.01: \"This Agreement shall begin on the Effective Date and shall continue from year to year on the anniversary of the Effective Date thereafter, unless terminated as set forth below.\" This indicates an automatic renewal structure rather than a fixed termination date.\n", + "\n", + "CONTRACT_AUTO_RENEWAL_IND:\n", + "Based on section 8.01, which states the agreement continues from year to year unless terminated, this contract does automatically renew. Therefore, the answer is Y.\n", + "\n", + "CONTRACT_AUTO_RENEWAL_TERM:\n", + "The auto-renewal term is specified in section 8.01 as \"from year to year\", which indicates a one-year renewal term.\n", + "\n", + "Here's the JSON with the final answers:\n", + "\n", + "{\n", + " \"CONTRACT_EFFECTIVE_DT\": \"04/01/2021\",\n", + " \"CONTRACT_TITLE\": \"CARESOURCE PROVIDER AGREEMENT\",\n", + " \"CONTRACT_TERMINATION_DT\": \"year to year\",\n", + " \"CONTRACT_AUTO_RENEWAL_IND\": \"Y\",\n", + " \"CONTRACT_AUTO_RENEWAL_TERM\": \"one year\"\n", + "}\n", + "Running full context fields...\n", + "One to One Complete - Adena Health System National Template Agreement C19460229AA.txt\n", + "The following is from a legal contract. Please provide the initial termination date of the contract (exclusive of renewals). Effective date: 04/01/2021; termination information: year to year\n", + "Feel free to justify your answer, but enclose your final answer in |pipes|. If the question cannot be answered with the given context, return |N/A|. \n", + " Based on the information provided, we can determine the initial termination date as follows:\n", + "\n", + "The contract has an effective date of 04/01/2021 (April 1, 2021).\n", + "The termination information states \"year to year,\" which typically means the contract will automatically renew annually unless terminated by either party.\n", + "\n", + "Given this, the initial termination date would be one year from the effective date, which is:\n", + "\n", + "|03/31/2022|\n", + "\n", + "This date represents the end of the first contract year, which is the initial termination date before any automatic renewals would take effect. The contract would then renew for another year starting 04/01/2022 if not terminated by either party.\n", + "Postprocessing Complete - Adena Health System National Template Agreement C19460229AA.txt\n" + ] + } + ], + "source": [ + "df_out = investment_main(testing=True, test_params=test_params)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
CONTRACT_EFFECTIVE_DTCONTRACT_TITLECONTRACT_TERMINATION_DTCONTRACT_AUTO_RENEWAL_INDCONTRACT_AUTO_RENEWAL_TERMCONTRACT_FILE_NAMECLIENT_NAMEAARETE_DERIVED_TERMINATION_DATE
004/01/2021CARESOURCE PROVIDER AGREEMENTyear to yearYone yearAdena Health System National Template Agreemen...Test-Client03/31/2022
\n", + "
" + ], + "text/plain": [ + " CONTRACT_EFFECTIVE_DT CONTRACT_TITLE \\\n", + "0 04/01/2021 CARESOURCE PROVIDER AGREEMENT \n", + "\n", + " CONTRACT_TERMINATION_DT CONTRACT_AUTO_RENEWAL_IND \\\n", + "0 year to year Y \n", + "\n", + " CONTRACT_AUTO_RENEWAL_TERM \\\n", + "0 one year \n", + "\n", + " CONTRACT_FILE_NAME CLIENT_NAME \\\n", + "0 Adena Health System National Template Agreemen... Test-Client \n", + "\n", + " AARETE_DERIVED_TERMINATION_DATE \n", + "0 03/31/2022 " + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_out.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# The following is not functional because we first need to clean the test bed" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['CONTRACT_EFFECTIVE_DT',\n", + " 'CONTRACT_TITLE',\n", + " 'CONTRACT_TERMINATION_DT',\n", + " 'CONTRACT_AUTO_RENEWAL_IND',\n", + " 'CONTRACT_AUTO_RENEWAL_TERM',\n", + " 'AARETE_DERIVED_TERMINATION_DATE']" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "one_to_one_fields = []\n", + "for col in df_out.columns:\n", + " if df_out[col].nunique() == 1:\n", + " # print(col, df_out[col].nunique())\n", + " one_to_one_fields.append(col)\n", + "\n", + "exclude_list = ['CONTRACT_CARVEOUT_IND', # 1:n\n", + " \"CONTRACT_GREATER_OF_IND\", # 1:n\n", + " \"CONTRACT_REIMBURSEMENT_FEE_RATE\", # 1:n\n", + " \"CONTRACT_NETWORK\", # dynamic\n", + " \"CONTRACT_CLAIM_TYPE_CD\", # not found in the source spreadsheet\n", + " \"CONTRACT_AMENDMENT_NUM\", # not found in the source spreadsheet\n", + " \"PROV_GROUP_NAME_FULL\", # not found in the source spreadsheet\n", + " \"PROV_FULL_NAME\", # This is called PROV_GROUP_NAME in the source spreadsheet\n", + " \"CONTRACT_FILE_NAME\", # not found in the source spreadsheet\n", + " \"CONTRACT_CARVEOUT_CD\", # 1:n, also not found in the source spreadsheet\n", + " 'PAYER_NAME',\n", + " 'CLIENT_NAME',\n", + " \n", + " ]\n", + "\n", + "one_to_one_fields = [item for item in one_to_one_fields \n", + " if item not in exclude_list]\n", + "\n", + "one_to_one_fields" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "mapping_main_to_testbed = {\"CONTRACT_AUTO_RENEWAL_TERM\" : \"CONTRACT_AUTO_RENEWAL_TERM_LENGTH\", \"AARETE_DERIVED_TERMINATION_DATE\" : \"CONTRACT_TERMINATION_DT\"}\n", + "\n", + "def apply_name_mapping(col: str):\n", + " if col in mapping_main_to_testbed:\n", + " return mapping_main_to_testbed[col]\n", + " return col" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 2023-01-08 00:00:00\n", + "1 2023-01-08 00:00:00\n", + "2 2023-01-08 00:00:00\n", + "3 2023-01-08 00:00:00\n", + "4 2017-01-07 00:00:00\n", + "Name: CONTRACT_EFFECTIVE_DT, dtype: object\n", + "0 Amendment Number Six to Molina Healthcare of T...\n", + "1 Amendment Number Six to Molina Healthcare of T...\n", + "2 Amendment Number Six to Molina Healthcare of T...\n", + "3 Amendment Number Six to Molina Healthcare of T...\n", + "4 Amendment to the Hospital Provider Services Ag...\n", + "Name: CONTRACT_TITLE, dtype: object\n", + "0 2025-01-08 00:00:00\n", + "1 2025-01-08 00:00:00\n", + "2 2025-01-08 00:00:00\n", + "3 2025-01-08 00:00:00\n", + "4 30-06-2019\n", + "Name: CONTRACT_TERMINATION_DT, dtype: object\n", + "0 N\n", + "1 N\n", + "2 N\n", + "3 N\n", + "4 Y\n", + "Name: CONTRACT_AUTO_RENEWAL_IND, dtype: object\n", + "0 NaN\n", + "1 NaN\n", + "2 NaN\n", + "3 NaN\n", + "4 1 year\n", + "Name: CONTRACT_AUTO_RENEWAL_TERM_LENGTH, dtype: object\n", + "0 2025-01-08 00:00:00\n", + "1 2025-01-08 00:00:00\n", + "2 2025-01-08 00:00:00\n", + "3 2025-01-08 00:00:00\n", + "4 30-06-2019\n", + "Name: CONTRACT_TERMINATION_DT, dtype: object\n" + ] + } + ], + "source": [ + "exclude_list = [\"CONTRACT_CLAIM_TYPE_CD\"]\n", + "\n", + "for field in one_to_one_fields:\n", + " field = apply_name_mapping(field)\n", + " # if field == \"CONTRACT_AUTO_RENEWAL_TERM\":\n", + " # field = \"CONTRACT_AUTO_RENEWAL_TERM_LENGTH\"\n", + " # elif field == \"AARETE_DERIVED_TERMINATION_DATE\":\n", + " # field = \"CONTRACT_TERMINATION_DT\"\n", + " # elif field == \"CONTRACT_CLAIM_TYPE_CD\":\n", + " print(df_targets[field].head())" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "CONTRACT_EFFECTIVE_DT 04/01/2021\n", + "CONTRACT_TITLE CARESOURCE PROVIDER AGREEMENT\n", + "CONTRACT_TERMINATION_DT year to year\n", + "CONTRACT_AUTO_RENEWAL_IND Y\n", + "CONTRACT_AUTO_RENEWAL_TERM one year\n", + "AARETE_DERIVED_TERMINATION_DATE 03/31/2022\n", + "Name: 0, dtype: object" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "row = df_out[one_to_one_fields].iloc[0]\n", + "row" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CONTRACT_EFFECTIVE_DT LLM answer: 04/01/2021\n", + "the date set forth on the signature page \n", + "\n", + "CONTRACT_TITLE LLM answer: CARESOURCE PROVIDER AGREEMENT\n", + "CARESOURCE_x000D_\n", + "PROVIDER AGREEMENT \n", + "\n", + "CONTRACT_TERMINATION_DT LLM answer: year to year\n", + "2022-01-04 00:00:00 \n", + "\n", + "CONTRACT_AUTO_RENEWAL_IND LLM answer: Y\n", + "Yes \n", + "\n", + "CONTRACT_AUTO_RENEWAL_TERM LLM answer: one year\n", + "1 year \n", + "\n", + "AARETE_DERIVED_TERMINATION_DATE LLM answer: 03/31/2022\n", + "2022-01-04 00:00:00 \n", + "\n" + ] + } + ], + "source": [ + "filter_target = df_targets[df_targets['Contract Name'] == 'Adena Health System National Template Agreement C19460229AA.txt']\n", + "\n", + "for field, answer in row.items():\n", + " print(field, f\"LLM answer: {answer}\")\n", + " print(filter_target[apply_name_mapping(field)].iloc[0],\"\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "# filter_target" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "doczy-smart-chunking-blZ6pB_m-py3.12", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.7" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/fieldExtraction/src/investment/main.py b/fieldExtraction/src/investment/main.py index 70fc342..0e0d074 100644 --- a/fieldExtraction/src/investment/main.py +++ b/fieldExtraction/src/investment/main.py @@ -16,25 +16,34 @@ from src import config # from src.tracking.batch_tracking import BatchTracker -def main(): +def main(testing=False, test_params = {}): run_timestamp = datetime.now().strftime(f"run_%Y%m%d_%H:%M_{config.BATCH_ID}") # Read and process input - input_dict = io_utils.read_input() - print(f"Total Input Files : {len(input_dict)}") + if not testing: + input_dict = io_utils.read_input() + max_workers = config.MAX_WORKERS + else: + input_dict = io_utils.read_input(test_params['local_input_dir']) + max_workers = test_params['max_workers'] + + print(f"Total Input Files : {len(input_dict)}") # Process files with run_timestamp all_results = [] - with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor: + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: results = executor.map(lambda item: file_processing.process_file(item, run_timestamp), input_dict.items()) all_results.extend(results) FINAL_RESULT_DF = pd.concat(all_results) - if config.WRITE_TO_S3: - io_utils.write_s3(FINAL_RESULT_DF, "", run_timestamp, "final") + if not testing: + if config.WRITE_TO_S3: + io_utils.write_s3(FINAL_RESULT_DF, "", run_timestamp, "final") + else: + io_utils.write_local(FINAL_RESULT_DF, "", run_timestamp, "final") else: - io_utils.write_local(FINAL_RESULT_DF, "", run_timestamp, "final") + return FINAL_RESULT_DF if __name__ == "__main__": diff --git a/fieldExtraction/src/investment/smart_chunking_funcs.py b/fieldExtraction/src/investment/smart_chunking_funcs.py index 7730a11..ff942bd 100644 --- a/fieldExtraction/src/investment/smart_chunking_funcs.py +++ b/fieldExtraction/src/investment/smart_chunking_funcs.py @@ -108,6 +108,9 @@ def field_context(one_to_one_fields: FieldSet, contract_text: str, filename: str """ chroma_filename = f"{filename}_chroma" + # Convert contract_text to lowercase for case-insensitive processing + contract_text = contract_text.lower() + text_splitter = RecursiveCharacterTextSplitter(chunk_size=CHUNK_SIZE, chunk_overlap=CHUNK_OVERLAP) splits = text_splitter.split_text(contract_text) @@ -127,6 +130,9 @@ def field_context(one_to_one_fields: FieldSet, contract_text: str, filename: str questions[field.field_name] = field.get_prompt() question = json.dumps(questions) + # Convert field keywords to lowercase for case-insensitive processing + field_keywords = [keyword.lower() for keyword in field.keywords] + # get relevant chunks based on semantic search and keyword search and merge them semantic_relevant_docs = vectorstore_retriever.invoke(question) keyword_relevant_docs = keyword_retriever.invoke(' ;'.join(field.keywords)) diff --git a/fieldExtraction/src/prompts/investment_prompts.json b/fieldExtraction/src/prompts/investment_prompts.json index 9168656..9a66c9a 100644 --- a/fieldExtraction/src/prompts/investment_prompts.json +++ b/fieldExtraction/src/prompts/investment_prompts.json @@ -191,8 +191,8 @@ "field_name": "CONTRACT_TERMINATION_DT", "relationship": "one_to_one", "field_type": "smart_chunked", - "prompt": "What is the contract termination date? This might be listed within the Terms or Terms of Agreement section of the contract. This may either be a date, or a duration. Example outputs: '12/31/2024' or 'three years from the contract effective date'.", - "keywords" : ["TERM AND TERMINATION", "Term and Termination", "Term and termination", "term and termination", "Term"], + "prompt": "What is the contract termination date? This might be listed within the Terms or Terms of Agreement section of the contract. This may either be a date, or a duration. Example outputs: '12/31/2024' or 'three years from the contract effective date' or 'year to year'.", + "keywords" : ["term and termination", "Term"], "format": "date" }, { diff --git a/fieldExtraction/src/prompts/investment_prompts.py b/fieldExtraction/src/prompts/investment_prompts.py index fd98f27..85516f8 100644 --- a/fieldExtraction/src/prompts/investment_prompts.py +++ b/fieldExtraction/src/prompts/investment_prompts.py @@ -376,7 +376,7 @@ def derived_term_date_prompt(effective_date:str=None, termination_information:st Returns: str: A formatted string prompt requesting the termination date of the contract. """ - return f"The following is from a legal contract. Please provide the termination date of the contract. Effective date: {effective_date}; termination information: {termination_information}\n{PIPE_FORMAT_INSTRUCTIONS}" + return f"The following is from a legal contract. Please provide the initial termination date of the contract (exclusive of renewals). Effective date: {effective_date}; termination information: {termination_information}\n{PIPE_FORMAT_INSTRUCTIONS}" # KATON: Any thoughts on if this function is ok here or if it should be moved elsewhere? @cache diff --git a/fieldExtraction/tests/test_investment_prompts.py b/fieldExtraction/tests/test_investment_prompts.py index 503b655..53f0d64 100644 --- a/fieldExtraction/tests/test_investment_prompts.py +++ b/fieldExtraction/tests/test_investment_prompts.py @@ -5,7 +5,7 @@ def test_derived_term_date_prompt_with_all_params(): effective_date = "January 1, 2024" termination_information = "The contract terminates after 3 years." expected_output = ( - "The following is from a legal contract. Please provide the termination date of the contract. " + "The following is from a legal contract. Please provide the initial termination date of the contract (exclusive of renewals). " "Effective date: January 1, 2024; termination information: The contract terminates after 3 years.\n" "Feel free to justify your answer, but enclose your final answer in |pipes|. If the question cannot be answered with the given context, return |N/A|." ) @@ -15,7 +15,7 @@ def test_derived_term_date_prompt_with_only_effective_date(): effective_date = "January 1, 2024" termination_information = None expected_output = ( - "The following is from a legal contract. Please provide the termination date of the contract. " + "The following is from a legal contract. Please provide the initial termination date of the contract (exclusive of renewals). " "Effective date: January 1, 2024; termination information: None\n" "Feel free to justify your answer, but enclose your final answer in |pipes|. If the question cannot be answered with the given context, return |N/A|." ) @@ -25,7 +25,7 @@ def test_derived_term_date_prompt_with_only_termination_information(): effective_date = None termination_information = "The contract terminates after 3 years." expected_output = ( - "The following is from a legal contract. Please provide the termination date of the contract. " + "The following is from a legal contract. Please provide the initial termination date of the contract (exclusive of renewals). " "Effective date: None; termination information: The contract terminates after 3 years.\n" "Feel free to justify your answer, but enclose your final answer in |pipes|. If the question cannot be answered with the given context, return |N/A|." ) @@ -35,7 +35,7 @@ def test_derived_term_date_prompt_with_no_params(): effective_date = None termination_information = None expected_output = ( - "The following is from a legal contract. Please provide the termination date of the contract. " + "The following is from a legal contract. Please provide the initial termination date of the contract (exclusive of renewals). " "Effective date: None; termination information: None\n" "Feel free to justify your answer, but enclose your final answer in |pipes|. If the question cannot be answered with the given context, return |N/A|." )