Alex Galarce
c094fa35b0
Table Continuation Fix * Add EXHIBIT_HEADER_MARKERS for improved document header recognition in table handling * Refactor EXHIBIT_HEADER_MARKERS definition for improved clarity and organization * remove unused page span tracking code; add unit tests for new functionality * remove debugging logs from combine_continuous_tables function * Refactor logging setup and update test file configuration in investment field testing notebook * Merge remote-tracking branch 'origin/main' into feature/further-phase-1-and-2-fixes Approved-by: Katon Minhas
129 lines
3.6 KiB
Plaintext
129 lines
3.6 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Manual Test Bed\n",
|
|
"Runs contracts through from `input_data`\n",
|
|
"\n",
|
|
"To make this work: in `config.py`, change READ_MODE and RUN_MODE to 'local'.\n",
|
|
"Create an .env in the base directory of your project according to these instructions:\n",
|
|
"https://aarete.atlassian.net/wiki/spaces/DoczyAI/pages/1283751954/Using+.env+for+AWS+keys\n",
|
|
"\n",
|
|
"Feel free to comment out parts of `file_processing.py` that do not apply to the fields you are optimizing. For instance, if you're optimizing a smart chunked field, comment out the `one_to_n_fields` block and feed just `one_to_n_results` into `postprocess.postprocess`."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import logging\n",
|
|
"import sys\n",
|
|
"\n",
|
|
"logging.basicConfig(\n",
|
|
" level=logging.DEBUG,\n",
|
|
" format='%(asctime)s.%(msecs)03d - %(levelname)s - %(message)s',\n",
|
|
" datefmt='%Y-%m-%d %H:%M:%S',\n",
|
|
" stream=sys.stdout, # Output to standard output (for consistent notebook formatting)\n",
|
|
" force=True # Overwrite any previous logging configuration\n",
|
|
")\n",
|
|
"\n",
|
|
"logging.getLogger('botocore').setLevel(logging.WARNING)\n",
|
|
"logging.getLogger('boto3').setLevel(logging.WARNING)\n",
|
|
"logging.getLogger('faiss').setLevel(logging.WARNING)\n",
|
|
"logging.getLogger('urllib3').setLevel(logging.WARNING)\n",
|
|
"logging.getLogger('sentence_transformers').setLevel(logging.WARNING)\n",
|
|
"logging.getLogger('s3transfer').setLevel(logging.WARNING)\n",
|
|
"logging.getLogger('chromadb').setLevel(logging.WARNING)\n",
|
|
"logging.getLogger('chroma').setLevel(logging.WARNING)\n",
|
|
"\n",
|
|
"import os\n",
|
|
"import pandas as pd"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"os.chdir(\"../../\") # go to FieldExtraction folder\n",
|
|
"TEST_DATA_INPUT_DIR = \"scripts/investment_field_testing/input_data\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Also change config.RUN_MODE and config.READ_MODE to 'local'\n",
|
|
"test_params = {}\n",
|
|
"test_params['local_input_dir'] = TEST_DATA_INPUT_DIR\n",
|
|
"test_params['max_workers'] = 1\n",
|
|
"test_params['input_files'] = [ # you can specify specific files to run here\n",
|
|
" \"test_file.txt\", # replace with your test file(s)\n",
|
|
"]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from src.investment.main import main as investment_main"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"df_out, df_error = investment_main(testing=True, test_params=test_params)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"df_out"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "doczy-smart-chunking-py3.12",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.12.7"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|