Alex Galarce
3be2f44cb1
Feature/global lesser of * Merge remote-tracking branch 'origin/feature/reimb-primary-updates' * First try at global "lesser of" * Generate reimb_ids further upstream * Implement global lesser of injection and extraction logic in file processing * Update logic to change LESSER_OF_IND in-place for global affected rows; add docstrings * Refactor global lesser of injection logic to create new rows for REIMB_IDs lacking lesser of logic; update original rows in place and improve tracking flags. * Merge remote-tracking branch 'origin/main' into feature/global-lesser-of * Add logging for global lesser of logic injection and update handling of original rows * Refactor global lesser of logic: move functions to one_to_one_funcs.py and update inject_global_lesser_of_rows to use new implementations * Move imports for METHODOLOGY_BREAKOUT_QUESTIONS and REIMB_TERM_BREAKOUT to the top of one_to_one_funcs.py for better organization and to remove prototype comments. * isort * Refactor import statement for generate_reimb_ids to use the full module path for clarity * load field from JSON instead of hardcoded values * Update logging levels in investment_field_testing notebook * Merged main into feature/global-lesser-of * Remove unnecessary comment from investment_postprocessing_funcs import statement for clarity * Merge branch 'feature/global-lesser-of' of https://bitbucket.org/aarete/doczy.ai into feature/global-lesser-of * Merge remote-tracking branch 'origin/main' into feature/global-lesser-of * Remove debug print statement from process_file function * fix call to Field.load_from_file * Merge remote-tracking branch 'origin/main' into feature/global-lesser-of * Add tracking flag for global lesser of logic when none is found * explicitly sort by FILE_NAME, EXHIBIT_PAGE, and REIMB_LESSER_OF_ID columns * Add new field type, "full_context_separate", to make sure that we don't run it twice. Also formatting fix * Fix iteration over separate fields in run_full_context_fields function Approved-by: Katon Minhas
114 lines
3.6 KiB
Plaintext
114 lines
3.6 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Manual Test Bed\n",
|
|
"Runs contracts through from `input_data`\n",
|
|
"\n",
|
|
"To make this work: in `config.py`, change READ_MODE and RUN_MODE to 'local'.\n",
|
|
"Create an .env in the base directory of your project according to these instructions:\n",
|
|
"https://aarete.atlassian.net/wiki/spaces/DoczyAI/pages/1283751954/Using+.env+for+AWS+keys\n",
|
|
"\n",
|
|
"Feel free to comment out parts of `file_processing.py` that do not apply to the fields you are optimizing. For instance, if you're optimizing a smart chunked field, comment out the `one_to_n_fields` block and feed just `one_to_n_results` into `postprocess.postprocess`."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import logging\n",
|
|
"\n",
|
|
"logging.basicConfig(\n",
|
|
" level=logging.DEBUG,\n",
|
|
" format='%(asctime)s.%(msecs)03d - %(levelname)s - %(message)s',\n",
|
|
" datefmt='%Y-%m-%d %H:%M:%S',\n",
|
|
" force=True # Overwrite any previous logging configuration\n",
|
|
")\n",
|
|
"\n",
|
|
"logging.getLogger('botocore').setLevel(logging.WARNING)\n",
|
|
"logging.getLogger('boto3').setLevel(logging.WARNING)\n",
|
|
"logging.getLogger('faiss').setLevel(logging.WARNING)\n",
|
|
"logging.getLogger('urllib3').setLevel(logging.WARNING)\n",
|
|
"logging.getLogger('sentence_transformers').setLevel(logging.WARNING)\n",
|
|
"logging.getLogger('s3transfer').setLevel(logging.WARNING)\n",
|
|
"logging.getLogger('chromadb').setLevel(logging.WARNING)\n",
|
|
"logging.getLogger('chroma').setLevel(logging.WARNING)\n",
|
|
"\n",
|
|
"import os\n",
|
|
"import pandas as pd"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"os.chdir(\"../../\") # go to FieldExtraction folder\n",
|
|
"TEST_DATA_INPUT_DIR = \"scripts/investment_field_testing/input_data\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Also change config.RUN_MODE and config.READ_MODE to 'local'\n",
|
|
"test_params = {}\n",
|
|
"test_params['local_input_dir'] = TEST_DATA_INPUT_DIR\n",
|
|
"test_params['max_workers'] = 1\n",
|
|
"test_params['input_files'] = [ # you can specify specific files to run here\n",
|
|
" # \"00-125434-Central MS Diagnostic, LLC-ICMProviderAgreement_78285.txt\",\n",
|
|
" \"02-0677066-Eyemasters-ICMProviderAgreement_64220_2.txt\",\n",
|
|
" # \"CustomFac_AMD - AMD - UNIVERSITY MEDICAL CENTER OF EL PASO - 74-6000756 MU.txt\", # short contract for testing\n",
|
|
" # \"20-2401676-Community Hospital of LaGrange County, Inc dba Parkview LaGrange Hospital-ICMProviderAgreement_89356_1.txt\",\n",
|
|
" ]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from src.investment.main import main as investment_main"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"df_out, df_error = investment_main(testing=True, test_params=test_params)"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": ".venv",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.12.7"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|