doczyai-pipelines/fieldExtraction/scripts/investment_field_testing/testing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Manual Test Bed\n",
    "Runs contracts through from `input_data`\n",
    "\n",
    "To make this work: in `config.py`, change READ_MODE and RUN_MODE to 'local'.\n",
    "Create an .env in the base directory of your project according to these instructions:\n",
    "https://aarete.atlassian.net/wiki/spaces/DoczyAI/pages/1283751954/Using+.env+for+AWS+keys\n",
    "\n",
    "Feel free to comment out parts of `file_processing.py` that do not apply to the fields you are optimizing.  For instance, if you're optimizing a smart chunked field, comment out the `one_to_n_fields` block and feed just `one_to_n_results` into `postprocess.postprocess`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### BASE IMPORTS ####\n",
    "\n",
    "import logging\n",
    "import sys\n",
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### LOGGING ####\n",
    "\n",
    "os.chdir(\"../../\") # go to FieldExtraction folder\n",
    "\n",
    "import src.config as config\n",
    "# For this testing notebook, we want per-file logging\n",
    "config.ENABLE_PER_FILE_LOGGING = True\n",
    "config.PER_FILE_LOG_LEVEL = \"DEBUG\"\n",
    "\n",
    "import src.utils.logging_utils as logging_utils\n",
    "\n",
    "# Clear any existing handlers first\n",
    "logging.getLogger().handlers.clear()\n",
    "\n",
    "# Set up per-file logging (this includes console output)\n",
    "# logging_utils.setup_per_file_logging()\n",
    "per_file_handler = logging_utils.PerFileHandler()\n",
    "per_file_handler.setLevel(logging.DEBUG)\n",
    "logging.getLogger().addHandler(per_file_handler)\n",
    "\n",
    "# Add a single console handler for notebook monitoring\n",
    "console_handler = logging.StreamHandler(sys.stdout)\n",
    "console_handler.setLevel(logging.INFO) # Only show INFO and above in notebook\n",
    "console_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')\n",
    "console_handler.setFormatter(console_formatter)\n",
    "logging.getLogger().addHandler(console_handler)\n",
    "\n",
    "# Set level for the root logger\n",
    "logging.getLogger().setLevel(logging.DEBUG)\n",
    "\n",
    "# Suppress noisy third-party loggers\n",
    "for logger_name in ['botocore', 'boto3', 'faiss', 'urllib3', 'sentence_transformers', 's3transfer']:\n",
    "    logging.getLogger(logger_name).setLevel(logging.WARNING)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### TEST CONFIG ####\n",
    "\n",
    "TEST_DATA_INPUT_DIR = \"scripts/investment_field_testing/input_data\"\n",
    "\n",
    "# Also change config.RUN_MODE and config.READ_MODE to 'local'\n",
    "test_params = {}\n",
    "test_params['local_input_dir'] = TEST_DATA_INPUT_DIR\n",
    "test_params['max_workers'] = 1\n",
    "test_params['input_files'] = [ # you can specify specific files to run here\n",
    "    \"Akron General Health System_Eleventh Amendment_20171001.txt\", # replace with your test file(s)\n",
    "]\n",
    "\n",
    "# Specify state(s) for testing, if needed\n",
    "# Specify them here instead of in `test_params` bc `config` is initialized \n",
    "# before `test_params` is read in main.py, which means the state-level constants\n",
    "# won't be set correctly if we put it in `test_params`.\n",
    "config.STATE = [\"OH\"] # for single states\n",
    "# config.STATE = [\"OH\", \"CA\"] # for multi-state testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### MAIN IMPORTS ####\n",
    "\n",
    "from src.investment.main import main as investment_main"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### TEST EXECUTION ####\n",
    "\n",
    "df_out, df_error = investment_main(testing=True, test_params=test_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "doczy-field-extraction-py3.12 (3.12.7)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}