diff --git a/.gitignore b/.gitignore index a6168a1..386ddb7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,89 +1,89 @@ -# These are some examples of commonly ignored file patterns. -# You should customize this list as applicable to your project. -# Learn more about .gitignore: -# https://www.atlassian.com/git/tutorials/saving-changes/gitignore - -# Node artifact files -node_modules/ -dist/ - -# Compiled Java class files -*.class - -# Compiled Python bytecode -*.py[cod] - -# Log files -*.log - -# Package files -*.jar - -# Maven -target/ -dist/ - -# JetBrains IDE -.idea/ - -# Unit test reports -TEST*.xml - -# Generated by MacOS -.DS_Store - -# Generated by Windows -Thumbs.db - -# Applications -*.app -*.exe -*.war - -# Large media files -*.mp4 -*.tiff -*.avi -*.flv -*.mov -*.wmv - -# Terraform files -.terraform/ -terraform.tfstate.backup -terraform.tfstate -.terraform.lock.hcl -build/ -textract-pipeline/terraform/builds/ - -# Data Files -streamlit/history.csv -streamlit/RESULTS -streamlit/DB/ -streamlit/RAW_DOCUMENTS/ -streamlit/SOURCE_DOCUMENTS/ -streamlit/contract_field_values.csv -streamlit/contract_fields.csv -streamlit/sample.csv -streamlit/temp1.csv -streamlit/temp2.csv -streamlit/results.csv - -# env -streamlit/venv - -*.tfplan -*.pem -textfiles/ -texts/ -*.csv -*.json -myenv/ -*.env -subset/ -output/ -docs/ - -.mypy_cache/ -.pytest_cache/ -__pycache__/ \ No newline at end of file +# These are some examples of commonly ignored file patterns. +# You should customize this list as applicable to your project. +# Learn more about .gitignore: +# https://www.atlassian.com/git/tutorials/saving-changes/gitignore + +# Node artifact files +node_modules/ +dist/ + +# Compiled Java class files +*.class + +# Compiled Python bytecode +*.py[cod] + +# Log files +*.log + +# Package files +*.jar + +# Maven +target/ +dist/ + +# JetBrains IDE +.idea/ + +# Unit test reports +TEST*.xml + +# Generated by MacOS +.DS_Store + +# Generated by Windows +Thumbs.db + +# Applications +*.app +*.exe +*.war + +# Large media files +*.mp4 +*.tiff +*.avi +*.flv +*.mov +*.wmv + +# Terraform files +.terraform/ +terraform.tfstate.backup +terraform.tfstate +.terraform.lock.hcl +build/ +textract-pipeline/terraform/builds/ + +# Data Files +streamlit/history.csv +streamlit/RESULTS +streamlit/DB/ +streamlit/RAW_DOCUMENTS/ +streamlit/SOURCE_DOCUMENTS/ +streamlit/contract_field_values.csv +streamlit/contract_fields.csv +streamlit/sample.csv +streamlit/temp1.csv +streamlit/temp2.csv +streamlit/results.csv + +# env +streamlit/venv + +*.tfplan +*.pem +textfiles/ +texts/ +*.csv +*.json +myenv/ +*.env +subset/ +output/ +docs/ + +.mypy_cache/ +.pytest_cache/ +__pycache__/ diff --git a/fieldExtraction/.gitignore b/fieldExtraction/.gitignore new file mode 100644 index 0000000..69d4fed --- /dev/null +++ b/fieldExtraction/.gitignore @@ -0,0 +1,87 @@ +# These are some examples of commonly ignored file patterns. +# You should customize this list as applicable to your project. +# Learn more about .gitignore: +# https://www.atlassian.com/git/tutorials/saving-changes/gitignore + +# Node artifact files +node_modules/ +dist/ + +# Compiled Java class files +*.class + +# Compiled Python bytecode +*.py[cod] + +# Log files +*.log + +# Package files +*.jar + +# Maven +target/ +dist/ + +# JetBrains IDE +.idea/ + +# Unit test reports +TEST*.xml + +# Generated by MacOS +.DS_Store + +# Generated by Windows +Thumbs.db + +# Applications +*.app +*.exe +*.war + +# Large media files +*.mp4 +*.tiff +*.avi +*.flv +*.mov +*.wmv + +# Data Files +streamlit/history.csv +streamlit/RESULTS +streamlit/DB/ +streamlit/RAW_DOCUMENTS/ +streamlit/SOURCE_DOCUMENTS/ +streamlit/contract_field_values.csv +# streamlit/contract_fields.csv +streamlit/sample.csv +streamlit/temp1.csv +streamlit/temp2.csv +streamlit/results.csv + +# env +streamlit/venv + +textfiles/ +texts/ +*.csv +*.json +myenv/ +*.env +*.txt +*.pdf +!requirements.txt +subset/ +output/ +docs/ +allfiles/ +CNC_testruns_AC/ +cnc-abc-output/ +humana-input/ +humana-output/ +output_consolidated/ +*.zip +*.xlsx + diff --git a/fieldExtraction/.python-version b/fieldExtraction/.python-version new file mode 100644 index 0000000..56bb660 --- /dev/null +++ b/fieldExtraction/.python-version @@ -0,0 +1 @@ +3.12.7 diff --git a/fieldExtraction/poetry.lock b/fieldExtraction/poetry.lock new file mode 100644 index 0000000..a991364 --- /dev/null +++ b/fieldExtraction/poetry.lock @@ -0,0 +1,1321 @@ +# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. + +[[package]] +name = "annotated-types" +version = "0.7.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.8" +files = [ + {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, + {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, +] + +[[package]] +name = "anthropic" +version = "0.36.2" +description = "The official Python library for the anthropic API" +optional = false +python-versions = ">=3.7" +files = [ + {file = "anthropic-0.36.2-py3-none-any.whl", hash = "sha256:308ddc6c538de03c081552e456bc0b387b6f7c7d1dea0c20122cc11c7cdbaf6a"}, + {file = "anthropic-0.36.2.tar.gz", hash = "sha256:d5a3fa56d1c82a51944f9dc7b0dc72048deb89f8df5ebfd09e2d1b59c62eb8eb"}, +] + +[package.dependencies] +anyio = ">=3.5.0,<5" +distro = ">=1.7.0,<2" +httpx = ">=0.23.0,<1" +jiter = ">=0.4.0,<1" +pydantic = ">=1.9.0,<3" +sniffio = "*" +tokenizers = ">=0.13.0" +typing-extensions = ">=4.7,<5" + +[package.extras] +bedrock = ["boto3 (>=1.28.57)", "botocore (>=1.31.57)"] +vertex = ["google-auth (>=2,<3)"] + +[[package]] +name = "anyio" +version = "4.6.2.post1" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +optional = false +python-versions = ">=3.9" +files = [ + {file = "anyio-4.6.2.post1-py3-none-any.whl", hash = "sha256:6d170c36fba3bdd840c73d3868c1e777e33676a69c3a72cf0a0d5d6d8009b61d"}, + {file = "anyio-4.6.2.post1.tar.gz", hash = "sha256:4c8bc31ccdb51c7f7bd251f51c609e038d63e34219b44aa86e47576389880b4c"}, +] + +[package.dependencies] +idna = ">=2.8" +sniffio = ">=1.1" + +[package.extras] +doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21.0b1)"] +trio = ["trio (>=0.26.1)"] + +[[package]] +name = "black" +version = "24.10.0" +description = "The uncompromising code formatter." +optional = false +python-versions = ">=3.9" +files = [ + {file = "black-24.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e6668650ea4b685440857138e5fe40cde4d652633b1bdffc62933d0db4ed9812"}, + {file = "black-24.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1c536fcf674217e87b8cc3657b81809d3c085d7bf3ef262ead700da345bfa6ea"}, + {file = "black-24.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:649fff99a20bd06c6f727d2a27f401331dc0cc861fb69cde910fe95b01b5928f"}, + {file = "black-24.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:fe4d6476887de70546212c99ac9bd803d90b42fc4767f058a0baa895013fbb3e"}, + {file = "black-24.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5a2221696a8224e335c28816a9d331a6c2ae15a2ee34ec857dcf3e45dbfa99ad"}, + {file = "black-24.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f9da3333530dbcecc1be13e69c250ed8dfa67f43c4005fb537bb426e19200d50"}, + {file = "black-24.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4007b1393d902b48b36958a216c20c4482f601569d19ed1df294a496eb366392"}, + {file = "black-24.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:394d4ddc64782e51153eadcaaca95144ac4c35e27ef9b0a42e121ae7e57a9175"}, + {file = "black-24.10.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b5e39e0fae001df40f95bd8cc36b9165c5e2ea88900167bddf258bacef9bbdc3"}, + {file = "black-24.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d37d422772111794b26757c5b55a3eade028aa3fde43121ab7b673d050949d65"}, + {file = "black-24.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:14b3502784f09ce2443830e3133dacf2c0110d45191ed470ecb04d0f5f6fcb0f"}, + {file = "black-24.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:30d2c30dc5139211dda799758559d1b049f7f14c580c409d6ad925b74a4208a8"}, + {file = "black-24.10.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1cbacacb19e922a1d75ef2b6ccaefcd6e93a2c05ede32f06a21386a04cedb981"}, + {file = "black-24.10.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1f93102e0c5bb3907451063e08b9876dbeac810e7da5a8bfb7aeb5a9ef89066b"}, + {file = "black-24.10.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ddacb691cdcdf77b96f549cf9591701d8db36b2f19519373d60d31746068dbf2"}, + {file = "black-24.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:680359d932801c76d2e9c9068d05c6b107f2584b2a5b88831c83962eb9984c1b"}, + {file = "black-24.10.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:17374989640fbca88b6a448129cd1745c5eb8d9547b464f281b251dd00155ccd"}, + {file = "black-24.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:63f626344343083322233f175aaf372d326de8436f5928c042639a4afbbf1d3f"}, + {file = "black-24.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfa1d0cb6200857f1923b602f978386a3a2758a65b52e0950299ea014be6800"}, + {file = "black-24.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:2cd9c95431d94adc56600710f8813ee27eea544dd118d45896bb734e9d7a0dc7"}, + {file = "black-24.10.0-py3-none-any.whl", hash = "sha256:3bb2b7a1f7b685f85b11fed1ef10f8a9148bceb49853e47a294a3dd963c1dd7d"}, + {file = "black-24.10.0.tar.gz", hash = "sha256:846ea64c97afe3bc677b761787993be4991810ecc7a4a937816dd6bddedc4875"}, +] + +[package.dependencies] +click = ">=8.0.0" +mypy-extensions = ">=0.4.3" +packaging = ">=22.0" +pathspec = ">=0.9.0" +platformdirs = ">=2" + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.10)"] +jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] +uvloop = ["uvloop (>=0.15.2)"] + +[[package]] +name = "boto3" +version = "1.35.50" +description = "The AWS SDK for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "boto3-1.35.50-py3-none-any.whl", hash = "sha256:14724b905fd13f26d9d8f7cdcea0fa65a9acad79f60f41f7662667f4e233d97c"}, + {file = "boto3-1.35.50.tar.gz", hash = "sha256:4f15d1ccb481d66f6925b8c91c970ce41b956b6ecf7c479f23e2159531b37eec"}, +] + +[package.dependencies] +botocore = ">=1.35.50,<1.36.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.10.0,<0.11.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.35.50" +description = "Low-level, data-driven core of boto 3." +optional = false +python-versions = ">=3.8" +files = [ + {file = "botocore-1.35.50-py3-none-any.whl", hash = "sha256:965d3b99179ac04aa98e4c4baf4a970ebce77a5e02bb2a0a21cb6304e2bc0955"}, + {file = "botocore-1.35.50.tar.gz", hash = "sha256:136ecef8d5a1088f1ba485c0bbfca40abd42b9f9fe9e11d8cde4e53b4c05b188"}, +] + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""} + +[package.extras] +crt = ["awscrt (==0.22.0)"] + +[[package]] +name = "certifi" +version = "2024.8.30" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, + {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.0" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:4f9fc98dad6c2eaa32fc3af1417d95b5e3d08aff968df0cd320066def971f9a6"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0de7b687289d3c1b3e8660d0741874abe7888100efe14bd0f9fd7141bcbda92b"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ed2e36c3e9b4f21dd9422f6893dec0abf2cca553af509b10cd630f878d3eb99"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40d3ff7fc90b98c637bda91c89d51264a3dcf210cade3a2c6f838c7268d7a4ca"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1110e22af8ca26b90bd6364fe4c763329b0ebf1ee213ba32b68c73de5752323d"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:86f4e8cca779080f66ff4f191a685ced73d2f72d50216f7112185dc02b90b9b7"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f683ddc7eedd742e2889d2bfb96d69573fde1d92fcb811979cdb7165bb9c7d3"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27623ba66c183eca01bf9ff833875b459cad267aeeb044477fedac35e19ba907"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f606a1881d2663630ea5b8ce2efe2111740df4b687bd78b34a8131baa007f79b"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:0b309d1747110feb25d7ed6b01afdec269c647d382c857ef4663bbe6ad95a912"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:136815f06a3ae311fae551c3df1f998a1ebd01ddd424aa5603a4336997629e95"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:14215b71a762336254351b00ec720a8e85cada43b987da5a042e4ce3e82bd68e"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79983512b108e4a164b9c8d34de3992f76d48cadc9554c9e60b43f308988aabe"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-win32.whl", hash = "sha256:c94057af19bc953643a33581844649a7fdab902624d2eb739738a30e2b3e60fc"}, + {file = "charset_normalizer-3.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:55f56e2ebd4e3bc50442fbc0888c9d8c94e4e06a933804e2af3e89e2f9c1c749"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce031db0408e487fd2775d745ce30a7cd2923667cf3b69d48d219f1d8f5ddeb6"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82357d85de703176b5587dbe6ade8ff67f9f69a41c0733cf2425378b49954de5"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8ce7fd6767a1cc5a92a639b391891bf1c268b03ec7e021c7d6d902285259685c"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1a2f519ae173b5b6a2c9d5fa3116ce16e48b3462c8b96dfdded11055e3d6365"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-win32.whl", hash = "sha256:9ae4ef0b3f6b41bad6366fb0ea4fc1d7ed051528e113a60fa2a65a9abb5b1d99"}, + {file = "charset_normalizer-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:cee4373f4d3ad28f1ab6290684d8e2ebdb9e7a1b74fdc39e4c211995f77bec27"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0713f3adb9d03d49d365b70b84775d0a0d18e4ab08d12bc46baa6132ba78aaf6"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:de7376c29d95d6719048c194a9cf1a1b0393fbe8488a22008610b0361d834ecf"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4a51b48f42d9358460b78725283f04bddaf44a9358197b889657deba38f329db"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b295729485b06c1a0683af02a9e42d2caa9db04a373dc38a6a58cdd1e8abddf1"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ee803480535c44e7f5ad00788526da7d85525cfefaf8acf8ab9a310000be4b03"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d59d125ffbd6d552765510e3f31ed75ebac2c7470c7274195b9161a32350284"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cda06946eac330cbe6598f77bb54e690b4ca93f593dee1568ad22b04f347c15"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07afec21bbbbf8a5cc3651aa96b980afe2526e7f048fdfb7f1014d84acc8b6d8"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6b40e8d38afe634559e398cc32b1472f376a4099c75fe6299ae607e404c033b2"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b8dcd239c743aa2f9c22ce674a145e0a25cb1566c495928440a181ca1ccf6719"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:84450ba661fb96e9fd67629b93d2941c871ca86fc38d835d19d4225ff946a631"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:44aeb140295a2f0659e113b31cfe92c9061622cadbc9e2a2f7b8ef6b1e29ef4b"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1db4e7fefefd0f548d73e2e2e041f9df5c59e178b4c72fbac4cc6f535cfb1565"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-win32.whl", hash = "sha256:5726cf76c982532c1863fb64d8c6dd0e4c90b6ece9feb06c9f202417a31f7dd7"}, + {file = "charset_normalizer-3.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:b197e7094f232959f8f20541ead1d9862ac5ebea1d58e9849c1bf979255dfac9"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:dd4eda173a9fcccb5f2e2bd2a9f423d180194b1bf17cf59e3269899235b2a114"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e9e3c4c9e1ed40ea53acf11e2a386383c3304212c965773704e4603d589343ed"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92a7e36b000bf022ef3dbb9c46bfe2d52c047d5e3f3343f43204263c5addc250"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54b6a92d009cbe2fb11054ba694bc9e284dad30a26757b1e372a1fdddaf21920"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ffd9493de4c922f2a38c2bf62b831dcec90ac673ed1ca182fe11b4d8e9f2a64"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:35c404d74c2926d0287fbd63ed5d27eb911eb9e4a3bb2c6d294f3cfd4a9e0c23"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4796efc4faf6b53a18e3d46343535caed491776a22af773f366534056c4e1fbc"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e7fdd52961feb4c96507aa649550ec2a0d527c086d284749b2f582f2d40a2e0d"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:92db3c28b5b2a273346bebb24857fda45601aef6ae1c011c0a997106581e8a88"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ab973df98fc99ab39080bfb0eb3a925181454d7c3ac8a1e695fddfae696d9e90"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b67fdab07fdd3c10bb21edab3cbfe8cf5696f453afce75d815d9d7223fbe88b"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:aa41e526a5d4a9dfcfbab0716c7e8a1b215abd3f3df5a45cf18a12721d31cb5d"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffc519621dce0c767e96b9c53f09c5d215578e10b02c285809f76509a3931482"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-win32.whl", hash = "sha256:f19c1585933c82098c2a520f8ec1227f20e339e33aca8fa6f956f6691b784e67"}, + {file = "charset_normalizer-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:707b82d19e65c9bd28b81dde95249b07bf9f5b90ebe1ef17d9b57473f8a64b7b"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:dbe03226baf438ac4fda9e2d0715022fd579cb641c4cf639fa40d53b2fe6f3e2"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd9a8bd8900e65504a305bf8ae6fa9fbc66de94178c420791d0293702fce2df7"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8831399554b92b72af5932cdbbd4ddc55c55f631bb13ff8fe4e6536a06c5c51"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a14969b8691f7998e74663b77b4c36c0337cb1df552da83d5c9004a93afdb574"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dcaf7c1524c0542ee2fc82cc8ec337f7a9f7edee2532421ab200d2b920fc97cf"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:425c5f215d0eecee9a56cdb703203dda90423247421bf0d67125add85d0c4455"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:d5b054862739d276e09928de37c79ddeec42a6e1bfc55863be96a36ba22926f6"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:f3e73a4255342d4eb26ef6df01e3962e73aa29baa3124a8e824c5d3364a65748"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:2f6c34da58ea9c1a9515621f4d9ac379871a8f21168ba1b5e09d74250de5ad62"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_s390x.whl", hash = "sha256:f09cb5a7bbe1ecae6e87901a2eb23e0256bb524a79ccc53eb0b7629fbe7677c4"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:0099d79bdfcf5c1f0c2c72f91516702ebf8b0b8ddd8905f97a8aecf49712c621"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-win32.whl", hash = "sha256:9c98230f5042f4945f957d006edccc2af1e03ed5e37ce7c373f00a5a4daa6149"}, + {file = "charset_normalizer-3.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:62f60aebecfc7f4b82e3f639a7d1433a20ec32824db2199a11ad4f5e146ef5ee"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:af73657b7a68211996527dbfeffbb0864e043d270580c5aef06dc4b659a4b578"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cab5d0b79d987c67f3b9e9c53f54a61360422a5a0bc075f43cab5621d530c3b6"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9289fd5dddcf57bab41d044f1756550f9e7cf0c8e373b8cdf0ce8773dc4bd417"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b493a043635eb376e50eedf7818f2f322eabbaa974e948bd8bdd29eb7ef2a51"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fa2566ca27d67c86569e8c85297aaf413ffab85a8960500f12ea34ff98e4c41"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8e538f46104c815be19c975572d74afb53f29650ea2025bbfaef359d2de2f7f"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fd30dc99682dc2c603c2b315bded2799019cea829f8bf57dc6b61efde6611c8"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2006769bd1640bdf4d5641c69a3d63b71b81445473cac5ded39740a226fa88ab"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:dc15e99b2d8a656f8e666854404f1ba54765871104e50c8e9813af8a7db07f12"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:ab2e5bef076f5a235c3774b4f4028a680432cded7cad37bba0fd90d64b187d19"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:4ec9dd88a5b71abfc74e9df5ebe7921c35cbb3b641181a531ca65cdb5e8e4dea"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:43193c5cda5d612f247172016c4bb71251c784d7a4d9314677186a838ad34858"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:aa693779a8b50cd97570e5a0f343538a8dbd3e496fa5dcb87e29406ad0299654"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-win32.whl", hash = "sha256:7706f5850360ac01d80c89bcef1640683cc12ed87f42579dab6c5d3ed6888613"}, + {file = "charset_normalizer-3.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:c3e446d253bd88f6377260d07c895816ebf33ffffd56c1c792b13bff9c3e1ade"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:980b4f289d1d90ca5efcf07958d3eb38ed9c0b7676bf2831a54d4f66f9c27dfa"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f28f891ccd15c514a0981f3b9db9aa23d62fe1a99997512b0491d2ed323d229a"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8aacce6e2e1edcb6ac625fb0f8c3a9570ccc7bfba1f63419b3769ccf6a00ed0"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd7af3717683bea4c87acd8c0d3d5b44d56120b26fd3f8a692bdd2d5260c620a"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ff2ed8194587faf56555927b3aa10e6fb69d931e33953943bc4f837dfee2242"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e91f541a85298cf35433bf66f3fab2a4a2cff05c127eeca4af174f6d497f0d4b"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:309a7de0a0ff3040acaebb35ec45d18db4b28232f21998851cfa709eeff49d62"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:285e96d9d53422efc0d7a17c60e59f37fbf3dfa942073f666db4ac71e8d726d0"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5d447056e2ca60382d460a604b6302d8db69476fd2015c81e7c35417cfabe4cd"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:20587d20f557fe189b7947d8e7ec5afa110ccf72a3128d61a2a387c3313f46be"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:130272c698667a982a5d0e626851ceff662565379baf0ff2cc58067b81d4f11d"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ab22fbd9765e6954bc0bcff24c25ff71dcbfdb185fcdaca49e81bac68fe724d3"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7782afc9b6b42200f7362858f9e73b1f8316afb276d316336c0ec3bd73312742"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-win32.whl", hash = "sha256:2de62e8801ddfff069cd5c504ce3bc9672b23266597d4e4f50eda28846c322f2"}, + {file = "charset_normalizer-3.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:95c3c157765b031331dd4db3c775e58deaee050a3042fcad72cbc4189d7c8dca"}, + {file = "charset_normalizer-3.4.0-py3-none-any.whl", hash = "sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079"}, + {file = "charset_normalizer-3.4.0.tar.gz", hash = "sha256:223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e"}, +] + +[[package]] +name = "click" +version = "8.1.7" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "distro" +version = "1.9.0" +description = "Distro - an OS platform information API" +optional = false +python-versions = ">=3.6" +files = [ + {file = "distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2"}, + {file = "distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed"}, +] + +[[package]] +name = "et-xmlfile" +version = "2.0.0" +description = "An implementation of lxml.xmlfile for the standard library" +optional = false +python-versions = ">=3.8" +files = [ + {file = "et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa"}, + {file = "et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54"}, +] + +[[package]] +name = "filelock" +version = "3.16.1" +description = "A platform independent file lock." +optional = false +python-versions = ">=3.8" +files = [ + {file = "filelock-3.16.1-py3-none-any.whl", hash = "sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0"}, + {file = "filelock-3.16.1.tar.gz", hash = "sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435"}, +] + +[package.extras] +docs = ["furo (>=2024.8.6)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4.1)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.6.1)", "diff-cover (>=9.2)", "pytest (>=8.3.3)", "pytest-asyncio (>=0.24)", "pytest-cov (>=5)", "pytest-mock (>=3.14)", "pytest-timeout (>=2.3.1)", "virtualenv (>=20.26.4)"] +typing = ["typing-extensions (>=4.12.2)"] + +[[package]] +name = "fsspec" +version = "2024.10.0" +description = "File-system specification" +optional = false +python-versions = ">=3.8" +files = [ + {file = "fsspec-2024.10.0-py3-none-any.whl", hash = "sha256:03b9a6785766a4de40368b88906366755e2819e758b83705c88cd7cb5fe81871"}, + {file = "fsspec-2024.10.0.tar.gz", hash = "sha256:eda2d8a4116d4f2429db8550f2457da57279247dd930bb12f821b58391359493"}, +] + +[package.extras] +abfs = ["adlfs"] +adl = ["adlfs"] +arrow = ["pyarrow (>=1)"] +dask = ["dask", "distributed"] +dev = ["pre-commit", "ruff"] +doc = ["numpydoc", "sphinx", "sphinx-design", "sphinx-rtd-theme", "yarl"] +dropbox = ["dropbox", "dropboxdrivefs", "requests"] +full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] +fuse = ["fusepy"] +gcs = ["gcsfs"] +git = ["pygit2"] +github = ["requests"] +gs = ["gcsfs"] +gui = ["panel"] +hdfs = ["pyarrow (>=1)"] +http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)"] +libarchive = ["libarchive-c"] +oci = ["ocifs"] +s3 = ["s3fs"] +sftp = ["paramiko"] +smb = ["smbprotocol"] +ssh = ["paramiko"] +test = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "numpy", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "requests"] +test-downstream = ["aiobotocore (>=2.5.4,<3.0.0)", "dask-expr", "dask[dataframe,test]", "moto[server] (>4,<5)", "pytest-timeout", "xarray"] +test-full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "cloudpickle", "dask", "distributed", "dropbox", "dropboxdrivefs", "fastparquet", "fusepy", "gcsfs", "jinja2", "kerchunk", "libarchive-c", "lz4", "notebook", "numpy", "ocifs", "pandas", "panel", "paramiko", "pyarrow", "pyarrow (>=1)", "pyftpdlib", "pygit2", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "python-snappy", "requests", "smbprotocol", "tqdm", "urllib3", "zarr", "zstandard"] +tqdm = ["tqdm"] + +[[package]] +name = "h11" +version = "0.14.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +optional = false +python-versions = ">=3.7" +files = [ + {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, + {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, +] + +[[package]] +name = "httpcore" +version = "1.0.6" +description = "A minimal low-level HTTP client." +optional = false +python-versions = ">=3.8" +files = [ + {file = "httpcore-1.0.6-py3-none-any.whl", hash = "sha256:27b59625743b85577a8c0e10e55b50b5368a4f2cfe8cc7bcfa9cf00829c2682f"}, + {file = "httpcore-1.0.6.tar.gz", hash = "sha256:73f6dbd6eb8c21bbf7ef8efad555481853f5f6acdeaff1edb0694289269ee17f"}, +] + +[package.dependencies] +certifi = "*" +h11 = ">=0.13,<0.15" + +[package.extras] +asyncio = ["anyio (>=4.0,<5.0)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +trio = ["trio (>=0.22.0,<1.0)"] + +[[package]] +name = "httpx" +version = "0.27.2" +description = "The next generation HTTP client." +optional = false +python-versions = ">=3.8" +files = [ + {file = "httpx-0.27.2-py3-none-any.whl", hash = "sha256:7bb2708e112d8fdd7829cd4243970f0c223274051cb35ee80c03301ee29a3df0"}, + {file = "httpx-0.27.2.tar.gz", hash = "sha256:f7c2be1d2f3c3c3160d441802406b206c2b76f5947b11115e6df10c6c65e66c2"}, +] + +[package.dependencies] +anyio = "*" +certifi = "*" +httpcore = "==1.*" +idna = "*" +sniffio = "*" + +[package.extras] +brotli = ["brotli", "brotlicffi"] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "huggingface-hub" +version = "0.26.2" +description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "huggingface_hub-0.26.2-py3-none-any.whl", hash = "sha256:98c2a5a8e786c7b2cb6fdeb2740893cba4d53e312572ed3d8afafda65b128c46"}, + {file = "huggingface_hub-0.26.2.tar.gz", hash = "sha256:b100d853465d965733964d123939ba287da60a547087783ddff8a323f340332b"}, +] + +[package.dependencies] +filelock = "*" +fsspec = ">=2023.5.0" +packaging = ">=20.9" +pyyaml = ">=5.1" +requests = "*" +tqdm = ">=4.42.1" +typing-extensions = ">=3.7.4.3" + +[package.extras] +all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio (>=4.0.0)", "jedi", "libcst (==1.4.0)", "mypy (==1.5.1)", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.5.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] +cli = ["InquirerPy (==0.3.4)"] +dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio (>=4.0.0)", "jedi", "libcst (==1.4.0)", "mypy (==1.5.1)", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.5.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] +fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] +hf-transfer = ["hf-transfer (>=0.1.4)"] +inference = ["aiohttp"] +quality = ["libcst (==1.4.0)", "mypy (==1.5.1)", "ruff (>=0.5.0)"] +tensorflow = ["graphviz", "pydot", "tensorflow"] +tensorflow-testing = ["keras (<3.0)", "tensorflow"] +testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio (>=4.0.0)", "jedi", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] +torch = ["safetensors[torch]", "torch"] +typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] + +[[package]] +name = "idna" +version = "3.10" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.6" +files = [ + {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, + {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, +] + +[package.extras] +all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "jiter" +version = "0.6.1" +description = "Fast iterable JSON parser." +optional = false +python-versions = ">=3.8" +files = [ + {file = "jiter-0.6.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:d08510593cb57296851080018006dfc394070178d238b767b1879dc1013b106c"}, + {file = "jiter-0.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:adef59d5e2394ebbad13b7ed5e0306cceb1df92e2de688824232a91588e77aa7"}, + {file = "jiter-0.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b3e02f7a27f2bcc15b7d455c9df05df8ffffcc596a2a541eeda9a3110326e7a3"}, + {file = "jiter-0.6.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed69a7971d67b08f152c17c638f0e8c2aa207e9dd3a5fcd3cba294d39b5a8d2d"}, + {file = "jiter-0.6.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b2019d966e98f7c6df24b3b8363998575f47d26471bfb14aade37630fae836a1"}, + {file = "jiter-0.6.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:36c0b51a285b68311e207a76c385650322734c8717d16c2eb8af75c9d69506e7"}, + {file = "jiter-0.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:220e0963b4fb507c525c8f58cde3da6b1be0bfddb7ffd6798fb8f2531226cdb1"}, + {file = "jiter-0.6.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:aa25c7a9bf7875a141182b9c95aed487add635da01942ef7ca726e42a0c09058"}, + {file = "jiter-0.6.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e90552109ca8ccd07f47ca99c8a1509ced93920d271bb81780a973279974c5ab"}, + {file = "jiter-0.6.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:67723a011964971864e0b484b0ecfee6a14de1533cff7ffd71189e92103b38a8"}, + {file = "jiter-0.6.1-cp310-none-win32.whl", hash = "sha256:33af2b7d2bf310fdfec2da0177eab2fedab8679d1538d5b86a633ebfbbac4edd"}, + {file = "jiter-0.6.1-cp310-none-win_amd64.whl", hash = "sha256:7cea41c4c673353799906d940eee8f2d8fd1d9561d734aa921ae0f75cb9732f4"}, + {file = "jiter-0.6.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:b03c24e7da7e75b170c7b2b172d9c5e463aa4b5c95696a368d52c295b3f6847f"}, + {file = "jiter-0.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:47fee1be677b25d0ef79d687e238dc6ac91a8e553e1a68d0839f38c69e0ee491"}, + {file = "jiter-0.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25f0d2f6e01a8a0fb0eab6d0e469058dab2be46ff3139ed2d1543475b5a1d8e7"}, + {file = "jiter-0.6.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0b809e39e342c346df454b29bfcc7bca3d957f5d7b60e33dae42b0e5ec13e027"}, + {file = "jiter-0.6.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e9ac7c2f092f231f5620bef23ce2e530bd218fc046098747cc390b21b8738a7a"}, + {file = "jiter-0.6.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e51a2d80d5fe0ffb10ed2c82b6004458be4a3f2b9c7d09ed85baa2fbf033f54b"}, + {file = "jiter-0.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3343d4706a2b7140e8bd49b6c8b0a82abf9194b3f0f5925a78fc69359f8fc33c"}, + {file = "jiter-0.6.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:82521000d18c71e41c96960cb36e915a357bc83d63a8bed63154b89d95d05ad1"}, + {file = "jiter-0.6.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:3c843e7c1633470708a3987e8ce617ee2979ee18542d6eb25ae92861af3f1d62"}, + {file = "jiter-0.6.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a2e861658c3fe849efc39b06ebb98d042e4a4c51a8d7d1c3ddc3b1ea091d0784"}, + {file = "jiter-0.6.1-cp311-none-win32.whl", hash = "sha256:7d72fc86474862c9c6d1f87b921b70c362f2b7e8b2e3c798bb7d58e419a6bc0f"}, + {file = "jiter-0.6.1-cp311-none-win_amd64.whl", hash = "sha256:3e36a320634f33a07794bb15b8da995dccb94f944d298c8cfe2bd99b1b8a574a"}, + {file = "jiter-0.6.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:1fad93654d5a7dcce0809aff66e883c98e2618b86656aeb2129db2cd6f26f867"}, + {file = "jiter-0.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4e6e340e8cd92edab7f6a3a904dbbc8137e7f4b347c49a27da9814015cc0420c"}, + {file = "jiter-0.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:691352e5653af84ed71763c3c427cff05e4d658c508172e01e9c956dfe004aba"}, + {file = "jiter-0.6.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:defee3949313c1f5b55e18be45089970cdb936eb2a0063f5020c4185db1b63c9"}, + {file = "jiter-0.6.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:26d2bdd5da097e624081c6b5d416d3ee73e5b13f1703bcdadbb1881f0caa1933"}, + {file = "jiter-0.6.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18aa9d1626b61c0734b973ed7088f8a3d690d0b7f5384a5270cd04f4d9f26c86"}, + {file = "jiter-0.6.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a3567c8228afa5ddcce950631c6b17397ed178003dc9ee7e567c4c4dcae9fa0"}, + {file = "jiter-0.6.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e5c0507131c922defe3f04c527d6838932fcdfd69facebafd7d3574fa3395314"}, + {file = "jiter-0.6.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:540fcb224d7dc1bcf82f90f2ffb652df96f2851c031adca3c8741cb91877143b"}, + {file = "jiter-0.6.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e7b75436d4fa2032b2530ad989e4cb0ca74c655975e3ff49f91a1a3d7f4e1df2"}, + {file = "jiter-0.6.1-cp312-none-win32.whl", hash = "sha256:883d2ced7c21bf06874fdeecab15014c1c6d82216765ca6deef08e335fa719e0"}, + {file = "jiter-0.6.1-cp312-none-win_amd64.whl", hash = "sha256:91e63273563401aadc6c52cca64a7921c50b29372441adc104127b910e98a5b6"}, + {file = "jiter-0.6.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:852508a54fe3228432e56019da8b69208ea622a3069458252f725d634e955b31"}, + {file = "jiter-0.6.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f491cc69ff44e5a1e8bc6bf2b94c1f98d179e1aaf4a554493c171a5b2316b701"}, + {file = "jiter-0.6.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc56c8f0b2a28ad4d8047f3ae62d25d0e9ae01b99940ec0283263a04724de1f3"}, + {file = "jiter-0.6.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:51b58f7a0d9e084a43b28b23da2b09fc5e8df6aa2b6a27de43f991293cab85fd"}, + {file = "jiter-0.6.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5f79ce15099154c90ef900d69c6b4c686b64dfe23b0114e0971f2fecd306ec6c"}, + {file = "jiter-0.6.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:03a025b52009f47e53ea619175d17e4ded7c035c6fbd44935cb3ada11e1fd592"}, + {file = "jiter-0.6.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c74a8d93718137c021d9295248a87c2f9fdc0dcafead12d2930bc459ad40f885"}, + {file = "jiter-0.6.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:40b03b75f903975f68199fc4ec73d546150919cb7e534f3b51e727c4d6ccca5a"}, + {file = "jiter-0.6.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:825651a3f04cf92a661d22cad61fc913400e33aa89b3e3ad9a6aa9dc8a1f5a71"}, + {file = "jiter-0.6.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:928bf25eb69ddb292ab8177fe69d3fbf76c7feab5fce1c09265a7dccf25d3991"}, + {file = "jiter-0.6.1-cp313-none-win32.whl", hash = "sha256:352cd24121e80d3d053fab1cc9806258cad27c53cad99b7a3cac57cf934b12e4"}, + {file = "jiter-0.6.1-cp313-none-win_amd64.whl", hash = "sha256:be7503dd6f4bf02c2a9bacb5cc9335bc59132e7eee9d3e931b13d76fd80d7fda"}, + {file = "jiter-0.6.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:31d8e00e1fb4c277df8ab6f31a671f509ebc791a80e5c61fdc6bc8696aaa297c"}, + {file = "jiter-0.6.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:77c296d65003cd7ee5d7b0965f6acbe6cffaf9d1fa420ea751f60ef24e85fed5"}, + {file = "jiter-0.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aeeb0c0325ef96c12a48ea7e23e2e86fe4838e6e0a995f464cf4c79fa791ceeb"}, + {file = "jiter-0.6.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a31c6fcbe7d6c25d6f1cc6bb1cba576251d32795d09c09961174fe461a1fb5bd"}, + {file = "jiter-0.6.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:59e2b37f3b9401fc9e619f4d4badcab2e8643a721838bcf695c2318a0475ae42"}, + {file = "jiter-0.6.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bae5ae4853cb9644144e9d0755854ce5108d470d31541d83f70ca7ecdc2d1637"}, + {file = "jiter-0.6.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9df588e9c830b72d8db1dd7d0175af6706b0904f682ea9b1ca8b46028e54d6e9"}, + {file = "jiter-0.6.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:15f8395e835cf561c85c1adee72d899abf2733d9df72e9798e6d667c9b5c1f30"}, + {file = "jiter-0.6.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5a99d4e0b5fc3b05ea732d67eb2092fe894e95a90e6e413f2ea91387e228a307"}, + {file = "jiter-0.6.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:a311df1fa6be0ccd64c12abcd85458383d96e542531bafbfc0a16ff6feda588f"}, + {file = "jiter-0.6.1-cp38-none-win32.whl", hash = "sha256:81116a6c272a11347b199f0e16b6bd63f4c9d9b52bc108991397dd80d3c78aba"}, + {file = "jiter-0.6.1-cp38-none-win_amd64.whl", hash = "sha256:13f9084e3e871a7c0b6e710db54444088b1dd9fbefa54d449b630d5e73bb95d0"}, + {file = "jiter-0.6.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:f1c53615fcfec3b11527c08d19cff6bc870da567ce4e57676c059a3102d3a082"}, + {file = "jiter-0.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f791b6a4da23238c17a81f44f5b55d08a420c5692c1fda84e301a4b036744eb1"}, + {file = "jiter-0.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c97e90fec2da1d5f68ef121444c2c4fa72eabf3240829ad95cf6bbeca42a301"}, + {file = "jiter-0.6.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3cbc1a66b4e41511209e97a2866898733c0110b7245791ac604117b7fb3fedb7"}, + {file = "jiter-0.6.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e4e85f9e12cd8418ab10e1fcf0e335ae5bb3da26c4d13a0fd9e6a17a674783b6"}, + {file = "jiter-0.6.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:08be33db6dcc374c9cc19d3633af5e47961a7b10d4c61710bd39e48d52a35824"}, + {file = "jiter-0.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:677be9550004f5e010d673d3b2a2b815a8ea07a71484a57d3f85dde7f14cf132"}, + {file = "jiter-0.6.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e8bd065be46c2eecc328e419d6557bbc37844c88bb07b7a8d2d6c91c7c4dedc9"}, + {file = "jiter-0.6.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bd95375ce3609ec079a97c5d165afdd25693302c071ca60c7ae1cf826eb32022"}, + {file = "jiter-0.6.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:db459ed22d0208940d87f614e1f0ea5a946d29a3cfef71f7e1aab59b6c6b2afb"}, + {file = "jiter-0.6.1-cp39-none-win32.whl", hash = "sha256:d71c962f0971347bd552940ab96aa42ceefcd51b88c4ced8a27398182efa8d80"}, + {file = "jiter-0.6.1-cp39-none-win_amd64.whl", hash = "sha256:d465db62d2d10b489b7e7a33027c4ae3a64374425d757e963f86df5b5f2e7fc5"}, + {file = "jiter-0.6.1.tar.gz", hash = "sha256:e19cd21221fc139fb032e4112986656cb2739e9fe6d84c13956ab30ccc7d4449"}, +] + +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] + +[[package]] +name = "mypy" +version = "1.13.0" +description = "Optional static typing for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "mypy-1.13.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6607e0f1dd1fb7f0aca14d936d13fd19eba5e17e1cd2a14f808fa5f8f6d8f60a"}, + {file = "mypy-1.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8a21be69bd26fa81b1f80a61ee7ab05b076c674d9b18fb56239d72e21d9f4c80"}, + {file = "mypy-1.13.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b2353a44d2179846a096e25691d54d59904559f4232519d420d64da6828a3a7"}, + {file = "mypy-1.13.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0730d1c6a2739d4511dc4253f8274cdd140c55c32dfb0a4cf8b7a43f40abfa6f"}, + {file = "mypy-1.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:c5fc54dbb712ff5e5a0fca797e6e0aa25726c7e72c6a5850cfd2adbc1eb0a372"}, + {file = "mypy-1.13.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:581665e6f3a8a9078f28d5502f4c334c0c8d802ef55ea0e7276a6e409bc0d82d"}, + {file = "mypy-1.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3ddb5b9bf82e05cc9a627e84707b528e5c7caaa1c55c69e175abb15a761cec2d"}, + {file = "mypy-1.13.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:20c7ee0bc0d5a9595c46f38beb04201f2620065a93755704e141fcac9f59db2b"}, + {file = "mypy-1.13.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3790ded76f0b34bc9c8ba4def8f919dd6a46db0f5a6610fb994fe8efdd447f73"}, + {file = "mypy-1.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:51f869f4b6b538229c1d1bcc1dd7d119817206e2bc54e8e374b3dfa202defcca"}, + {file = "mypy-1.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5c7051a3461ae84dfb5dd15eff5094640c61c5f22257c8b766794e6dd85e72d5"}, + {file = "mypy-1.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:39bb21c69a5d6342f4ce526e4584bc5c197fd20a60d14a8624d8743fffb9472e"}, + {file = "mypy-1.13.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:164f28cb9d6367439031f4c81e84d3ccaa1e19232d9d05d37cb0bd880d3f93c2"}, + {file = "mypy-1.13.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a4c1bfcdbce96ff5d96fc9b08e3831acb30dc44ab02671eca5953eadad07d6d0"}, + {file = "mypy-1.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0affb3a79a256b4183ba09811e3577c5163ed06685e4d4b46429a271ba174d2"}, + {file = "mypy-1.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a7b44178c9760ce1a43f544e595d35ed61ac2c3de306599fa59b38a6048e1aa7"}, + {file = "mypy-1.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5d5092efb8516d08440e36626f0153b5006d4088c1d663d88bf79625af3d1d62"}, + {file = "mypy-1.13.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:de2904956dac40ced10931ac967ae63c5089bd498542194b436eb097a9f77bc8"}, + {file = "mypy-1.13.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:7bfd8836970d33c2105562650656b6846149374dc8ed77d98424b40b09340ba7"}, + {file = "mypy-1.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:9f73dba9ec77acb86457a8fc04b5239822df0c14a082564737833d2963677dbc"}, + {file = "mypy-1.13.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:100fac22ce82925f676a734af0db922ecfea991e1d7ec0ceb1e115ebe501301a"}, + {file = "mypy-1.13.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7bcb0bb7f42a978bb323a7c88f1081d1b5dee77ca86f4100735a6f541299d8fb"}, + {file = "mypy-1.13.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bde31fc887c213e223bbfc34328070996061b0833b0a4cfec53745ed61f3519b"}, + {file = "mypy-1.13.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:07de989f89786f62b937851295ed62e51774722e5444a27cecca993fc3f9cd74"}, + {file = "mypy-1.13.0-cp38-cp38-win_amd64.whl", hash = "sha256:4bde84334fbe19bad704b3f5b78c4abd35ff1026f8ba72b29de70dda0916beb6"}, + {file = "mypy-1.13.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0246bcb1b5de7f08f2826451abd947bf656945209b140d16ed317f65a17dc7dc"}, + {file = "mypy-1.13.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7f5b7deae912cf8b77e990b9280f170381fdfbddf61b4ef80927edd813163732"}, + {file = "mypy-1.13.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7029881ec6ffb8bc233a4fa364736789582c738217b133f1b55967115288a2bc"}, + {file = "mypy-1.13.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3e38b980e5681f28f033f3be86b099a247b13c491f14bb8b1e1e134d23bb599d"}, + {file = "mypy-1.13.0-cp39-cp39-win_amd64.whl", hash = "sha256:a6789be98a2017c912ae6ccb77ea553bbaf13d27605d2ca20a76dfbced631b24"}, + {file = "mypy-1.13.0-py3-none-any.whl", hash = "sha256:9c250883f9fd81d212e0952c92dbfcc96fc237f4b7c92f56ac81fd48460b3e5a"}, + {file = "mypy-1.13.0.tar.gz", hash = "sha256:0291a61b6fbf3e6673e3405cfcc0e7650bebc7939659fdca2702958038bd835e"}, +] + +[package.dependencies] +mypy-extensions = ">=1.0.0" +typing-extensions = ">=4.6.0" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +faster-cache = ["orjson"] +install-types = ["pip"] +mypyc = ["setuptools (>=50)"] +reports = ["lxml"] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + +[[package]] +name = "numpy" +version = "2.1.2" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = ">=3.10" +files = [ + {file = "numpy-2.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:30d53720b726ec36a7f88dc873f0eec8447fbc93d93a8f079dfac2629598d6ee"}, + {file = "numpy-2.1.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e8d3ca0a72dd8846eb6f7dfe8f19088060fcb76931ed592d29128e0219652884"}, + {file = "numpy-2.1.2-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:fc44e3c68ff00fd991b59092a54350e6e4911152682b4782f68070985aa9e648"}, + {file = "numpy-2.1.2-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:7c1c60328bd964b53f8b835df69ae8198659e2b9302ff9ebb7de4e5a5994db3d"}, + {file = "numpy-2.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6cdb606a7478f9ad91c6283e238544451e3a95f30fb5467fbf715964341a8a86"}, + {file = "numpy-2.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d666cb72687559689e9906197e3bec7b736764df6a2e58ee265e360663e9baf7"}, + {file = "numpy-2.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c6eef7a2dbd0abfb0d9eaf78b73017dbfd0b54051102ff4e6a7b2980d5ac1a03"}, + {file = "numpy-2.1.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:12edb90831ff481f7ef5f6bc6431a9d74dc0e5ff401559a71e5e4611d4f2d466"}, + {file = "numpy-2.1.2-cp310-cp310-win32.whl", hash = "sha256:a65acfdb9c6ebb8368490dbafe83c03c7e277b37e6857f0caeadbbc56e12f4fb"}, + {file = "numpy-2.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:860ec6e63e2c5c2ee5e9121808145c7bf86c96cca9ad396c0bd3e0f2798ccbe2"}, + {file = "numpy-2.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b42a1a511c81cc78cbc4539675713bbcf9d9c3913386243ceff0e9429ca892fe"}, + {file = "numpy-2.1.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:faa88bc527d0f097abdc2c663cddf37c05a1c2f113716601555249805cf573f1"}, + {file = "numpy-2.1.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:c82af4b2ddd2ee72d1fc0c6695048d457e00b3582ccde72d8a1c991b808bb20f"}, + {file = "numpy-2.1.2-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:13602b3174432a35b16c4cfb5de9a12d229727c3dd47a6ce35111f2ebdf66ff4"}, + {file = "numpy-2.1.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ebec5fd716c5a5b3d8dfcc439be82a8407b7b24b230d0ad28a81b61c2f4659a"}, + {file = "numpy-2.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2b49c3c0804e8ecb05d59af8386ec2f74877f7ca8fd9c1e00be2672e4d399b1"}, + {file = "numpy-2.1.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2cbba4b30bf31ddbe97f1c7205ef976909a93a66bb1583e983adbd155ba72ac2"}, + {file = "numpy-2.1.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8e00ea6fc82e8a804433d3e9cedaa1051a1422cb6e443011590c14d2dea59146"}, + {file = "numpy-2.1.2-cp311-cp311-win32.whl", hash = "sha256:5006b13a06e0b38d561fab5ccc37581f23c9511879be7693bd33c7cd15ca227c"}, + {file = "numpy-2.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:f1eb068ead09f4994dec71c24b2844f1e4e4e013b9629f812f292f04bd1510d9"}, + {file = "numpy-2.1.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d7bf0a4f9f15b32b5ba53147369e94296f5fffb783db5aacc1be15b4bf72f43b"}, + {file = "numpy-2.1.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b1d0fcae4f0949f215d4632be684a539859b295e2d0cb14f78ec231915d644db"}, + {file = "numpy-2.1.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:f751ed0a2f250541e19dfca9f1eafa31a392c71c832b6bb9e113b10d050cb0f1"}, + {file = "numpy-2.1.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:bd33f82e95ba7ad632bc57837ee99dba3d7e006536200c4e9124089e1bf42426"}, + {file = "numpy-2.1.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b8cde4f11f0a975d1fd59373b32e2f5a562ade7cde4f85b7137f3de8fbb29a0"}, + {file = "numpy-2.1.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d95f286b8244b3649b477ac066c6906fbb2905f8ac19b170e2175d3d799f4df"}, + {file = "numpy-2.1.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ab4754d432e3ac42d33a269c8567413bdb541689b02d93788af4131018cbf366"}, + {file = "numpy-2.1.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e585c8ae871fd38ac50598f4763d73ec5497b0de9a0ab4ef5b69f01c6a046142"}, + {file = "numpy-2.1.2-cp312-cp312-win32.whl", hash = "sha256:9c6c754df29ce6a89ed23afb25550d1c2d5fdb9901d9c67a16e0b16eaf7e2550"}, + {file = "numpy-2.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:456e3b11cb79ac9946c822a56346ec80275eaf2950314b249b512896c0d2505e"}, + {file = "numpy-2.1.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a84498e0d0a1174f2b3ed769b67b656aa5460c92c9554039e11f20a05650f00d"}, + {file = "numpy-2.1.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4d6ec0d4222e8ffdab1744da2560f07856421b367928026fb540e1945f2eeeaf"}, + {file = "numpy-2.1.2-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:259ec80d54999cc34cd1eb8ded513cb053c3bf4829152a2e00de2371bd406f5e"}, + {file = "numpy-2.1.2-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:675c741d4739af2dc20cd6c6a5c4b7355c728167845e3c6b0e824e4e5d36a6c3"}, + {file = "numpy-2.1.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:05b2d4e667895cc55e3ff2b56077e4c8a5604361fc21a042845ea3ad67465aa8"}, + {file = "numpy-2.1.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:43cca367bf94a14aca50b89e9bc2061683116cfe864e56740e083392f533ce7a"}, + {file = "numpy-2.1.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:76322dcdb16fccf2ac56f99048af32259dcc488d9b7e25b51e5eca5147a3fb98"}, + {file = "numpy-2.1.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:32e16a03138cabe0cb28e1007ee82264296ac0983714094380b408097a418cfe"}, + {file = "numpy-2.1.2-cp313-cp313-win32.whl", hash = "sha256:242b39d00e4944431a3cd2db2f5377e15b5785920421993770cddb89992c3f3a"}, + {file = "numpy-2.1.2-cp313-cp313-win_amd64.whl", hash = "sha256:f2ded8d9b6f68cc26f8425eda5d3877b47343e68ca23d0d0846f4d312ecaa445"}, + {file = "numpy-2.1.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2ffef621c14ebb0188a8633348504a35c13680d6da93ab5cb86f4e54b7e922b5"}, + {file = "numpy-2.1.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:ad369ed238b1959dfbade9018a740fb9392c5ac4f9b5173f420bd4f37ba1f7a0"}, + {file = "numpy-2.1.2-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:d82075752f40c0ddf57e6e02673a17f6cb0f8eb3f587f63ca1eaab5594da5b17"}, + {file = "numpy-2.1.2-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:1600068c262af1ca9580a527d43dc9d959b0b1d8e56f8a05d830eea39b7c8af6"}, + {file = "numpy-2.1.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a26ae94658d3ba3781d5e103ac07a876b3e9b29db53f68ed7df432fd033358a8"}, + {file = "numpy-2.1.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13311c2db4c5f7609b462bc0f43d3c465424d25c626d95040f073e30f7570e35"}, + {file = "numpy-2.1.2-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:2abbf905a0b568706391ec6fa15161fad0fb5d8b68d73c461b3c1bab6064dd62"}, + {file = "numpy-2.1.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ef444c57d664d35cac4e18c298c47d7b504c66b17c2ea91312e979fcfbdfb08a"}, + {file = "numpy-2.1.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:bdd407c40483463898b84490770199d5714dcc9dd9b792f6c6caccc523c00952"}, + {file = "numpy-2.1.2-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:da65fb46d4cbb75cb417cddf6ba5e7582eb7bb0b47db4b99c9fe5787ce5d91f5"}, + {file = "numpy-2.1.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c193d0b0238638e6fc5f10f1b074a6993cb13b0b431f64079a509d63d3aa8b7"}, + {file = "numpy-2.1.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:a7d80b2e904faa63068ead63107189164ca443b42dd1930299e0d1cb041cec2e"}, + {file = "numpy-2.1.2.tar.gz", hash = "sha256:13532a088217fa624c99b843eeb54640de23b3414b14aa66d023805eb731066c"}, +] + +[[package]] +name = "openpyxl" +version = "3.1.5" +description = "A Python library to read/write Excel 2010 xlsx/xlsm files" +optional = false +python-versions = ">=3.8" +files = [ + {file = "openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2"}, + {file = "openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050"}, +] + +[package.dependencies] +et-xmlfile = "*" + +[[package]] +name = "packaging" +version = "24.1" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, + {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, +] + +[[package]] +name = "pandas" +version = "2.2.3" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pandas-2.2.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1948ddde24197a0f7add2bdc4ca83bf2b1ef84a1bc8ccffd95eda17fd836ecb5"}, + {file = "pandas-2.2.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:381175499d3802cde0eabbaf6324cce0c4f5d52ca6f8c377c29ad442f50f6348"}, + {file = "pandas-2.2.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d9c45366def9a3dd85a6454c0e7908f2b3b8e9c138f5dc38fed7ce720d8453ed"}, + {file = "pandas-2.2.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86976a1c5b25ae3f8ccae3a5306e443569ee3c3faf444dfd0f41cda24667ad57"}, + {file = "pandas-2.2.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b8661b0238a69d7aafe156b7fa86c44b881387509653fdf857bebc5e4008ad42"}, + {file = "pandas-2.2.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:37e0aced3e8f539eccf2e099f65cdb9c8aa85109b0be6e93e2baff94264bdc6f"}, + {file = "pandas-2.2.3-cp310-cp310-win_amd64.whl", hash = "sha256:56534ce0746a58afaf7942ba4863e0ef81c9c50d3f0ae93e9497d6a41a057645"}, + {file = "pandas-2.2.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:66108071e1b935240e74525006034333f98bcdb87ea116de573a6a0dccb6c039"}, + {file = "pandas-2.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7c2875855b0ff77b2a64a0365e24455d9990730d6431b9e0ee18ad8acee13dbd"}, + {file = "pandas-2.2.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd8d0c3be0515c12fed0bdbae072551c8b54b7192c7b1fda0ba56059a0179698"}, + {file = "pandas-2.2.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c124333816c3a9b03fbeef3a9f230ba9a737e9e5bb4060aa2107a86cc0a497fc"}, + {file = "pandas-2.2.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:63cc132e40a2e084cf01adf0775b15ac515ba905d7dcca47e9a251819c575ef3"}, + {file = "pandas-2.2.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:29401dbfa9ad77319367d36940cd8a0b3a11aba16063e39632d98b0e931ddf32"}, + {file = "pandas-2.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:3fc6873a41186404dad67245896a6e440baacc92f5b716ccd1bc9ed2995ab2c5"}, + {file = "pandas-2.2.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b1d432e8d08679a40e2a6d8b2f9770a5c21793a6f9f47fdd52c5ce1948a5a8a9"}, + {file = "pandas-2.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a5a1595fe639f5988ba6a8e5bc9649af3baf26df3998a0abe56c02609392e0a4"}, + {file = "pandas-2.2.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5de54125a92bb4d1c051c0659e6fcb75256bf799a732a87184e5ea503965bce3"}, + {file = "pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319"}, + {file = "pandas-2.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dfcb5ee8d4d50c06a51c2fffa6cff6272098ad6540aed1a76d15fb9318194d8"}, + {file = "pandas-2.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:062309c1b9ea12a50e8ce661145c6aab431b1e99530d3cd60640e255778bd43a"}, + {file = "pandas-2.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:59ef3764d0fe818125a5097d2ae867ca3fa64df032331b7e0917cf5d7bf66b13"}, + {file = "pandas-2.2.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f00d1345d84d8c86a63e476bb4955e46458b304b9575dcf71102b5c705320015"}, + {file = "pandas-2.2.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3508d914817e153ad359d7e069d752cdd736a247c322d932eb89e6bc84217f28"}, + {file = "pandas-2.2.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22a9d949bfc9a502d320aa04e5d02feab689d61da4e7764b62c30b991c42c5f0"}, + {file = "pandas-2.2.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3a255b2c19987fbbe62a9dfd6cff7ff2aa9ccab3fc75218fd4b7530f01efa24"}, + {file = "pandas-2.2.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:800250ecdadb6d9c78eae4990da62743b857b470883fa27f652db8bdde7f6659"}, + {file = "pandas-2.2.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6374c452ff3ec675a8f46fd9ab25c4ad0ba590b71cf0656f8b6daa5202bca3fb"}, + {file = "pandas-2.2.3-cp313-cp313-win_amd64.whl", hash = "sha256:61c5ad4043f791b61dd4752191d9f07f0ae412515d59ba8f005832a532f8736d"}, + {file = "pandas-2.2.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3b71f27954685ee685317063bf13c7709a7ba74fc996b84fc6821c59b0f06468"}, + {file = "pandas-2.2.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:38cf8125c40dae9d5acc10fa66af8ea6fdf760b2714ee482ca691fc66e6fcb18"}, + {file = "pandas-2.2.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ba96630bc17c875161df3818780af30e43be9b166ce51c9a18c1feae342906c2"}, + {file = "pandas-2.2.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1db71525a1538b30142094edb9adc10be3f3e176748cd7acc2240c2f2e5aa3a4"}, + {file = "pandas-2.2.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:15c0e1e02e93116177d29ff83e8b1619c93ddc9c49083f237d4312337a61165d"}, + {file = "pandas-2.2.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ad5b65698ab28ed8d7f18790a0dc58005c7629f227be9ecc1072aa74c0c1d43a"}, + {file = "pandas-2.2.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc6b93f9b966093cb0fd62ff1a7e4c09e6d546ad7c1de191767baffc57628f39"}, + {file = "pandas-2.2.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5dbca4c1acd72e8eeef4753eeca07de9b1db4f398669d5994086f788a5d7cc30"}, + {file = "pandas-2.2.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8cd6d7cc958a3910f934ea8dbdf17b2364827bb4dafc38ce6eef6bb3d65ff09c"}, + {file = "pandas-2.2.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99df71520d25fade9db7c1076ac94eb994f4d2673ef2aa2e86ee039b6746d20c"}, + {file = "pandas-2.2.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:31d0ced62d4ea3e231a9f228366919a5ea0b07440d9d4dac345376fd8e1477ea"}, + {file = "pandas-2.2.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7eee9e7cea6adf3e3d24e304ac6b8300646e2a5d1cd3a3c2abed9101b0846761"}, + {file = "pandas-2.2.3-cp39-cp39-win_amd64.whl", hash = "sha256:4850ba03528b6dd51d6c5d273c46f183f39a9baf3f0143e566b89450965b105e"}, + {file = "pandas-2.2.3.tar.gz", hash = "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667"}, +] + +[package.dependencies] +numpy = {version = ">=1.26.0", markers = "python_version >= \"3.12\""} +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.7" + +[package.extras] +all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] +aws = ["s3fs (>=2022.11.0)"] +clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] +compression = ["zstandard (>=0.19.0)"] +computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] +consortium-standard = ["dataframe-api-compat (>=0.1.7)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] +feather = ["pyarrow (>=10.0.1)"] +fss = ["fsspec (>=2022.11.0)"] +gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] +hdf5 = ["tables (>=3.8.0)"] +html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] +mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] +parquet = ["pyarrow (>=10.0.1)"] +performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] +plot = ["matplotlib (>=3.6.3)"] +postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] +pyarrow = ["pyarrow (>=10.0.1)"] +spss = ["pyreadstat (>=1.2.0)"] +sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.9.2)"] + +[[package]] +name = "pathspec" +version = "0.12.1" +description = "Utility library for gitignore style pattern matching of file paths." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, + {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, +] + +[[package]] +name = "platformdirs" +version = "4.3.6" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb"}, + {file = "platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907"}, +] + +[package.extras] +docs = ["furo (>=2024.8.6)", "proselint (>=0.14)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=8.3.2)", "pytest-cov (>=5)", "pytest-mock (>=3.14)"] +type = ["mypy (>=1.11.2)"] + +[[package]] +name = "pluggy" +version = "1.5.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pydantic" +version = "2.9.2" +description = "Data validation using Python type hints" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic-2.9.2-py3-none-any.whl", hash = "sha256:f048cec7b26778210e28a0459867920654d48e5e62db0958433636cde4254f12"}, + {file = "pydantic-2.9.2.tar.gz", hash = "sha256:d155cef71265d1e9807ed1c32b4c8deec042a44a50a4188b25ac67ecd81a9c0f"}, +] + +[package.dependencies] +annotated-types = ">=0.6.0" +pydantic-core = "2.23.4" +typing-extensions = [ + {version = ">=4.12.2", markers = "python_version >= \"3.13\""}, + {version = ">=4.6.1", markers = "python_version < \"3.13\""}, +] + +[package.extras] +email = ["email-validator (>=2.0.0)"] +timezone = ["tzdata"] + +[[package]] +name = "pydantic-core" +version = "2.23.4" +description = "Core functionality for Pydantic validation and serialization" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic_core-2.23.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:b10bd51f823d891193d4717448fab065733958bdb6a6b351967bd349d48d5c9b"}, + {file = "pydantic_core-2.23.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4fc714bdbfb534f94034efaa6eadd74e5b93c8fa6315565a222f7b6f42ca1166"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63e46b3169866bd62849936de036f901a9356e36376079b05efa83caeaa02ceb"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed1a53de42fbe34853ba90513cea21673481cd81ed1be739f7f2efb931b24916"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cfdd16ab5e59fc31b5e906d1a3f666571abc367598e3e02c83403acabc092e07"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:255a8ef062cbf6674450e668482456abac99a5583bbafb73f9ad469540a3a232"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a7cd62e831afe623fbb7aabbb4fe583212115b3ef38a9f6b71869ba644624a2"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f09e2ff1f17c2b51f2bc76d1cc33da96298f0a036a137f5440ab3ec5360b624f"}, + {file = "pydantic_core-2.23.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e38e63e6f3d1cec5a27e0afe90a085af8b6806ee208b33030e65b6516353f1a3"}, + {file = "pydantic_core-2.23.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0dbd8dbed2085ed23b5c04afa29d8fd2771674223135dc9bc937f3c09284d071"}, + {file = "pydantic_core-2.23.4-cp310-none-win32.whl", hash = "sha256:6531b7ca5f951d663c339002e91aaebda765ec7d61b7d1e3991051906ddde119"}, + {file = "pydantic_core-2.23.4-cp310-none-win_amd64.whl", hash = "sha256:7c9129eb40958b3d4500fa2467e6a83356b3b61bfff1b414c7361d9220f9ae8f"}, + {file = "pydantic_core-2.23.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:77733e3892bb0a7fa797826361ce8a9184d25c8dffaec60b7ffe928153680ba8"}, + {file = "pydantic_core-2.23.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b84d168f6c48fabd1f2027a3d1bdfe62f92cade1fb273a5d68e621da0e44e6d"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df49e7a0861a8c36d089c1ed57d308623d60416dab2647a4a17fe050ba85de0e"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ff02b6d461a6de369f07ec15e465a88895f3223eb75073ffea56b84d9331f607"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:996a38a83508c54c78a5f41456b0103c30508fed9abcad0a59b876d7398f25fd"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d97683ddee4723ae8c95d1eddac7c192e8c552da0c73a925a89fa8649bf13eea"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:216f9b2d7713eb98cb83c80b9c794de1f6b7e3145eef40400c62e86cee5f4e1e"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6f783e0ec4803c787bcea93e13e9932edab72068f68ecffdf86a99fd5918878b"}, + {file = "pydantic_core-2.23.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d0776dea117cf5272382634bd2a5c1b6eb16767c223c6a5317cd3e2a757c61a0"}, + {file = "pydantic_core-2.23.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d5f7a395a8cf1621939692dba2a6b6a830efa6b3cee787d82c7de1ad2930de64"}, + {file = "pydantic_core-2.23.4-cp311-none-win32.whl", hash = "sha256:74b9127ffea03643e998e0c5ad9bd3811d3dac8c676e47db17b0ee7c3c3bf35f"}, + {file = "pydantic_core-2.23.4-cp311-none-win_amd64.whl", hash = "sha256:98d134c954828488b153d88ba1f34e14259284f256180ce659e8d83e9c05eaa3"}, + {file = "pydantic_core-2.23.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f3e0da4ebaef65158d4dfd7d3678aad692f7666877df0002b8a522cdf088f231"}, + {file = "pydantic_core-2.23.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f69a8e0b033b747bb3e36a44e7732f0c99f7edd5cea723d45bc0d6e95377ffee"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:723314c1d51722ab28bfcd5240d858512ffd3116449c557a1336cbe3919beb87"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bb2802e667b7051a1bebbfe93684841cc9351004e2badbd6411bf357ab8d5ac8"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d18ca8148bebe1b0a382a27a8ee60350091a6ddaf475fa05ef50dc35b5df6327"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33e3d65a85a2a4a0dc3b092b938a4062b1a05f3a9abde65ea93b233bca0e03f2"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:128585782e5bfa515c590ccee4b727fb76925dd04a98864182b22e89a4e6ed36"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:68665f4c17edcceecc112dfed5dbe6f92261fb9d6054b47d01bf6371a6196126"}, + {file = "pydantic_core-2.23.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:20152074317d9bed6b7a95ade3b7d6054845d70584216160860425f4fbd5ee9e"}, + {file = "pydantic_core-2.23.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9261d3ce84fa1d38ed649c3638feefeae23d32ba9182963e465d58d62203bd24"}, + {file = "pydantic_core-2.23.4-cp312-none-win32.whl", hash = "sha256:4ba762ed58e8d68657fc1281e9bb72e1c3e79cc5d464be146e260c541ec12d84"}, + {file = "pydantic_core-2.23.4-cp312-none-win_amd64.whl", hash = "sha256:97df63000f4fea395b2824da80e169731088656d1818a11b95f3b173747b6cd9"}, + {file = "pydantic_core-2.23.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:7530e201d10d7d14abce4fb54cfe5b94a0aefc87da539d0346a484ead376c3cc"}, + {file = "pydantic_core-2.23.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:df933278128ea1cd77772673c73954e53a1c95a4fdf41eef97c2b779271bd0bd"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cb3da3fd1b6a5d0279a01877713dbda118a2a4fc6f0d821a57da2e464793f05"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:42c6dcb030aefb668a2b7009c85b27f90e51e6a3b4d5c9bc4c57631292015b0d"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:696dd8d674d6ce621ab9d45b205df149399e4bb9aa34102c970b721554828510"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2971bb5ffe72cc0f555c13e19b23c85b654dd2a8f7ab493c262071377bfce9f6"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8394d940e5d400d04cad4f75c0598665cbb81aecefaca82ca85bd28264af7f9b"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0dff76e0602ca7d4cdaacc1ac4c005e0ce0dcfe095d5b5259163a80d3a10d327"}, + {file = "pydantic_core-2.23.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7d32706badfe136888bdea71c0def994644e09fff0bfe47441deaed8e96fdbc6"}, + {file = "pydantic_core-2.23.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ed541d70698978a20eb63d8c5d72f2cc6d7079d9d90f6b50bad07826f1320f5f"}, + {file = "pydantic_core-2.23.4-cp313-none-win32.whl", hash = "sha256:3d5639516376dce1940ea36edf408c554475369f5da2abd45d44621cb616f769"}, + {file = "pydantic_core-2.23.4-cp313-none-win_amd64.whl", hash = "sha256:5a1504ad17ba4210df3a045132a7baeeba5a200e930f57512ee02909fc5c4cb5"}, + {file = "pydantic_core-2.23.4-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:d4488a93b071c04dc20f5cecc3631fc78b9789dd72483ba15d423b5b3689b555"}, + {file = "pydantic_core-2.23.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:81965a16b675b35e1d09dd14df53f190f9129c0202356ed44ab2728b1c905658"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ffa2ebd4c8530079140dd2d7f794a9d9a73cbb8e9d59ffe24c63436efa8f271"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:61817945f2fe7d166e75fbfb28004034b48e44878177fc54d81688e7b85a3665"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:29d2c342c4bc01b88402d60189f3df065fb0dda3654744d5a165a5288a657368"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5e11661ce0fd30a6790e8bcdf263b9ec5988e95e63cf901972107efc49218b13"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d18368b137c6295db49ce7218b1a9ba15c5bc254c96d7c9f9e924a9bc7825ad"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ec4e55f79b1c4ffb2eecd8a0cfba9955a2588497d96851f4c8f99aa4a1d39b12"}, + {file = "pydantic_core-2.23.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:374a5e5049eda9e0a44c696c7ade3ff355f06b1fe0bb945ea3cac2bc336478a2"}, + {file = "pydantic_core-2.23.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5c364564d17da23db1106787675fc7af45f2f7b58b4173bfdd105564e132e6fb"}, + {file = "pydantic_core-2.23.4-cp38-none-win32.whl", hash = "sha256:d7a80d21d613eec45e3d41eb22f8f94ddc758a6c4720842dc74c0581f54993d6"}, + {file = "pydantic_core-2.23.4-cp38-none-win_amd64.whl", hash = "sha256:5f5ff8d839f4566a474a969508fe1c5e59c31c80d9e140566f9a37bba7b8d556"}, + {file = "pydantic_core-2.23.4-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:a4fa4fc04dff799089689f4fd502ce7d59de529fc2f40a2c8836886c03e0175a"}, + {file = "pydantic_core-2.23.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0a7df63886be5e270da67e0966cf4afbae86069501d35c8c1b3b6c168f42cb36"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dcedcd19a557e182628afa1d553c3895a9f825b936415d0dbd3cd0bbcfd29b4b"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f54b118ce5de9ac21c363d9b3caa6c800341e8c47a508787e5868c6b79c9323"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86d2f57d3e1379a9525c5ab067b27dbb8a0642fb5d454e17a9ac434f9ce523e3"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:de6d1d1b9e5101508cb37ab0d972357cac5235f5c6533d1071964c47139257df"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1278e0d324f6908e872730c9102b0112477a7f7cf88b308e4fc36ce1bdb6d58c"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a6b5099eeec78827553827f4c6b8615978bb4b6a88e5d9b93eddf8bb6790f55"}, + {file = "pydantic_core-2.23.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e55541f756f9b3ee346b840103f32779c695a19826a4c442b7954550a0972040"}, + {file = "pydantic_core-2.23.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a5c7ba8ffb6d6f8f2ab08743be203654bb1aaa8c9dcb09f82ddd34eadb695605"}, + {file = "pydantic_core-2.23.4-cp39-none-win32.whl", hash = "sha256:37b0fe330e4a58d3c58b24d91d1eb102aeec675a3db4c292ec3928ecd892a9a6"}, + {file = "pydantic_core-2.23.4-cp39-none-win_amd64.whl", hash = "sha256:1498bec4c05c9c787bde9125cfdcc63a41004ff167f495063191b863399b1a29"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f455ee30a9d61d3e1a15abd5068827773d6e4dc513e795f380cdd59932c782d5"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:1e90d2e3bd2c3863d48525d297cd143fe541be8bbf6f579504b9712cb6b643ec"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e203fdf807ac7e12ab59ca2bfcabb38c7cf0b33c41efeb00f8e5da1d86af480"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e08277a400de01bc72436a0ccd02bdf596631411f592ad985dcee21445bd0068"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f220b0eea5965dec25480b6333c788fb72ce5f9129e8759ef876a1d805d00801"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d06b0c8da4f16d1d1e352134427cb194a0a6e19ad5db9161bf32b2113409e728"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:ba1a0996f6c2773bd83e63f18914c1de3c9dd26d55f4ac302a7efe93fb8e7433"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:9a5bce9d23aac8f0cf0836ecfc033896aa8443b501c58d0602dbfd5bd5b37753"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:78ddaaa81421a29574a682b3179d4cf9e6d405a09b99d93ddcf7e5239c742e21"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:883a91b5dd7d26492ff2f04f40fbb652de40fcc0afe07e8129e8ae779c2110eb"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88ad334a15b32a791ea935af224b9de1bf99bcd62fabf745d5f3442199d86d59"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:233710f069d251feb12a56da21e14cca67994eab08362207785cf8c598e74577"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:19442362866a753485ba5e4be408964644dd6a09123d9416c54cd49171f50744"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:624e278a7d29b6445e4e813af92af37820fafb6dcc55c012c834f9e26f9aaaef"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f5ef8f42bec47f21d07668a043f077d507e5bf4e668d5c6dfe6aaba89de1a5b8"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:aea443fffa9fbe3af1a9ba721a87f926fe548d32cab71d188a6ede77d0ff244e"}, + {file = "pydantic_core-2.23.4.tar.gz", hash = "sha256:2584f7cf844ac4d970fba483a717dbe10c1c1c96a969bf65d61ffe94df1b2863"}, +] + +[package.dependencies] +typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" + +[[package]] +name = "pytest" +version = "8.3.3" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-8.3.3-py3-none-any.whl", hash = "sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2"}, + {file = "pytest-8.3.3.tar.gz", hash = "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=1.5,<2" + +[package.extras] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytz" +version = "2024.2" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2024.2-py2.py3-none-any.whl", hash = "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725"}, + {file = "pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.2" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, + {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68"}, + {file = "PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99"}, + {file = "PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e"}, + {file = "PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5"}, + {file = "PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b"}, + {file = "PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4"}, + {file = "PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652"}, + {file = "PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183"}, + {file = "PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563"}, + {file = "PyYAML-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083"}, + {file = "PyYAML-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706"}, + {file = "PyYAML-6.0.2-cp38-cp38-win32.whl", hash = "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a"}, + {file = "PyYAML-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725"}, + {file = "PyYAML-6.0.2-cp39-cp39-win32.whl", hash = "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631"}, + {file = "PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8"}, + {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, +] + +[[package]] +name = "requests" +version = "2.32.3" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.8" +files = [ + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "s3transfer" +version = "0.10.3" +description = "An Amazon S3 Transfer Manager" +optional = false +python-versions = ">=3.8" +files = [ + {file = "s3transfer-0.10.3-py3-none-any.whl", hash = "sha256:263ed587a5803c6c708d3ce44dc4dfedaab4c1a32e8329bab818933d79ddcf5d"}, + {file = "s3transfer-0.10.3.tar.gz", hash = "sha256:4f50ed74ab84d474ce614475e0b8d5047ff080810aac5d01ea25231cfc944b0c"}, +] + +[package.dependencies] +botocore = ">=1.33.2,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "sniffio" +version = "1.3.1" +description = "Sniff out which async library your code is running under" +optional = false +python-versions = ">=3.7" +files = [ + {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"}, + {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, +] + +[[package]] +name = "tokenizers" +version = "0.20.1" +description = "" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tokenizers-0.20.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:439261da7c0a5c88bda97acb284d49fbdaf67e9d3b623c0bfd107512d22787a9"}, + {file = "tokenizers-0.20.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:03dae629d99068b1ea5416d50de0fea13008f04129cc79af77a2a6392792d93c"}, + {file = "tokenizers-0.20.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b61f561f329ffe4b28367798b89d60c4abf3f815d37413b6352bc6412a359867"}, + {file = "tokenizers-0.20.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ec870fce1ee5248a10be69f7a8408a234d6f2109f8ea827b4f7ecdbf08c9fd15"}, + {file = "tokenizers-0.20.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d388d1ea8b7447da784e32e3b86a75cce55887e3b22b31c19d0b186b1c677800"}, + {file = "tokenizers-0.20.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:299c85c1d21135bc01542237979bf25c32efa0d66595dd0069ae259b97fb2dbe"}, + {file = "tokenizers-0.20.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e96f6c14c9752bb82145636b614d5a78e9cde95edfbe0a85dad0dd5ddd6ec95c"}, + {file = "tokenizers-0.20.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc9e95ad49c932b80abfbfeaf63b155761e695ad9f8a58c52a47d962d76e310f"}, + {file = "tokenizers-0.20.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:f22dee205329a636148c325921c73cf3e412e87d31f4d9c3153b302a0200057b"}, + {file = "tokenizers-0.20.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a2ffd9a8895575ac636d44500c66dffaef133823b6b25067604fa73bbc5ec09d"}, + {file = "tokenizers-0.20.1-cp310-none-win32.whl", hash = "sha256:2847843c53f445e0f19ea842a4e48b89dd0db4e62ba6e1e47a2749d6ec11f50d"}, + {file = "tokenizers-0.20.1-cp310-none-win_amd64.whl", hash = "sha256:f9aa93eacd865f2798b9e62f7ce4533cfff4f5fbd50c02926a78e81c74e432cd"}, + {file = "tokenizers-0.20.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:4a717dcb08f2dabbf27ae4b6b20cbbb2ad7ed78ce05a829fae100ff4b3c7ff15"}, + {file = "tokenizers-0.20.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3f84dad1ff1863c648d80628b1b55353d16303431283e4efbb6ab1af56a75832"}, + {file = "tokenizers-0.20.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:929c8f3afa16a5130a81ab5079c589226273ec618949cce79b46d96e59a84f61"}, + {file = "tokenizers-0.20.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d10766473954397e2d370f215ebed1cc46dcf6fd3906a2a116aa1d6219bfedc3"}, + {file = "tokenizers-0.20.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9300fac73ddc7e4b0330acbdda4efaabf74929a4a61e119a32a181f534a11b47"}, + {file = "tokenizers-0.20.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0ecaf7b0e39caeb1aa6dd6e0975c405716c82c1312b55ac4f716ef563a906969"}, + {file = "tokenizers-0.20.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5170be9ec942f3d1d317817ced8d749b3e1202670865e4fd465e35d8c259de83"}, + {file = "tokenizers-0.20.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef3f1ae08fa9aea5891cbd69df29913e11d3841798e0bfb1ff78b78e4e7ea0a4"}, + {file = "tokenizers-0.20.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ee86d4095d3542d73579e953c2e5e07d9321af2ffea6ecc097d16d538a2dea16"}, + {file = "tokenizers-0.20.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:86dcd08da163912e17b27bbaba5efdc71b4fbffb841530fdb74c5707f3c49216"}, + {file = "tokenizers-0.20.1-cp311-none-win32.whl", hash = "sha256:9af2dc4ee97d037bc6b05fa4429ddc87532c706316c5e11ce2f0596dfcfa77af"}, + {file = "tokenizers-0.20.1-cp311-none-win_amd64.whl", hash = "sha256:899152a78b095559c287b4c6d0099469573bb2055347bb8154db106651296f39"}, + {file = "tokenizers-0.20.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:407ab666b38e02228fa785e81f7cf79ef929f104bcccf68a64525a54a93ceac9"}, + {file = "tokenizers-0.20.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2f13a2d16032ebc8bd812eb8099b035ac65887d8f0c207261472803b9633cf3e"}, + {file = "tokenizers-0.20.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e98eee4dca22849fbb56a80acaa899eec5b72055d79637dd6aa15d5e4b8628c9"}, + {file = "tokenizers-0.20.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:47c1bcdd61e61136087459cb9e0b069ff23b5568b008265e5cbc927eae3387ce"}, + {file = "tokenizers-0.20.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:128c1110e950534426e2274837fc06b118ab5f2fa61c3436e60e0aada0ccfd67"}, + {file = "tokenizers-0.20.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e2e2d47a819d2954f2c1cd0ad51bb58ffac6f53a872d5d82d65d79bf76b9896d"}, + {file = "tokenizers-0.20.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bdd67a0e3503a9a7cf8bc5a4a49cdde5fa5bada09a51e4c7e1c73900297539bd"}, + {file = "tokenizers-0.20.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:689b93d2e26d04da337ac407acec8b5d081d8d135e3e5066a88edd5bdb5aff89"}, + {file = "tokenizers-0.20.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0c6a796ddcd9a19ad13cf146997cd5895a421fe6aec8fd970d69f9117bddb45c"}, + {file = "tokenizers-0.20.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3ea919687aa7001a8ff1ba36ac64f165c4e89035f57998fa6cedcfd877be619d"}, + {file = "tokenizers-0.20.1-cp312-none-win32.whl", hash = "sha256:6d3ac5c1f48358ffe20086bf065e843c0d0a9fce0d7f0f45d5f2f9fba3609ca5"}, + {file = "tokenizers-0.20.1-cp312-none-win_amd64.whl", hash = "sha256:b0874481aea54a178f2bccc45aa2d0c99cd3f79143a0948af6a9a21dcc49173b"}, + {file = "tokenizers-0.20.1-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:96af92e833bd44760fb17f23f402e07a66339c1dcbe17d79a9b55bb0cc4f038e"}, + {file = "tokenizers-0.20.1-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:65f34e5b731a262dfa562820818533c38ce32a45864437f3d9c82f26c139ca7f"}, + {file = "tokenizers-0.20.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:17f98fccb5c12ab1ce1f471731a9cd86df5d4bd2cf2880c5a66b229802d96145"}, + {file = "tokenizers-0.20.1-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b8c0fc3542cf9370bf92c932eb71bdeb33d2d4aeeb4126d9fd567b60bd04cb30"}, + {file = "tokenizers-0.20.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b39356df4575d37f9b187bb623aab5abb7b62c8cb702867a1768002f814800c"}, + {file = "tokenizers-0.20.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfdad27b0e50544f6b838895a373db6114b85112ba5c0cefadffa78d6daae563"}, + {file = "tokenizers-0.20.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:094663dd0e85ee2e573126918747bdb40044a848fde388efb5b09d57bc74c680"}, + {file = "tokenizers-0.20.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14e4cf033a2aa207d7ac790e91adca598b679999710a632c4a494aab0fc3a1b2"}, + {file = "tokenizers-0.20.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:9310951c92c9fb91660de0c19a923c432f110dbfad1a2d429fbc44fa956bf64f"}, + {file = "tokenizers-0.20.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:05e41e302c315bd2ed86c02e917bf03a6cf7d2f652c9cee1a0eb0d0f1ca0d32c"}, + {file = "tokenizers-0.20.1-cp37-none-win32.whl", hash = "sha256:212231ab7dfcdc879baf4892ca87c726259fa7c887e1688e3f3cead384d8c305"}, + {file = "tokenizers-0.20.1-cp37-none-win_amd64.whl", hash = "sha256:896195eb9dfdc85c8c052e29947169c1fcbe75a254c4b5792cdbd451587bce85"}, + {file = "tokenizers-0.20.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:741fb22788482d09d68e73ece1495cfc6d9b29a06c37b3df90564a9cfa688e6d"}, + {file = "tokenizers-0.20.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:10be14ebd8082086a342d969e17fc2d6edc856c59dbdbddd25f158fa40eaf043"}, + {file = "tokenizers-0.20.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:514cf279b22fa1ae0bc08e143458c74ad3b56cd078b319464959685a35c53d5e"}, + {file = "tokenizers-0.20.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a647c5b7cb896d6430cf3e01b4e9a2d77f719c84cefcef825d404830c2071da2"}, + {file = "tokenizers-0.20.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7cdf379219e1e1dd432091058dab325a2e6235ebb23e0aec8d0508567c90cd01"}, + {file = "tokenizers-0.20.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ba72260449e16c4c2f6f3252823b059fbf2d31b32617e582003f2b18b415c39"}, + {file = "tokenizers-0.20.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:910b96ed87316e4277b23c7bcaf667ce849c7cc379a453fa179e7e09290eeb25"}, + {file = "tokenizers-0.20.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e53975a6694428a0586534cc1354b2408d4e010a3103117f617cbb550299797c"}, + {file = "tokenizers-0.20.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:07c4b7be58da142b0730cc4e5fd66bb7bf6f57f4986ddda73833cd39efef8a01"}, + {file = "tokenizers-0.20.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b605c540753e62199bf15cf69c333e934077ef2350262af2ccada46026f83d1c"}, + {file = "tokenizers-0.20.1-cp38-none-win32.whl", hash = "sha256:88b3bc76ab4db1ab95ead623d49c95205411e26302cf9f74203e762ac7e85685"}, + {file = "tokenizers-0.20.1-cp38-none-win_amd64.whl", hash = "sha256:d412a74cf5b3f68a90c615611a5aa4478bb303d1c65961d22db45001df68afcb"}, + {file = "tokenizers-0.20.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:a25dcb2f41a0a6aac31999e6c96a75e9152fa0127af8ece46c2f784f23b8197a"}, + {file = "tokenizers-0.20.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a12c3cebb8c92e9c35a23ab10d3852aee522f385c28d0b4fe48c0b7527d59762"}, + {file = "tokenizers-0.20.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:02e18da58cf115b7c40de973609c35bde95856012ba42a41ee919c77935af251"}, + {file = "tokenizers-0.20.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f326a1ac51ae909b9760e34671c26cd0dfe15662f447302a9d5bb2d872bab8ab"}, + {file = "tokenizers-0.20.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0b4872647ea6f25224e2833b044b0b19084e39400e8ead3cfe751238b0802140"}, + {file = "tokenizers-0.20.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce6238a3311bb8e4c15b12600927d35c267b92a52c881ef5717a900ca14793f7"}, + {file = "tokenizers-0.20.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:57b7a8880b208866508b06ce365dc631e7a2472a3faa24daa430d046fb56c885"}, + {file = "tokenizers-0.20.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a908c69c2897a68f412aa05ba38bfa87a02980df70f5a72fa8490479308b1f2d"}, + {file = "tokenizers-0.20.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:da1001aa46f4490099c82e2facc4fbc06a6a32bf7de3918ba798010954b775e0"}, + {file = "tokenizers-0.20.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:42c097390e2f0ed0a5c5d569e6669dd4e9fff7b31c6a5ce6e9c66a61687197de"}, + {file = "tokenizers-0.20.1-cp39-none-win32.whl", hash = "sha256:3d4d218573a3d8b121a1f8c801029d70444ffb6d8f129d4cca1c7b672ee4a24c"}, + {file = "tokenizers-0.20.1-cp39-none-win_amd64.whl", hash = "sha256:37d1e6f616c84fceefa7c6484a01df05caf1e207669121c66213cb5b2911d653"}, + {file = "tokenizers-0.20.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:48689da7a395df41114f516208d6550e3e905e1239cc5ad386686d9358e9cef0"}, + {file = "tokenizers-0.20.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:712f90ea33f9bd2586b4a90d697c26d56d0a22fd3c91104c5858c4b5b6489a79"}, + {file = "tokenizers-0.20.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:359eceb6a620c965988fc559cebc0a98db26713758ec4df43fb76d41486a8ed5"}, + {file = "tokenizers-0.20.1-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0d3caf244ce89d24c87545aafc3448be15870096e796c703a0d68547187192e1"}, + {file = "tokenizers-0.20.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:03b03cf8b9a32254b1bf8a305fb95c6daf1baae0c1f93b27f2b08c9759f41dee"}, + {file = "tokenizers-0.20.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:218e5a3561561ea0f0ef1559c6d95b825308dbec23fb55b70b92589e7ff2e1e8"}, + {file = "tokenizers-0.20.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f40df5e0294a95131cc5f0e0eb91fe86d88837abfbee46b9b3610b09860195a7"}, + {file = "tokenizers-0.20.1-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:08aaa0d72bb65058e8c4b0455f61b840b156c557e2aca57627056624c3a93976"}, + {file = "tokenizers-0.20.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:998700177b45f70afeb206ad22c08d9e5f3a80639dae1032bf41e8cbc4dada4b"}, + {file = "tokenizers-0.20.1-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62f7fbd3c2c38b179556d879edae442b45f68312019c3a6013e56c3947a4e648"}, + {file = "tokenizers-0.20.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31e87fca4f6bbf5cc67481b562147fe932f73d5602734de7dd18a8f2eee9c6dd"}, + {file = "tokenizers-0.20.1-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:956f21d359ae29dd51ca5726d2c9a44ffafa041c623f5aa33749da87cfa809b9"}, + {file = "tokenizers-0.20.1-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:1fbbaf17a393c78d8aedb6a334097c91cb4119a9ced4764ab8cfdc8d254dc9f9"}, + {file = "tokenizers-0.20.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ebe63e31f9c1a970c53866d814e35ec2ec26fda03097c486f82f3891cee60830"}, + {file = "tokenizers-0.20.1-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:81970b80b8ac126910295f8aab2d7ef962009ea39e0d86d304769493f69aaa1e"}, + {file = "tokenizers-0.20.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:130e35e76f9337ed6c31be386e75d4925ea807055acf18ca1a9b0eec03d8fe23"}, + {file = "tokenizers-0.20.1-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd28a8614f5c82a54ab2463554e84ad79526c5184cf4573bbac2efbbbcead457"}, + {file = "tokenizers-0.20.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9041ee665d0fa7f5c4ccf0f81f5e6b7087f797f85b143c094126fc2611fec9d0"}, + {file = "tokenizers-0.20.1-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:62eb9daea2a2c06bcd8113a5824af8ef8ee7405d3a71123ba4d52c79bb3d9f1a"}, + {file = "tokenizers-0.20.1-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f861889707b54a9ab1204030b65fd6c22bdd4a95205deec7994dc22a8baa2ea4"}, + {file = "tokenizers-0.20.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:89d5c337d74ea6e5e7dc8af124cf177be843bbb9ca6e58c01f75ea103c12c8a9"}, + {file = "tokenizers-0.20.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:0b7f515c83397e73292accdbbbedc62264e070bae9682f06061e2ddce67cacaf"}, + {file = "tokenizers-0.20.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e0305fc1ec6b1e5052d30d9c1d5c807081a7bd0cae46a33d03117082e91908c"}, + {file = "tokenizers-0.20.1-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5dc611e6ac0fa00a41de19c3bf6391a05ea201d2d22b757d63f5491ec0e67faa"}, + {file = "tokenizers-0.20.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5ffe0d7f7bfcfa3b2585776ecf11da2e01c317027c8573c78ebcb8985279e23"}, + {file = "tokenizers-0.20.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:e7edb8ec12c100d5458d15b1e47c0eb30ad606a05641f19af7563bc3d1608c14"}, + {file = "tokenizers-0.20.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:de291633fb9303555793cc544d4a86e858da529b7d0b752bcaf721ae1d74b2c9"}, + {file = "tokenizers-0.20.1.tar.gz", hash = "sha256:84edcc7cdeeee45ceedb65d518fffb77aec69311c9c8e30f77ad84da3025f002"}, +] + +[package.dependencies] +huggingface-hub = ">=0.16.4,<1.0" + +[package.extras] +dev = ["tokenizers[testing]"] +docs = ["setuptools-rust", "sphinx", "sphinx-rtd-theme"] +testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests", "ruff"] + +[[package]] +name = "tqdm" +version = "4.66.6" +description = "Fast, Extensible Progress Meter" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tqdm-4.66.6-py3-none-any.whl", hash = "sha256:223e8b5359c2efc4b30555531f09e9f2f3589bcd7fdd389271191031b49b7a63"}, + {file = "tqdm-4.66.6.tar.gz", hash = "sha256:4bdd694238bef1485ce839d67967ab50af8f9272aab687c0d7702a01da0be090"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["pytest (>=6)", "pytest-cov", "pytest-timeout", "pytest-xdist"] +notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] +telegram = ["requests"] + +[[package]] +name = "typing-extensions" +version = "4.12.2" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, + {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, +] + +[[package]] +name = "tzdata" +version = "2024.2" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2024.2-py2.py3-none-any.whl", hash = "sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd"}, + {file = "tzdata-2024.2.tar.gz", hash = "sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc"}, +] + +[[package]] +name = "urllib3" +version = "2.2.3" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.3-py3-none-any.whl", hash = "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac"}, + {file = "urllib3-2.2.3.tar.gz", hash = "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.12" +content-hash = "195ab38a76cbe281ddfaf343f1e5da592ac8b3e6e323a3f03f4ff640fc3f7398" diff --git a/fieldExtraction/pyproject.toml b/fieldExtraction/pyproject.toml new file mode 100644 index 0000000..0811440 --- /dev/null +++ b/fieldExtraction/pyproject.toml @@ -0,0 +1,39 @@ +[tool.poetry] +name = "doczy-smart-chunking" +version = "0.1.0" +description = "" +authors = ["Alex Galarce "] +readme = "README.md" +package-mode = false + +[tool.poetry.dependencies] +python = "^3.12" +pandas = "^2.2.3" +boto3 = "^1.35.40" +anthropic = "^0.36.0" + +[tool.poetry.group.dev.dependencies] +black = "^24.10.0" +openpyxl = "^3.1.5" +mypy = "^1.12.0" + +[tool.poetry.group.test.dependencies] +pytest = "^8.3.3" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +[tool.black] +line-length = 88 +target-version = ['py312'] + +[tool.mypy] +disable_error_code = ["import-untyped","assignment","name-defined","call-arg","var-annotated","attr-defined","arg-type"] +install_types = true +non_interactive = true + +[tool.pytest.ini_options] +testpaths = [ + "tests" +] \ No newline at end of file diff --git a/fieldExtraction/scripts/adhoc/carveouts.py b/fieldExtraction/scripts/adhoc/carveouts.py new file mode 100644 index 0000000..675cb88 --- /dev/null +++ b/fieldExtraction/scripts/adhoc/carveouts.py @@ -0,0 +1,163 @@ +import pandas as pd +from difflib import SequenceMatcher +import re +import time + +import prompts +import claude_funcs + + +def get_closest_substring_match(val, valid_values): + """Returns the first match from valid_values using a case-insensitive substring match.""" + if pd.isna(val): + return None + val = val.strip().upper() + for valid_value in valid_values: + if valid_value.upper() in val: + return valid_value + return None + + +def get_best_carveout_from_claude(carveout_list, service): + print("using claude to get best carveout...") + prompt = f"Given the service '{service}', please choose the best matching carveout from the following list: {', '.join(carveout_list)}. ONLY SELECT ONE FROM THE LIST. DO NOT RETURN A SENTENCE" + while True: + try: + response = claude_funcs.invoke_claude_3(prompt, max_tokens=100) + return response.strip() + except Exception as e: + print(f"Error occurred: {e}. Waiting for 60 seconds before retrying...") + time.sleep(60) + + +def check_prov_type_similarity(prov_type, carveout): + print("checking similarity between prov_type and carveout with claude...") + prompt = f"Do the provider type '{prov_type}' and the carveout '{carveout}' mean the same thing or are they very similar? ONLY ANSWER WITH 'True' OR 'False'" + while True: + try: + response = claude_funcs.invoke_claude_3(prompt, max_tokens=10) + return response.strip() + except Exception as e: + print(f"Error occurred: {e}. Waiting for 60 seconds before retrying...") + time.sleep(60) + + +def label_services(filepath, carveout_list, output_filepath): + # Read the CSV file + df = pd.read_csv(filepath) + + # Select the first 50 rows of the DataFrame + df = df.head(500) + + # Define primary service terms + primary_terms = [ + "Covered Services", + "Inpatient Services", + "Outpatient Services", + "Physician Services", + "Inpatient", + "Outpatient", + "Physician", + "Medical", + "Surgical", + "Diagnostic", + "Therapeutic", + ] + + # Initialize columns for the results + df["carveout_matched"] = "" + df["label"] = "" + df["IS_CARVEOUT"] = "" + + # Initialize a nested dictionary to track occurrences for each Filename and TD_LOB + occurrence_tracker = {} + + carveout_count = 0 + + # Iterate through the rows in the DataFrame + for index, row in df.iterrows(): + service = str(row["SERVICE"]) + prov_type = str(row["PROV_TYPE"]) if pd.notna(row["PROV_TYPE"]) else "" + td_lob = row["TD_LOB"] + filename = row["Filename"] + + # Initialize the nested dictionary for the filename and TD_LOB if not present + if filename not in occurrence_tracker: + occurrence_tracker[filename] = {} + if td_lob not in occurrence_tracker[filename]: + occurrence_tracker[filename][td_lob] = {term: 0 for term in primary_terms} + + # Check if the service is a primary or similar to a primary + primary = get_closest_substring_match(service, primary_terms) + if primary: + # Ensure the primary term is initialized in the occurrence tracker + if primary not in occurrence_tracker[filename][td_lob]: + occurrence_tracker[filename][td_lob][primary] = 0 + + # Determine the label based on the count of this primary term for the given Filename and TD_LOB + count = occurrence_tracker[filename][td_lob][primary] + if count == 0: + df.at[index, "label"] = "primary" + elif count == 1: + df.at[index, "label"] = "secondary" + elif count == 2: + df.at[index, "label"] = "tertiary" + else: + df.at[index, "label"] = "additional" + occurrence_tracker[filename][td_lob][primary] += 1 + df.at[index, "IS_CARVEOUT"] = "N" + else: + # If PROV_TYPE is blank, continue with carveout determination + best_carveout_response = get_best_carveout_from_claude( + carveout_list, service + ) + matched_carveout = best_carveout_response # Use the model response directly + + df.at[index, "carveout_matched"] = matched_carveout + + if matched_carveout: + # Check similarity between PROV_TYPE and carveout + prov_type_similarity_score = SequenceMatcher( + None, prov_type, matched_carveout + ).ratio() + if prov_type_similarity_score < 0.8: + prov_type_similarity = check_prov_type_similarity( + prov_type, matched_carveout + ) + else: + prov_type_similarity = "True" + + if re.search(r"\b(True|Yes)\b", prov_type_similarity, re.IGNORECASE): + if matched_carveout not in occurrence_tracker[filename][td_lob]: + print( + "claude returned a match between provider type and service" + ) + occurrence_tracker[filename][td_lob][matched_carveout] = 0 + count = occurrence_tracker[filename][td_lob][matched_carveout] + if count == 0: + df.at[index, "label"] = "primary" + elif count == 1: + df.at[index, "label"] = "secondary" + elif count == 2: + df.at[index, "label"] = "tertiary" + else: + df.at[index, "label"] = "additional" + occurrence_tracker[filename][td_lob][matched_carveout] += 1 + df.at[index, "IS_CARVEOUT"] = "N" + else: + df.at[index, "IS_CARVEOUT"] = "Y" + else: + df.at[index, "IS_CARVEOUT"] = "Y" + + carveout_count += 1 + + # Save the updated DataFrame to a new CSV file after every 10 carveouts + if carveout_count % 10 == 0: + df.to_csv(output_filepath, index=False) + print(f"Saved progress after processing {carveout_count} carveouts.") + + # Final save of the updated DataFrame to a new CSV file + df.to_csv(output_filepath, index=False) + + +label_services("service.csv", prompts.get_carveout_list(), "carveouts50.csv") diff --git a/fieldExtraction/scripts/adhoc/cnc_ac1_rerun.py b/fieldExtraction/scripts/adhoc/cnc_ac1_rerun.py new file mode 100644 index 0000000..03efa75 --- /dev/null +++ b/fieldExtraction/scripts/adhoc/cnc_ac1_rerun.py @@ -0,0 +1,228 @@ +import concurrent.futures +import traceback +import csv +import time +import sys +import random + +random.seed(42) +import pandas as pd +import os +import re + +import utils +import config +import claude_funcs +import file_processing +import tracking +import consolidate_output +import top_down_funcs +import valid +import ac_funcs +import preprocess +import prompts +import postprocess +import postprocessing_funcs +import preprocessing_funcs + +individual_path = "output_individual/cnc_3_ac1" +clean_path = "reference_files/CNC-Batch 3 Output_File1.xlsx" + + +AC_DICT = { + "CONTRACT_EFFECTIVE_DT": "What is the contract effective date mentioned in any of the following locations: the signatory section or the preamble of the agreement, or the start of the amendment ? Return this date converted to YYYY-MM-DD format.", + "PROV_GROUP_NPI": "What is the group provider's national provider identifier number mentioned in the contract? This may be found on the signature page or on the roster. It consists of 10 digits. Only return these 10 digits.", + "PROV_GROUP_TIN": "What is the Group provider's taxpayer identification number stated in the contract? This is a 9 digit long number typically with a hyphen after the first 2 digits. Return only this 9 digit number with a hyphen after the first 2 digits. Do not elaborate or add context.", +} + +combined_mapping = { + "Contract_Name": "Filename", + "Agreement_Name_Contract_Title": "CONTRACT_TITLE", + "Payer_Name": "PAYER_NAME", + "Health_Plan_State": "HEALTH_PLAN_STATE", + "Affiliate_Y_N": "AFFILIATION_CLAUSE_IND", + "Credentialing_Application_Indicator": "CREDENTIALING_APP_IND", + "Term_Clause": "TERM_CLAUSE", + "Contract_Auto_Renewal_Indicator": "CONTRACT_AUTO_RENEWAL_IND", + "Termination_Date": "CONTRACT_TERMINATION_DT", + "Termination_Upon_Notice_Days": "TERMINATION_UPON_NOTICE", + "Termination_With_Cause_Days": "TERMINATION_UPON_CAUSE", + "Amend_Contract_Upon_notice_Flag_Y_N": "AMEND_CONTRACT_NOTICE_IND", + "Timeframe_to_Object_Days": "TIME_TO_OBJECT", + "Assignments_Clause_Y_N": "ASSIGNMENTS_CLAUSE_IND", + "Contract_Effective_Date": "CONTRACT_EFFECTIVE_DT", + "IRS_Num": "PROV_GROUP_TIN", + "PROV_GROUP_NAME": "PROV_GROUP_NAME", + "NPI_10_digits": "PROV_GROUP_NPI", + "NPI_NAME": "NPI_NAME", + "PROV_GROUP_TIN_SIGNATORY": "PROV_GROUP_TIN_SIGNATORY", + "PROV_TIN_OTHER": "PROV_TIN_OTHER", + "PROV_NPI_OTHER": "PROV_NPI_OTHER", + "Notice_to_Provider_Name": "NOTICE_PROVIDER_NAME", + "Notice_to_Provider_Address": "NOTICE_PROVIDER_ADDRESS", + "Sequestration_Language": "SEQUESTRATION_LANGUAGE", + "Sequestration_Reductions_included_Medicare_only_Y_N": "SEQUESTRATION_REDUCTIONS_IND", + "Parent_Agreement_Code": "Parent Agreement Code", + "Pages": "Pages", + "Page_num": "page_num", + "Attachment_Exhibit": "EXHIBIT", + "Line_of_Business": "CONTRACT_LOB", + "Provider_Type": "PROV_TYPE", + "Provider_Type_Level_2": "PROV_TYPE_LEVEL_2", + "IP_OP": "IP_OP", + "Service_Type": "FULL_SERVICE", + "Plan_Type": "CONTRACT_PROGRAM", + "Lesser_of_Logic_Language_included_Y_N": "LESSER", + "Lesser_of_Rate": "LESSER_RATE", + "Reimb_Methodology": "FULL_METHODOLOGY", + "Reimb_Methodology_short": "SHORT_METHODOLOGY", + "If_rate_is_percent_of_Payer_or_MCR_STANDARD": "RATE_STANDARD", + "If_rate_is_percent_of_Payer_or_MCR_STANDARD_Short": "RATE_SHORT", + "Flat_Fee": "FLAT_FEE_STANDARD", + "Default_Term": "DEFAULT_TERM", + "Default_Rate": "DEFAULT_RATE", + "Inclusion_of_essential_RBRVS_Fee_Source_Language_Y_N": 'Inclusion of essential RBRVS "Fee Source" Language (Y/N)', + "CDM_Language_included_Y_N": "CDM_IND", + "CHARGEMASTER": "CHARGEMASTER", + "IP_DSH_IME_UC_included_Y_N": "IP - DSH/IME/UC, included (Y/N)", + "IP_Stoploss_Catastrophic_Threshold": "IP - Stoploss Catastrophic Threshold", + "EXCLUSIONS": "EXCLUSIONS", + "Not_to_Exceed": "NOT_TO_EXCEED", + "Escalator_or_COLA_Y_N": "RATE_ESCALATOR_IND", + "Escalator_I_Eff_Date": "RATE_ESCALATOR_DT", +} + +import ac_funcs +import config +import postprocess +import tracking + + +def ac_postprocess(final_df): + final_df = postprocessing_funcs.clean_ac_fields(final_df) + final_df = final_df.rename(columns=valid.AC_MAPPING) + final_df = final_df[[col for col in valid.ABC_COLUMNS if col in final_df.columns]] + return final_df + + +def run_ac_prompts(file_object): + ################## INITIATE PROCESSING ################## + filename, contract_text = file_object + print(f"Processing AC for {filename}...") + # print(f"Total contract word count: {len(contract_text.split())}") + + ################## PREPROCESS ################## + text_dict, exhibit_pages, num_pages, ac_chunks = preprocess.preprocess( + contract_text, filename, fields="ac" + ) + # print(f"AC Preprocessing Complete - {filename}") + + ################## RUN FULL CONTEXT PROMPTS ################## + ac_dict = {} + log_data = [] + + # Process 'full_context' fields together + full_context_fields = [field for field in AC_DICT] + full_context_questions = AC_DICT + + if full_context_questions: + full_context_prompt = ac_funcs.create_prompt( + contract_text[0 : min(len(contract_text), 600000)], + question=full_context_questions, + ) + # print(f"High accuracy prompt word count: {len(high_accuracy_prompt.split())}") + try: + full_context_answers = ac_funcs.get_ac_answer( + full_context_prompt, + filename, + fields=list(full_context_questions.keys()), + ) + ac_dict.update(full_context_answers) + + for field, answer in full_context_answers.items(): + log_data.append( + { + "Field": field, + "Prompt": f"Question: {full_context_questions[field]}\n\nContext: {contract_text}", # Include full context + "Context Type": "Full Context (High Accuracy)", + "Response": answer, + } + ) + except Exception as e: + # print(f"Error processing high accuracy fields: {str(e)}") + for field in full_context_questions.keys(): + ac_dict[field] = f"Error: {str(e)}" + log_data.append( + { + "Field": field, + "Prompt": f"Question: {full_context_questions[field]}\n\nContext: {contract_text}", # Include full context + "Context Type": "Full Context (High Accuracy)", + "Response": f"Error: {str(e)}", + } + ) + + ################## CREATE OUTPUT DIRECTORIES ################## + base_filename = os.path.splitext(filename)[0].strip() + output_dir = os.path.join(individual_path, base_filename) + os.makedirs(output_dir, exist_ok=True) + + ################## POSTPROCESS ################## + ac_dict["Contract Name"] = filename + ac_df = pd.DataFrame([ac_dict]) + ac_df = ac_postprocess(ac_df) + + ################## WRITE TO OUTPUT ################## + ac_df.to_csv(f"{output_dir}/ac_output.csv", index=False) + print(f"AC Output written: {filename}") + + +def process_ac_adhoc(item): + filename, contract_text = item + try: + results = run_ac_prompts(item) + if results is not None: + tracking.update_results_csv(item, results) + return item, results + except Exception as e: + print(f"Error processing item {filename}: {e}") + traceback.print_exc() + + +########################## Process Starts Here ########################## + +# Clean +abc = pd.read_excel(clean_path) +abc.rename(columns=combined_mapping, inplace=True) +abc_names = list(abc["Contract Name"].unique()) +abc_dicts = abc.to_dict(orient="records") + +# Input +input_dict = utils.read_input("data_cnc/batch3A") + +# Current Output +output_dirs = list(os.listdir(individual_path)) + +# Run only files that are not in abc, are in input, and are not in output +input_dict = { + k: v + for k, v in input_dict.items() + if k.split(".txt")[0] not in abc_names and k.split(".txt")[0] not in output_dirs +} +print("Input dict to run: ", len(input_dict)) + + +with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor: + if "a" in config.FIELDS and "c" in config.FIELDS: + futures = [ + executor.submit(process_ac_adhoc, item) for item in input_dict.items() + ] + + for future in concurrent.futures.as_completed(futures): + try: + result = future.result() + except Exception as e: + print(f"Error in future: {e}") + traceback.print_exc() + +# for item in input_dict.items(): +# process_ac_adhoc(item) diff --git a/fieldExtraction/scripts/adhoc/cnc_ad_hoc.py b/fieldExtraction/scripts/adhoc/cnc_ad_hoc.py new file mode 100644 index 0000000..41c7892 --- /dev/null +++ b/fieldExtraction/scripts/adhoc/cnc_ad_hoc.py @@ -0,0 +1,584 @@ +import concurrent.futures +import traceback +import csv +import time +import sys +import random + +random.seed(42) +import pandas as pd +import os +import re + +import utils +import config +import claude_funcs +import file_processing +import tracking +import consolidate_output +import top_down_funcs +import valid +import ac_funcs +import preprocess +import prompts +import postprocess +import postprocessing_funcs +import preprocessing_funcs + +# Ad-Hoc runs of sections of the script +""" +Batch 1 and 2 +Fields to Run: +AC +- All Chunked Fields +- All Part 2 Fields +- Postprocess AC + +B +- All B Postprocessing + +750 additional +- Run all ABC +""" +dup_mapping = { + "Payer Name": "PAYER_NAME", + "IRS Name": "IRS_Name", + "NPI NAME": "NPI_NAME", + "Sequestration Reductions, included [Medicare only] (Y/N)": "SEQUESTRATION_REDUCTIONS_IND", + "Page_num": "page_num", + "Lesser of Logic Language, included (Y/N)": "LESSER", + "Reimb. Methodology_short": "SHORT_METHODOLOGY", + "If rate is % of Payer or MCR [STANDARD]": "RATE_STANDARD", + "If rate is % of Payer or MCR [STANDARD]_Short": "RATE_SHORT", + "Flat Fee": "FLAT_FEE_STANDARD", + "CDM Language, included (Y/N)": "CDM_IND", + "CHARGEMASTER": "CONTRACT_CHARGEMASTER_PROTECTION_LANGUAGE", + "EXCLUSIONS": "Exclusions", +} + +combined_mapping = { + "Contract_Name": "Filename", + "Agreement_Name_Contract_Title": "CONTRACT_TITLE", + "Payer_Name": "PAYER_NAME", + "Health_Plan_State": "HEALTH_PLAN_STATE", + "Affiliate_Y_N": "AFFILIATION_CLAUSE_IND", + "Credentialing_Application_Indicator": "CREDENTIALING_APP_IND", + "Term_Clause": "TERM_CLAUSE", + "Contract_Auto_Renewal_Indicator": "CONTRACT_AUTO_RENEWAL_IND", + "Termination_Date": "CONTRACT_TERMINATION_DT", + "Termination_Upon_Notice_Days": "TERMINATION_UPON_NOTICE", + "Termination_With_Cause_Days": "TERMINATION_UPON_CAUSE", + "Amend_Contract_Upon_notice_Flag_Y_N": "AMEND_CONTRACT_NOTICE_IND", + "Timeframe_to_Object_Days": "TIME_TO_OBJECT", + "Assignments_Clause_Y_N": "ASSIGNMENTS_CLAUSE_IND", + "Contract_Effective_Date": "CONTRACT_EFFECTIVE_DT", + "IRS_Num": "PROV_GROUP_TIN", + "PROV_GROUP_NAME": "PROV_GROUP_NAME", + "NPI_10_digits": "PROV_GROUP_NPI", + "NPI_NAME": "NPI_NAME", + "PROV_GROUP_TIN_SIGNATORY": "PROV_GROUP_TIN_SIGNATORY", + "PROV_TIN_OTHER": "PROV_TIN_OTHER", + "PROV_NPI_OTHER": "PROV_NPI_OTHER", + "Notice_to_Provider_Name": "NOTICE_PROVIDER_NAME", + "Notice_to_Provider_Address": "NOTICE_PROVIDER_ADDRESS", + "Sequestration_Language": "SEQUESTRATION_LANGUAGE", + "Sequestration_Reductions_included_Medicare_only_Y_N": "SEQUESTRATION_REDUCTIONS_IND", + "Parent_Agreement_Code": "Parent Agreement Code", + "Pages": "Pages", + "Page_num": "page_num", + "Attachment_Exhibit": "EXHIBIT", + "Line_of_Business": "CONTRACT_LOB", + "Provider_Type": "PROV_TYPE", + "Provider_Type_Level_2": "PROV_TYPE_LEVEL_2", + "IP_OP": "IP_OP", + "Service_Type": "FULL_SERVICE", + "Plan_Type": "CONTRACT_PROGRAM", + "Lesser_of_Logic_Language_included_Y_N": "LESSER", + "Lesser_of_Rate": "LESSER_RATE", + "Reimb_Methodology": "FULL_METHODOLOGY", + "Reimb_Methodology_short": "SHORT_METHODOLOGY", + "If_rate_is_percent_of_Payer_or_MCR_STANDARD": "RATE_STANDARD", + "If_rate_is_percent_of_Payer_or_MCR_STANDARD_Short": "RATE_SHORT", + "Flat_Fee": "FLAT_FEE_STANDARD", + "Default_Term": "DEFAULT_TERM", + "Default_Rate": "DEFAULT_RATE", + "Inclusion_of_essential_RBRVS_Fee_Source_Language_Y_N": 'Inclusion of essential RBRVS "Fee Source" Language (Y/N)', + "CDM_Language_included_Y_N": "CDM_IND", + "CHARGEMASTER": "CHARGEMASTER", + "IP_DSH_IME_UC_included_Y_N": "IP - DSH/IME/UC, included (Y/N)", + "IP_Stoploss_Catastrophic_Threshold": "IP - Stoploss Catastrophic Threshold", + "EXCLUSIONS": "EXCLUSIONS", + "Not_to_Exceed": "NOT_TO_EXCEED", + "Escalator_or_COLA_Y_N": "RATE_ESCALATOR_IND", + "Escalator_I_Eff_Date": "RATE_ESCALATOR_DT", +} + + +AC_DICT = { + "ACCESS_TO_MEDICAL_RECORDS": "Access clause contains information regarding access to medical records. It may be found in section 4.2 of the contract. Extract the Access clause present in the contract. If it is not present, answer N/A.", + "CARVEOUT_VENDORS": "Carve-Out Vendors clause may be found in section 2.9 of the contract. Extract the Carve-Out Vendors clause present in the contract. If it is not present, answer N/A.", + "CLAIMS_EDITING_LANGUAGE": "Claims Editing Language is typically found in the claims section in the beginning parts of the contract. Extract Claims Editing Language present in the contract. If it is not present, answer N/A.", + "CLEAN_CLAIM": 'Extract the "Clean Claim" subsection present in definition section of the contract. If it is not present, answer N/A.', + "CONFLICTS_BETWEEN_CERTAIN_DOCUMENTS_LANGUAGE": "Extract the Conflicts Between Certain Documents clause present in the contract. If it is not present, answer N/A.", + "COST_SETTLEMENT_LANGUAGE": "Cost settlement language may be present in disputes and arbitration subsection of the contract. It usually contains keywords like settlementor cost settlement. Extract the cost settlement language present in the contract. If it is not present, answer N/A.", + "DEEMER_AMENDMENT": "Deemer Amendment clause may be found in section 8.7.2 of the contract. Extract the Deemer Amendment clause present in the contract. If it is not present, answer N/A.", + "DELEGATED_TERMS": "Extract Delegated Terms present in the contract. If it is not present, answer N/A.", + "ECM": 'What is the "ECM" number mentioned in the contract? Only Answer the ECM number, do not provide any context. Do not provide ICM number. If ECM number is not present, answer N/A.', + "EXCLUSIVITY_REQUIREMENT_LANGUAGE": "Extract Exclusivity requirement clause present in the contract. If it is not present, answer N/A.", + "GUARANTEE_OF_PROVIDER_YIELD_LANGUAGE": "Extract Guarantee of Provider Yield Language present in the contract. If it is not present, answer N/A.", + "HCBS_SERVICES": 'Extract HCBS services clause present either in the compensation or exhibit section of the contract. It may contain keywords like "for covered HCBS services, plan shall pay". Do not include definitions of HCBS. If it is not present, answer N/A.', + "INDEMNIFICATION": "Indemnification clause may be found in section 5.2 of the contract. Extract the Indemnification clause present in the contract. If it is not present, answer N/A.", + "INDEPENDENT_REVIEW_LANGUAGE": "Extract Independent Review language present in the contract. If it is not present, answer N/A.", + "INVOICE_PRICING_LANGUAGE": 'Extract table with heading "INVOICED SERVICES" if it is present in compensation schedule of the contract. If it is not present, answer N/A', + "LATE_PAID_CLAIMS_LANGUAGE": "Extract Late Paid Claims language present in the contract. This will be a sentence or paragraph describing late paid claims, interest payments or similar. Here is an example of a correct response: 'Any Clean Claim, as defined in 42 C.F.R. § 422.500, shall be paid within thirty (30) days of receipt by Plan at such address as may be designated by Plan, and Plan shall pay interest on any Clean Claim not paid within thirty (30) days of such receipt by Plan at the rate of interest required by law, or as otherwise set forth in the Provider Manual.'. Here is an example of an incorrect response: 'Any Clean Claim, as defined in 42 C.F.R. 422.500, shall be paid within thirty (30) days of receipt by Health Plan, Payor or (if Provider contracts with Downstream Entities) Provider, as applicable, as designated by Provider or such Downstream Entity, as applicable.'. If no valid answer is present, answer N/A.", + "MEMBER_CONFINEMENT_DAYS_LANGUAGE": "Extract Member Confinement Days Language present in the contract. If it is not present, answer N/A.", + "NATIONAL_AGREEMENT_IND": "Are more than one states covered in the contract? Answer with a Yes or No, do not provide any other context.", + "NETWORK_ACCESS_FEES_LANGUAGE": 'Extract "Network Access Fee" verbiage present in the contract. If it is not present, answer N/A.', + "NONSTANDARD_APPEALS_PROCESS_LANGUAGE": "Extract Nonstandard Appeals Process language present in the contract. If it is not present, answer N/A.", + "PAYMENT_IN_ADVANCE_OF_CLAIMS_SUBMISSION_LANGUAGE": "Extract Payment in Advance of Claims Submission language present in the contract. If it is not present, answer N/A.", + "PAYOR": '"Payor" is usually found in section 1.12 of the contract. Extract the "Payor" subsection present in definition section of the contract. If it is not present, answer N/A.', + "PMPM": "What is the PMPM rate mentioned in the contract? Only return the rate. do not provide any context. If it is not present, answer N/A.", + "PREAUTHORIZATION": "Preauthorization clause may be found in section 2.7 of the contract. Extract the Preauthorization clause present in the contract. If it is not present, answer N/A.", + "PRODUCT_REMOVAL": "Product removal may be found in the products attachment, stating a product is being removed. Extract product removal language present in the contract. If it is not present, answer N/A.", + "PROVIDER_BASED_BILLING_EXCLUSION_LANGUAGE": "Provider-based Billing Exclusion is often listed under the ‘Additional Provisions’ section. Extract the Provider-based Billing Exclusion language mentioned in the contract. If it is not present, answer N/A.", + "RECOVERY_RIGHTS": "Recovery Rights clause may be found in section 3.5 of the contract. Extract the entire Recovery Rights sentence or paragraph present in the contract. If it is not present, answer N/A.", + "TEMPLATE": "Is this contract in standard Centene template containing definition, terms clauses and other standard sections. Answer with a Yes or No, do not provide any other context.", + "ARBITRATION_AND_DISPUTES": "Dispute Resolution clause is usually found in section 6.1 of the contract. Arbitration clause is usually found in section 6.2 of the contract. Extract both the Dispute Resolution and the arbitration clause present in the contract. In case one of them is present, extract that one. If none of them are present, answer N/A.", + "DISPARAGEMENT_PROHIBITION_IND": "Is Disparagement Prohibition clause present in the contract? It may be found in section 2.1 of the contract. Answer with a Yes or No, do not provide any other context.", + "DISPARAGEMENT_PROHIBITION_LANGUAGE": "Disparagement Prohibition clause may be found in section 2.1 of the contract. Extract the Disparagement Prohibition clause present in the contract. If it is not present, answer N/A.", + "ELIGIBILITY_VERIFICATION": "Eligibility Verification or Determination clause may be found in section 2.6 of the contract. Extract the Eligibility Verification or Determination clause present in the contract. If it is not present, answer N/A.", + "INSURANCE_REQUIREMENT": "Extract Insurance requirement mentioned in the contract. If it is not present, answer N/A.", + "PARTICIPATION_IN_PRODUCTS": "Participation in Products clause may be found in section 2.2.2 of the contract. Extract the Participation in Products clause present in the contract. If it is not present, answer N/A.", + "POLICIES_AND_PROCEDURES": "Extract the entire Policies and Procedures sentence or paragraph present in the contract. Do not extract language about conflicts and construction. Do not extract language about PP (Preferred Provider) nor PPG (Preferred Provider Group). If it is not present, answer N/A.", + "REGULATORY_REQUIREMENTS": 'Extract the entire Regulatory Requirements paragraph present in the contract. It usually contains the keyword "Regulatory Requirements". If it is not present, answer N/A.', + "RELATIONSHIP_OF_PARTIES_LANGUAGE": "Extract the entire Relationship of Parties passage present in the contract. If it is not present, answer N/A.", + "TERM_CLAUSE": """Extract the subsection or paragraph labeled Term. It may also be labeled Termination. If there are no such sections, extract the paragraph the most closely resembles this label. If there is still no correct answer, simply return 'N/A'.""", + "LATE_PAID_CLAIMS_LANGUAGE": "Extract Late Paid Claims language present in the contract. This will be a sentence or paragraph describing late paid claims, interest payments or similar. Here is an example of a correct response: 'Any Clean Claim, as defined in 42 C.F.R. § 422.500, shall be paid within thirty (30) days of receipt by Plan at such address as may be designated by Plan, and Plan shall pay interest on any Clean Claim not paid within thirty (30) days of such receipt by Plan at the rate of interest required by law, or as otherwise set forth in the Provider Manual.'. Here is an example of an incorrect response: 'Any Clean Claim, as defined in 42 C.F.R. 422.500, shall be paid within thirty (30) days of receipt by Health Plan, Payor or (if Provider contracts with Downstream Entities) Provider, as applicable, as designated by Provider or such Downstream Entity, as applicable.'. If no valid answer is present, answer N/A.", + "NON_RENEWAL_LANGUAGE": "Extract the section of the contract that describes non-renewal obligations, especially the number of days notice that must be given. Write the full relevant sentence or paragraph. Ensure that the exact term 'non-renewal' is present in the answer. Do NOT return an answer that does not have 'non-renewal'.", + "HCBS_SERVICES": 'Extract HCBS services clause present either in the compensation or exhibit section of the contract. It may contain keywords like "for covered HCBS services, plan shall pay". Do not include definitions of HCBS. If it is not present, answer N/A.', +} + +KEYWORD_MAPPINGS = { + "TERM_CLAUSE": { + "methodology": "hierarchy", + "keywords": ["Term", "term", "duration", "agreement period"], + }, + "LATE_PAID_CLAIMS_LANGUAGE": { + "methodology": "hierarchy", + "keywords": [ + "late paid", + "late payment", + "interest", + "late fee", + "interest shall be paid", + "interest payment", + ], + }, + "NON_RENEWAL_LANGUAGE": { + "methodology": "or", + "keywords": ["not to renew", "non-renewal", "non renewal", "nonrenewal"], + }, + # "NON_RENEWAL_DAYS": {'methodology': 'or', 'keywords': ["not to renew", "non-renewal", "non renewal", "nonrenewal"]}, + "HCBS_SERVICES": { + "methodology": "hierarchy", + "keywords": ["%", "Plan shall pay", "hcbs"], + }, + "POLICIES_AND_PROCEDURES": { + "methodology": "or", + "keywords": ["2.4", "policies", "procedures"], + }, + "REGULATORY_REQUIREMENTS": { + "methodology": "or", + "keywords": ["8.3", "6.4", "regulatory requirements", "regulatory"], + }, + "RECOVERY_RIGHTS": {"methodology": "or", "keywords": ["3.5", "recovery rights"]}, + "RELATIONSHIP_OF_PARTIES_LANGUAGE": { + "methodology": "or", + "keywords": ["8.1", "relationship of parties"], + }, +} + + +def run_ac_prompts(file_object): + ################## INITIATE PROCESSING ################## + filename, contract_text = file_object + print(f"Processing AC for {filename}...") + # print(f"Total contract word count: {len(contract_text.split())}") + + ################## PREPROCESS ################## + text_dict, exhibit_pages, num_pages, ac_chunks = preprocess.preprocess( + contract_text, filename, fields="ac" + ) + # print(f"AC Preprocessing Complete - {filename}") + + ################## RUN FULL CONTEXT PROMPTS ################## + ac_dict = {} + log_data = [] + + # Process 'full_context' fields together + full_context_fields = [field for field in AC_DICT if field not in KEYWORD_MAPPINGS] + full_context_questions = { + field: prompts.AC_DICT.get(field) for field in full_context_fields + } + + if full_context_questions: + full_context_prompt = ac_funcs.create_prompt( + contract_text[0 : min(len(contract_text), 600000)], + question=full_context_questions, + ) + # print(f"High accuracy prompt word count: {len(high_accuracy_prompt.split())}") + try: + full_context_answers = ac_funcs.get_ac_answer( + full_context_prompt, + filename, + fields=list(full_context_questions.keys()), + ) + ac_dict.update(full_context_answers) + + for field, answer in full_context_answers.items(): + log_data.append( + { + "Field": field, + "Prompt": f"Question: {full_context_questions[field]}\n\nContext: {contract_text}", # Include full context + "Context Type": "Full Context (High Accuracy)", + "Response": answer, + } + ) + except Exception as e: + # print(f"Error processing high accuracy fields: {str(e)}") + for field in full_context_questions.keys(): + ac_dict[field] = f"Error: {str(e)}" + log_data.append( + { + "Field": field, + "Prompt": f"Question: {full_context_questions[field]}\n\nContext: {contract_text}", # Include full context + "Context Type": "Full Context (High Accuracy)", + "Response": f"Error: {str(e)}", + } + ) + + ################## RUN CHUNKED PROMPTS ################## + all_fields = set(prompts.AC_DICT.keys()) + chunked_fields = [field for field in KEYWORD_MAPPINGS] + + for field in chunked_fields: + if field in prompts.AC_DICT: + question = prompts.AC_DICT[field] + else: + # print(f"Field {field} not found in prompts. Skipping...") + continue + + # Use chunk if available and different from full context, otherwise use full context + if ( + field in ac_chunks + and ac_chunks[field].strip() + and ac_chunks[field] != contract_text + ): + context = ac_chunks[field] + context_type = "Smart Chunking" + else: + context = contract_text + context_type = "Full Context" + + prompt = prompts.AC_SINGLE_FIELD_TEMPLATE(context, question) + + print( + f"Field {field} prompt word count: {len(prompt.split())} ({context_type})" + ) + + try: + field_answer = claude_funcs.invoke_claude( + prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 8192 + ) + ac_dict[field] = field_answer + + log_data.append( + { + "Field": field, + "Prompt": f"Question: {question}\n\nContext: {context}", # Include full context + "Context Type": context_type, + "Response": field_answer, + } + ) + except Exception as e: + # print(f"Error processing field {field}: {str(e)}") + ac_dict[field] = f"Error: {str(e)}" + log_data.append( + { + "Field": field, + "Prompt": f"Question: {question}\n\nContext: {context}", # Include full context + "Context Type": context_type, + "Response": f"Error: {str(e)}", + } + ) + + ################## AC CONDITIONAL PROMPTS ################## + ## Non-Renewal - Days + nrd_prompt = prompts.AC_SINGLE_FIELD_TEMPLATE( + ac_dict["NON_RENEWAL_LANGUAGE"], prompts.AC_DICT["NON_RENEWAL_DAYS"] + ) + nrd_answer = claude_funcs.invoke_claude( + nrd_prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 8192 + ) + ac_dict["NON_RENEWAL_DAYS"] = nrd_answer + + ################## ENSURE ALL FIELDS ARE PRESENT ################## + ac_dict = {k: v for k, v in ac_dict.items() if k in valid.AC_MAPPING} + for field in all_fields: + if field not in ac_dict: + # print(f"Field not found in results. {field}") + # ac_dict[field] = "N/A" + log_data.append( + { + "Field": field, + "Prompt": "N/A", + "Context Type": "N/A", + "Response": "N/A", + } + ) + + ################## CREATE OUTPUT DIRECTORIES ################## + base_filename = os.path.splitext(filename)[0].strip() + output_dir = os.path.join(config.OUTPUT_DIRECTORY, base_filename) + os.makedirs(output_dir, exist_ok=True) + + ################## POSTPROCESS ################## + ac_dict["Contract Name"] = filename + ac_df = pd.DataFrame([ac_dict]) + ac_df = postprocess.ac_postprocess(ac_df) + + ################## WRITE TO OUTPUT ################## + ac_df.to_csv(f"{output_dir}/ac_output.csv", index=False) + print(f"AC Output written: {filename}") + + ################## WRITE LOG TO CSV ################## + # log_file_path = os.path.join(output_dir, f"{base_filename}_prompt_log.csv") + # with open(log_file_path, 'w', newline='', encoding='utf-8') as log_file: + # fieldnames = ['Field', 'Prompt', 'Context Type', 'Response'] + # writer = csv.DictWriter(log_file, fieldnames=fieldnames) + # writer.writeheader() + # writer.writerows(log_data) + # print(f"Prompt log written to {log_file_path}") + return ac_dict + + +def process_ac_adhoc(item): + filename, contract_text = item + try: + results = run_ac_prompts(item) + if results is not None: + tracking.update_results_csv(item, results) + return item, results + except Exception as e: + print(f"Error processing item {filename}: {e}") + traceback.print_exc() + + +def preprocess_ad_hoc(contract_text, filename, fields=config.FIELDS): + + contract_text = preprocessing_funcs.clean_newlines(contract_text) + contract_text = preprocessing_funcs.clean_law_symbols(contract_text) + + text_dict = preprocessing_funcs.split_text( + contract_text + ) # return a dictionary with keys - page_num (str), values as the page_text + + num_pages = len(text_dict.keys()) - 1 + text_dict = preprocessing_funcs.filter_quick_review(text_dict) + + if fields == "b": + exhibit_pages = top_down_funcs.get_exhibit_pages( + text_dict, filename + ) # All pages with exhibit headers + # text_dict = table_funcs.align_and_format_tables(text_dict, filename) + text_dict = preprocessing_funcs.chunk_consecutive(text_dict, exhibit_pages) + ac_chunks = "" + if fields == "ac": + exhibit_pages = [] + ac_chunks = preprocessing_funcs.smart_chunk_ac(text_dict) + + return text_dict, exhibit_pages, num_pages, ac_chunks + + +def get_add_on(final_dicts, text_dict): + add_on_dict = {} + for td_page in list(set([d["page_num"] for d in final_dicts])): + td_page = str(td_page) + # TD page is the original result top down page, it is not necessarily in the new text_dict. + # Find add_on_page = the key of the new text_dict that points to the correct page of the old text dict + + if td_page in text_dict.keys(): + add_on_page = td_page + else: + add_on_page = [ + str(page) for page in text_dict.keys() if int(page) > int(td_page) + ][0] + + if not add_on_page: + add_on_dict[td_page] = { + "ADD_ON_REIMBURSEMENT_LANGUAGE": "N/A", + "ADD_ON_REIMBURSEMENT_IND": "N", + } + continue + + print(f"TD Page: {td_page} | Add On Page: {add_on_page}") + + ad = {} + + add_on_prompt = f"## Page Start ## {text_dict[str(add_on_page)]} ## Page End ## . Extract the Add On Reimbursement language present in the page above. If there is no Add On language present, answer N/A. Write only the answer, with no additional commentary or explanation." + add_on_answer = claude_funcs.invoke_claude( + add_on_prompt, config.MODEL_ID_CLAUDE35_SONNET, "", 256 + ) + + ad["ADD_ON_REIMBURSEMENT_LANGUAGE"] = add_on_answer + + if "N/A" in ad["ADD_ON_REIMBURSEMENT_LANGUAGE"]: + ad["ADD_ON_REIMBURSEMENT_IND"] = "N" + else: + ad["ADD_ON_REIMBURSEMENT_IND"] = "Y" + add_on_dict[str(td_page)] = ad + + return add_on_dict + + +def process_b_adhoc(item): + filename, contract_text = item + print(f"Processing {filename}") + + merged_dicts = [d for d in abc_dicts if d["Filename"] + ".txt" == filename] + print(len(merged_dicts)) + + if len(merged_dicts) > 0: + + text_dict, exhibit_pages, num_pages, ac_chunks = preprocess_ad_hoc( + contract_text, filename + ) + print(f"Preprocessing {filename}") + + # Exclusions + exclusion_dict = top_down_funcs.get_td_dict( + text_dict, filename, prompts.TOP_DOWN_EXCLUSIONS, valid.EXCLUSION_TERMS + ) + final_dicts = top_down_funcs.add_exclusions(merged_dicts, exclusion_dict) + print(f"Exclusions - {filename}") + + # Medically Necessary + medically_necessary_dict = top_down_funcs.get_td_dict( + text_dict, + filename, + prompts.TOP_DOWN_MEDICALLY_NECESSARY, + valid.VALID_MEDICALLY_NECESSARY, + ) + medically_necessary_dict = postprocessing_funcs.filter_dict( + medically_necessary_dict, + pattern=re.compile( + "|".join( + re.escape(keyword) for keyword in valid.INVALID_MEDICALLY_NECESSARY + ), + re.IGNORECASE, + ), + ) + final_dicts = top_down_funcs.add_medically_necessary( + final_dicts, medically_necessary_dict + ) + print(f"Medically Necessary - {filename}") + + # Add On + add_on_dict = get_add_on(final_dicts, text_dict) + print(f"Add On Results: {add_on_dict}") + print(f"Add On - {filename}") + + for td_page in add_on_dict.keys(): + for d in final_dicts: + if str(d["page_num"]) == str(td_page): + print("Add On Match") + d.update(add_on_dict[str(td_page)]) + + # print(len(final_dicts)) + + output_df = pd.DataFrame(final_dicts) + print(f"B finished -{filename} - output {output_df.shape}") + print(output_df.columns) + + base_filename = os.path.splitext(filename)[0].strip() + output_dir = os.path.join(config.OUTPUT_DIRECTORY, base_filename) + os.makedirs(output_dir, exist_ok=True) + output_df.to_csv(f"{output_dir}/b_output.csv") + + +########################## Process Starts Here ########################## +individual_path = "output_individual/cnc_batch1_b" +clean_path = "reference_files/CNC-Batch 1 Output_File1.xlsx" +run_path = "reference_files/Batch 1_082024.xlsx" +dup_path = "reference_files/Batch 1 Duplicates_10092024.xlsx" + +# Run List +run_list = pd.read_excel(run_path) +run_list = list(run_list[run_list.Status == "1A"]["Filename"]) +print(f"Run List Length: {len(run_list)}") + +# Input +input_dict = utils.read_input() +print(f"Input Dict Length: {len(input_dict)}") + +# Current Output +output_dirs = list(os.listdir(individual_path)) +print(f"Current Outputs: {len(output_dirs)}") + +# Need to Run +input_dict = { + filename: page_text + for filename, page_text in input_dict.items() + if filename.split(".txt")[0] in run_list +} # and filename.split('.txt')[0] not in config.OUTPUT_DIRECTORY} +print(f"Need to Run: {len(input_dict)}") + +# Clean ABC +abc = pd.read_excel(clean_path) +abc.rename(columns=combined_mapping, inplace=True) +abc_names = list(abc["Filename"].unique()) +abc_dicts = abc.to_dict(orient="records") +print(f"ABC Clean File Length (B): {len(abc_names)}") + +# # ABC Dups +abc_dups = pd.read_excel(dup_path) +abc_dups.rename(columns={v: k for k, v in valid.B_MAPPING.items()}, inplace=True) +abc_dups.rename(columns={v: k for k, v in valid.AC_MAPPING.items()}, inplace=True) +abc_dups.rename(columns=dup_mapping, inplace=True) +abc_dup_names = list(abc_dups["Filename"].unique()) +print(f"ABC Dup File Length (B): {len(abc_dup_names)}") + +# # Concat ABC with Dups +abc = pd.concat([abc, abc_dups], axis=0, ignore_index=True) +abc_names = list(abc["Filename"].unique()) +abc_dicts = abc.to_dict(orient="records") +print(f"Final ABC File Length (B+Dups): {len(abc_names)}") + +# Run only files that are in abc, are in input, and are not in output +input_dict = { + k: v for k, v in input_dict.items() if k.split(".txt")[0] in abc_dup_names +} +input_dict = { + k: v + for k, v in input_dict.items() + if k + == "04-3122933-Dentsply IH, Inc. dba Wellspect Healthcare-ICMProviderAgreement_72694(2).txt" +} +print("Input dict to run: ", len(input_dict)) + +# with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor: +# if 'a' in config.FIELDS and 'c' in config.FIELDS: +# futures = [executor.submit(process_ac_adhoc, item) for item in input_dict.items()] +# if 'b' in config.FIELDS: +# futures = [executor.submit(process_b_adhoc, item) for item in input_dict.items()] + +# for future in concurrent.futures.as_completed(futures): +# try: +# result = future.result() +# except Exception as e: +# print(f"Error in future: {e}") +# traceback.print_exc() + + +for item in input_dict.items(): + process_b_adhoc(item) diff --git a/fieldExtraction/scripts/adhoc/cnc_ad_hoc_batch3.py b/fieldExtraction/scripts/adhoc/cnc_ad_hoc_batch3.py new file mode 100644 index 0000000..45d7db2 --- /dev/null +++ b/fieldExtraction/scripts/adhoc/cnc_ad_hoc_batch3.py @@ -0,0 +1,591 @@ +import concurrent.futures +import traceback +import csv +import time +import sys +import random + +random.seed(42) +import pandas as pd +import os +import re + +import utils +import config +import claude_funcs +import file_processing +import tracking +import consolidate_output +import top_down_funcs +import valid +import ac_funcs +import preprocess +import prompts +import postprocess +import postprocessing_funcs +import preprocessing_funcs + +input_path = "data_cnc/batch3" +ac2_output_path = "output_individual/cnc_3_ac2" +b_output_path = "output_individual/cnc_3_b" +clean_path = "reference_files/CNC-Batch 3 Output_File1.xlsx" +run_path = "reference_files/Batch 3_091124.xlsx" + + +# Ad-Hoc runs of sections of the script +""" +Batch 1 and 2 +Fields to Run: +AC +- All Chunked Fields +- All Part 2 Fields +- Postprocess AC + +B +- All B Postprocessing + +750 additional +- Run all ABC +""" +dup_mapping = { + "Payer Name": "PAYER_NAME", + "IRS Name": "IRS_Name", + "NPI NAME": "NPI_NAME", + "Sequestration Reductions, included [Medicare only] (Y/N)": "SEQUESTRATION_REDUCTIONS_IND", + "Page_num": "page_num", + "Lesser of Logic Language, included (Y/N)": "LESSER", + "Reimb. Methodology_short": "SHORT_METHODOLOGY", + "If rate is % of Payer or MCR [STANDARD]": "RATE_STANDARD", + "If rate is % of Payer or MCR [STANDARD]_Short": "RATE_SHORT", + "Flat Fee": "FLAT_FEE_STANDARD", + "CDM Language, included (Y/N)": "CDM_IND", + "CHARGEMASTER": "CONTRACT_CHARGEMASTER_PROTECTION_LANGUAGE", + "EXCLUSIONS": "Exclusions", +} + +combined_mapping = { + "Contract_Name": "Filename", + "Agreement_Name_Contract_Title": "CONTRACT_TITLE", + "Payer_Name": "PAYER_NAME", + "Health_Plan_State": "HEALTH_PLAN_STATE", + "Affiliate_Y_N": "AFFILIATION_CLAUSE_IND", + "Credentialing_Application_Indicator": "CREDENTIALING_APP_IND", + "Term_Clause": "TERM_CLAUSE", + "Contract_Auto_Renewal_Indicator": "CONTRACT_AUTO_RENEWAL_IND", + "Termination_Date": "CONTRACT_TERMINATION_DT", + "Termination_Upon_Notice_Days": "TERMINATION_UPON_NOTICE", + "Termination_With_Cause_Days": "TERMINATION_UPON_CAUSE", + "Amend_Contract_Upon_notice_Flag_Y_N": "AMEND_CONTRACT_NOTICE_IND", + "Timeframe_to_Object_Days": "TIME_TO_OBJECT", + "Assignments_Clause_Y_N": "ASSIGNMENTS_CLAUSE_IND", + "Contract_Effective_Date": "CONTRACT_EFFECTIVE_DT", + "IRS_Num": "PROV_GROUP_TIN", + "PROV_GROUP_NAME": "PROV_GROUP_NAME", + "NPI_10_digits": "PROV_GROUP_NPI", + "NPI_NAME": "NPI_NAME", + "PROV_GROUP_TIN_SIGNATORY": "PROV_GROUP_TIN_SIGNATORY", + "PROV_TIN_OTHER": "PROV_TIN_OTHER", + "PROV_NPI_OTHER": "PROV_NPI_OTHER", + "Notice_to_Provider_Name": "NOTICE_PROVIDER_NAME", + "Notice_to_Provider_Address": "NOTICE_PROVIDER_ADDRESS", + "Sequestration_Language": "SEQUESTRATION_LANGUAGE", + "Sequestration Reductions, included [Medicare only] (Y/N)": "SEQUESTRATION_REDUCTIONS_IND", + "Parent_Agreement_Code": "Parent Agreement Code", + "Pages": "Pages", + "Page_num": "page_num", + "Attachment_Exhibit": "EXHIBIT", + "Line_of_Business": "CONTRACT_LOB", + "Provider_Type": "PROV_TYPE", + "Provider_Type_Level_2": "PROV_TYPE_LEVEL_2", + "IP_OP": "IP_OP", + "Service_Type": "FULL_SERVICE", + "Plan_Type": "CONTRACT_PROGRAM", + "Lesser_of_Logic_Language_included_Y_N": "LESSER", + "Lesser_of_Rate": "LESSER_RATE", + "Reimb_Methodology": "FULL_METHODOLOGY", + "Reimb_Methodology_short": "SHORT_METHODOLOGY", + "If rate is % of Payer or MCR [STANDARD]": "RATE_STANDARD", + "If rate is % of Payer or MCR [STANDARD]_Short": "RATE_SHORT", + "Flat Fee": "FLAT_FEE_STANDARD", + "Default_Term": "DEFAULT_TERM", + "Default_Rate": "DEFAULT_RATE", + "IP - DSH/IME/UC, included (Y/N)": 'Inclusion of essential RBRVS "Fee Source" Language (Y/N)', + "CDM_Language_included_Y_N": "CDM_IND", + "CHARGEMASTER": "CHARGEMASTER", + "IP_DSH_IME_UC_included_Y_N": "IP - DSH/IME/UC, included (Y/N)", + "IP_Stoploss_Catastrophic_Threshold": "IP - Stoploss Catastrophic Threshold", + "EXCLUSIONS": "EXCLUSIONS", + "Not_to_Exceed": "NOT_TO_EXCEED", + "Escalator_or_COLA_Y_N": "RATE_ESCALATOR_IND", + "Escalator_I_Eff_Date": "RATE_ESCALATOR_DT", +} + + +AC_DICT = { + "ACCESS_TO_MEDICAL_RECORDS": "Access clause contains information regarding access to medical records. It may be found in section 4.2 of the contract. Extract the Access clause present in the contract. If it is not present, answer N/A.", + "CARVEOUT_VENDORS": "Carve-Out Vendors clause may be found in section 2.9 of the contract. Extract the Carve-Out Vendors clause present in the contract. If it is not present, answer N/A.", + "CLAIMS_EDITING_LANGUAGE": "Claims Editing Language is typically found in the claims section in the beginning parts of the contract. Extract Claims Editing Language present in the contract. If it is not present, answer N/A.", + "CLEAN_CLAIM": 'Extract the "Clean Claim" subsection present in definition section of the contract. If it is not present, answer N/A.', + "CONFLICTS_BETWEEN_CERTAIN_DOCUMENTS_LANGUAGE": "Extract the Conflicts Between Certain Documents clause present in the contract. If it is not present, answer N/A.", + "COST_SETTLEMENT_LANGUAGE": "Cost settlement language may be present in disputes and arbitration subsection of the contract. It usually contains keywords like settlementor cost settlement. Extract the cost settlement language present in the contract. If it is not present, answer N/A.", + "DEEMER_AMENDMENT": "Deemer Amendment clause may be found in section 8.7.2 of the contract. Extract the Deemer Amendment clause present in the contract. If it is not present, answer N/A.", + "DELEGATED_TERMS": "Extract Delegated Terms present in the contract. If it is not present, answer N/A.", + "ECM": 'What is the "ECM" number mentioned in the contract? Only Answer the ECM number, do not provide any context. Do not provide ICM number. If ECM number is not present, answer N/A.', + "EXCLUSIVITY_REQUIREMENT_LANGUAGE": "Extract Exclusivity requirement clause present in the contract. If it is not present, answer N/A.", + "GUARANTEE_OF_PROVIDER_YIELD_LANGUAGE": "Extract Guarantee of Provider Yield Language present in the contract. If it is not present, answer N/A.", + "HCBS_SERVICES": 'Extract HCBS services clause present either in the compensation or exhibit section of the contract. It may contain keywords like "for covered HCBS services, plan shall pay". Do not include definitions of HCBS. If it is not present, answer N/A.', + "INDEMNIFICATION": "Indemnification clause may be found in section 5.2 of the contract. Extract the Indemnification clause present in the contract. If it is not present, answer N/A.", + "INDEPENDENT_REVIEW_LANGUAGE": "Extract Independent Review language present in the contract. If it is not present, answer N/A.", + "INVOICE_PRICING_LANGUAGE": 'Extract table with heading "INVOICED SERVICES" if it is present in compensation schedule of the contract. If it is not present, answer N/A', + "LATE_PAID_CLAIMS_LANGUAGE": "Extract Late Paid Claims language present in the contract. This will be a sentence or paragraph describing late paid claims, interest payments or similar. Here is an example of a correct response: 'Any Clean Claim, as defined in 42 C.F.R. § 422.500, shall be paid within thirty (30) days of receipt by Plan at such address as may be designated by Plan, and Plan shall pay interest on any Clean Claim not paid within thirty (30) days of such receipt by Plan at the rate of interest required by law, or as otherwise set forth in the Provider Manual.'. Here is an example of an incorrect response: 'Any Clean Claim, as defined in 42 C.F.R. 422.500, shall be paid within thirty (30) days of receipt by Health Plan, Payor or (if Provider contracts with Downstream Entities) Provider, as applicable, as designated by Provider or such Downstream Entity, as applicable.'. If no valid answer is present, answer N/A.", + "MEMBER_CONFINEMENT_DAYS_LANGUAGE": "Extract Member Confinement Days Language present in the contract. If it is not present, answer N/A.", + "NATIONAL_AGREEMENT_IND": "Are more than one states covered in the contract? Answer with a Yes or No, do not provide any other context.", + "NETWORK_ACCESS_FEES_LANGUAGE": 'Extract "Network Access Fee" verbiage present in the contract. If it is not present, answer N/A.', + "NONSTANDARD_APPEALS_PROCESS_LANGUAGE": "Extract Nonstandard Appeals Process language present in the contract. If it is not present, answer N/A.", + "PAYMENT_IN_ADVANCE_OF_CLAIMS_SUBMISSION_LANGUAGE": "Extract Payment in Advance of Claims Submission language present in the contract. If it is not present, answer N/A.", + "PAYOR": '"Payor" is usually found in section 1.12 of the contract. Extract the "Payor" subsection present in definition section of the contract. If it is not present, answer N/A.', + "PMPM": "What is the PMPM rate mentioned in the contract? Only return the rate. do not provide any context. If it is not present, answer N/A.", + "PREAUTHORIZATION": "Preauthorization clause may be found in section 2.7 of the contract. Extract the Preauthorization clause present in the contract. If it is not present, answer N/A.", + "PRODUCT_REMOVAL": "Product removal may be found in the products attachment, stating a product is being removed. Extract product removal language present in the contract. If it is not present, answer N/A.", + "PROVIDER_BASED_BILLING_EXCLUSION_LANGUAGE": "Provider-based Billing Exclusion is often listed under the ‘Additional Provisions’ section. Extract the Provider-based Billing Exclusion language mentioned in the contract. If it is not present, answer N/A.", + "RECOVERY_RIGHTS": "Recovery Rights clause may be found in section 3.5 of the contract. Extract the entire Recovery Rights sentence or paragraph present in the contract. If it is not present, answer N/A.", + "TEMPLATE": "Is this contract in standard Centene template containing definition, terms clauses and other standard sections. Answer with a Yes or No, do not provide any other context.", + "ARBITRATION_AND_DISPUTES": "Dispute Resolution clause is usually found in section 6.1 of the contract. Arbitration clause is usually found in section 6.2 of the contract. Extract both the Dispute Resolution and the arbitration clause present in the contract. In case one of them is present, extract that one. If none of them are present, answer N/A.", + "DISPARAGEMENT_PROHIBITION_IND": "Is Disparagement Prohibition clause present in the contract? It may be found in section 2.1 of the contract. Answer with a Yes or No, do not provide any other context.", + "DISPARAGEMENT_PROHIBITION_LANGUAGE": "Disparagement Prohibition clause may be found in section 2.1 of the contract. Extract the Disparagement Prohibition clause present in the contract. If it is not present, answer N/A.", + "ELIGIBILITY_VERIFICATION": "Eligibility Verification or Determination clause may be found in section 2.6 of the contract. Extract the Eligibility Verification or Determination clause present in the contract. If it is not present, answer N/A.", + "INSURANCE_REQUIREMENT": "Extract Insurance requirement mentioned in the contract. If it is not present, answer N/A.", + "PARTICIPATION_IN_PRODUCTS": "Participation in Products clause may be found in section 2.2.2 of the contract. Extract the Participation in Products clause present in the contract. If it is not present, answer N/A.", + "POLICIES_AND_PROCEDURES": "Extract the entire Policies and Procedures sentence or paragraph present in the contract. Do not extract language about conflicts and construction. Do not extract language about PP (Preferred Provider) nor PPG (Preferred Provider Group). If it is not present, answer N/A.", + "REGULATORY_REQUIREMENTS": 'Extract the entire Regulatory Requirements paragraph present in the contract. It usually contains the keyword "Regulatory Requirements". If it is not present, answer N/A.', + "RELATIONSHIP_OF_PARTIES_LANGUAGE": "Extract the entire Relationship of Parties passage present in the contract. If it is not present, answer N/A.", + "TERM_CLAUSE": """Extract the subsection or paragraph labeled Term. It may also be labeled Termination. If there are no such sections, extract the paragraph the most closely resembles this label. If there is still no correct answer, simply return 'N/A'.""", + "LATE_PAID_CLAIMS_LANGUAGE": "Extract Late Paid Claims language present in the contract. This will be a sentence or paragraph describing late paid claims, interest payments or similar. Here is an example of a correct response: 'Any Clean Claim, as defined in 42 C.F.R. § 422.500, shall be paid within thirty (30) days of receipt by Plan at such address as may be designated by Plan, and Plan shall pay interest on any Clean Claim not paid within thirty (30) days of such receipt by Plan at the rate of interest required by law, or as otherwise set forth in the Provider Manual.'. Here is an example of an incorrect response: 'Any Clean Claim, as defined in 42 C.F.R. 422.500, shall be paid within thirty (30) days of receipt by Health Plan, Payor or (if Provider contracts with Downstream Entities) Provider, as applicable, as designated by Provider or such Downstream Entity, as applicable.'. If no valid answer is present, answer N/A.", + "NON_RENEWAL_LANGUAGE": "Extract the section of the contract that describes non-renewal obligations, especially the number of days notice that must be given. Write the full relevant sentence or paragraph. Ensure that the exact term 'non-renewal' is present in the answer. Do NOT return an answer that does not have 'non-renewal'.", + "HCBS_SERVICES": 'Extract HCBS services clause present either in the compensation or exhibit section of the contract. It may contain keywords like "for covered HCBS services, plan shall pay". Do not include definitions of HCBS. If it is not present, answer N/A.', +} + +KEYWORD_MAPPINGS = { + "TERM_CLAUSE": { + "methodology": "hierarchy", + "keywords": ["Term", "term", "duration", "agreement period"], + }, + "LATE_PAID_CLAIMS_LANGUAGE": { + "methodology": "hierarchy", + "keywords": [ + "late paid", + "late payment", + "interest", + "late fee", + "interest shall be paid", + "interest payment", + ], + }, + "NON_RENEWAL_LANGUAGE": { + "methodology": "or", + "keywords": ["not to renew", "non-renewal", "non renewal", "nonrenewal"], + }, + # "NON_RENEWAL_DAYS": {'methodology': 'or', 'keywords': ["not to renew", "non-renewal", "non renewal", "nonrenewal"]}, + "HCBS_SERVICES": { + "methodology": "hierarchy", + "keywords": ["%", "Plan shall pay", "hcbs"], + }, + "POLICIES_AND_PROCEDURES": { + "methodology": "or", + "keywords": ["2.4", "policies", "procedures"], + }, + "REGULATORY_REQUIREMENTS": { + "methodology": "or", + "keywords": ["8.3", "6.4", "regulatory requirements", "regulatory"], + }, + "RECOVERY_RIGHTS": {"methodology": "or", "keywords": ["3.5", "recovery rights"]}, + "RELATIONSHIP_OF_PARTIES_LANGUAGE": { + "methodology": "or", + "keywords": ["8.1", "relationship of parties"], + }, +} + + +def run_ac2_prompts(file_object): + ################## INITIATE PROCESSING ################## + filename, contract_text = file_object + print(f"Processing AC for {filename}...") + # print(f"Total contract word count: {len(contract_text.split())}") + + ################## PREPROCESS ################## + text_dict, exhibit_pages, num_pages, ac_chunks = preprocess.preprocess( + contract_text, filename, fields="ac" + ) + # print(f"AC Preprocessing Complete - {filename}") + + ################## RUN FULL CONTEXT PROMPTS ################## + ac_dict = {} + log_data = [] + + # Process 'full_context' fields together + full_context_fields = [field for field in AC_DICT if field not in KEYWORD_MAPPINGS] + full_context_questions = { + field: prompts.AC_DICT.get(field) for field in full_context_fields + } + + if full_context_questions: + full_context_prompt = ac_funcs.create_prompt( + contract_text[0 : min(len(contract_text), 600000)], + question=full_context_questions, + ) + # print(f"High accuracy prompt word count: {len(high_accuracy_prompt.split())}") + try: + full_context_answers = ac_funcs.get_ac_answer( + full_context_prompt, + filename, + fields=list(full_context_questions.keys()), + ) + ac_dict.update(full_context_answers) + + for field, answer in full_context_answers.items(): + log_data.append( + { + "Field": field, + "Prompt": f"Question: {full_context_questions[field]}\n\nContext: {contract_text}", # Include full context + "Context Type": "Full Context (High Accuracy)", + "Response": answer, + } + ) + except Exception as e: + # print(f"Error processing high accuracy fields: {str(e)}") + for field in full_context_questions.keys(): + ac_dict[field] = f"Error: {str(e)}" + log_data.append( + { + "Field": field, + "Prompt": f"Question: {full_context_questions[field]}\n\nContext: {contract_text}", # Include full context + "Context Type": "Full Context (High Accuracy)", + "Response": f"Error: {str(e)}", + } + ) + + ################## RUN CHUNKED PROMPTS ################## + all_fields = set(prompts.AC_DICT.keys()) + chunked_fields = [field for field in KEYWORD_MAPPINGS] + + for field in chunked_fields: + if field in prompts.AC_DICT: + question = prompts.AC_DICT[field] + else: + # print(f"Field {field} not found in prompts. Skipping...") + continue + + # Use chunk if available and different from full context, otherwise use full context + if ( + field in ac_chunks + and ac_chunks[field].strip() + and ac_chunks[field] != contract_text + ): + context = ac_chunks[field] + context_type = "Smart Chunking" + else: + context = contract_text + context_type = "Full Context" + + prompt = prompts.AC_SINGLE_FIELD_TEMPLATE(context, question) + + # print(f"Field {field} prompt word count: {len(prompt.split())} ({context_type})") + + try: + field_answer = claude_funcs.invoke_claude( + prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 8192 + ) + print(field, field_answer) + ac_dict[field] = field_answer + + log_data.append( + { + "Field": field, + "Prompt": f"Question: {question}\n\nContext: {context}", # Include full context + "Context Type": context_type, + "Response": field_answer, + } + ) + except Exception as e: + # print(f"Error processing field {field}: {str(e)}") + ac_dict[field] = f"Error: {str(e)}" + log_data.append( + { + "Field": field, + "Prompt": f"Question: {question}\n\nContext: {context}", # Include full context + "Context Type": context_type, + "Response": f"Error: {str(e)}", + } + ) + + ################## AC CONDITIONAL PROMPTS ################## + ## Non-Renewal - Days + nrd_prompt = prompts.AC_SINGLE_FIELD_TEMPLATE( + ac_dict["NON_RENEWAL_LANGUAGE"], prompts.AC_DICT["NON_RENEWAL_DAYS"] + ) + nrd_answer = claude_funcs.invoke_claude( + nrd_prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 8192 + ) + ac_dict["NON_RENEWAL_DAYS"] = nrd_answer + + ################## ENSURE ALL FIELDS ARE PRESENT ################## + ac_dict = {k: v for k, v in ac_dict.items() if k in valid.AC_MAPPING} + for field in all_fields: + if field not in ac_dict: + # print(f"Field not found in results. {field}") + # ac_dict[field] = "N/A" + log_data.append( + { + "Field": field, + "Prompt": "N/A", + "Context Type": "N/A", + "Response": "N/A", + } + ) + + ################## CREATE OUTPUT DIRECTORIES ################## + base_filename = os.path.splitext(filename)[0].strip() + output_dir = os.path.join(ac2_output_path, base_filename) + os.makedirs(output_dir, exist_ok=True) + + ################## POSTPROCESS ################## + ac_dict["Contract Name"] = filename + ac_df = pd.DataFrame([ac_dict]) + ac_df = postprocess.ac_postprocess(ac_df) + + ################## WRITE TO OUTPUT ################## + ac_df.to_csv(f"{output_dir}/ac_output.csv", index=False) + print(f"AC Output written: {filename}") + + ################## WRITE LOG TO CSV ################## + # log_file_path = os.path.join(output_dir, f"{base_filename}_prompt_log.csv") + # with open(log_file_path, 'w', newline='', encoding='utf-8') as log_file: + # fieldnames = ['Field', 'Prompt', 'Context Type', 'Response'] + # writer = csv.DictWriter(log_file, fieldnames=fieldnames) + # writer.writeheader() + # writer.writerows(log_data) + # print(f"Prompt log written to {log_file_path}") + return ac_dict + + +def process_ac2_adhoc(item): + filename, contract_text = item + try: + results = run_ac2_prompts(item) + if results is not None: + tracking.update_results_csv(item, results) + return item, results + except Exception as e: + print(f"Error processing item {filename}: {e}") + traceback.print_exc() + + +def preprocess_ad_hoc(contract_text, filename, fields=config.FIELDS): + + contract_text = preprocessing_funcs.clean_newlines(contract_text) + contract_text = preprocessing_funcs.clean_law_symbols(contract_text) + + text_dict = preprocessing_funcs.split_text( + contract_text + ) # return a dictionary with keys - page_num (str), values as the page_text + + num_pages = len(text_dict.keys()) - 1 + text_dict = preprocessing_funcs.filter_quick_review(text_dict) + + if fields == "b": + exhibit_pages = top_down_funcs.get_exhibit_pages( + text_dict, filename + ) # All pages with exhibit headers + # text_dict = table_funcs.align_and_format_tables(text_dict, filename) + text_dict = preprocessing_funcs.chunk_consecutive(text_dict, exhibit_pages) + ac_chunks = "" + if fields == "ac": + exhibit_pages = [] + ac_chunks = preprocessing_funcs.smart_chunk_ac(text_dict) + + return text_dict, exhibit_pages, num_pages, ac_chunks + + +def get_add_on(final_dicts, text_dict): + add_on_dict = {} + for td_page in list(set([d["page_num"] for d in final_dicts])): + td_page = str(td_page).split(".")[0] + print(f"TD Page : {td_page}") + # TD page is the original result top down page, it is not necessarily in the new text_dict. + # Find add_on_page = the key of the new text_dict that points to the correct page of the old text dict + + if str(td_page) in list(text_dict.keys()): + add_on_page = td_page + else: + add_on_page = [ + str(page) for page in text_dict.keys() if float(page) > float(td_page) + ] + print(add_on_page) + + if not add_on_page: + add_on_dict[td_page] = { + "ADD_ON_REIMBURSEMENT_LANGUAGE": "N/A", + "ADD_ON_REIMBURSEMENT_IND": "N", + } + continue + elif not isinstance(add_on_page, str): + add_on_page = add_on_page[0] + + ad = {} + + add_on_prompt = f"## Page Start ## {text_dict[str(add_on_page)]} ## Page End ## . Extract the Add On Reimbursement language present in the page above. If there is no Add On language present, answer N/A. Write only the answer, with no additional commentary or explanation." + add_on_answer = claude_funcs.invoke_claude( + add_on_prompt, config.MODEL_ID_CLAUDE35_SONNET, "", 256 + ) + + ad["ADD_ON_REIMBURSEMENT_LANGUAGE"] = add_on_answer + + if "N/A" in ad["ADD_ON_REIMBURSEMENT_LANGUAGE"]: + ad["ADD_ON_REIMBURSEMENT_IND"] = "N" + else: + ad["ADD_ON_REIMBURSEMENT_IND"] = "Y" + add_on_dict[str(td_page)] = ad + + return add_on_dict + + +def process_b_adhoc(item): + filename, contract_text = item + print(f"Processing {filename}") + + merged_dicts = [d for d in abc_dicts if d["Filename"] + ".txt" == filename] + print(len(merged_dicts)) + + if len(merged_dicts) > 0: + + text_dict, exhibit_pages, num_pages, ac_chunks = preprocess_ad_hoc( + contract_text, filename + ) + print(f"Preprocessing {filename}") + + # Exclusions + exclusion_dict = top_down_funcs.get_td_dict( + text_dict, filename, prompts.TOP_DOWN_EXCLUSIONS, valid.EXCLUSION_TERMS + ) + final_dicts = top_down_funcs.add_exclusions(merged_dicts, exclusion_dict) + print(f"Exclusions - {filename}") + + # Medically Necessary + medically_necessary_dict = top_down_funcs.get_td_dict( + text_dict, + filename, + prompts.TOP_DOWN_MEDICALLY_NECESSARY, + valid.VALID_MEDICALLY_NECESSARY, + ) + medically_necessary_dict = postprocessing_funcs.filter_dict( + medically_necessary_dict, + pattern=re.compile( + "|".join( + re.escape(keyword) for keyword in valid.INVALID_MEDICALLY_NECESSARY + ), + re.IGNORECASE, + ), + ) + final_dicts = top_down_funcs.add_medically_necessary( + final_dicts, medically_necessary_dict + ) + print(f"Medically Necessary - {filename}") + + # Add On + add_on_dict = get_add_on(final_dicts, text_dict) + print(f"Add On Results: {add_on_dict}") + print(f"Add On - {filename}") + + for td_page in add_on_dict.keys(): + for d in final_dicts: + if str(d["page_num"]) == str(td_page): + print("Add On Match") + d.update(add_on_dict[str(td_page)]) + + # print(len(final_dicts)) + + output_df = pd.DataFrame(final_dicts) + print(f"B finished -{filename} - output {output_df.shape}") + print(output_df.columns) + time.sleep(3) + base_filename = os.path.splitext(filename)[0].strip() + output_dir = os.path.join(b_output_path, base_filename) + os.makedirs(output_dir, exist_ok=True) + output_df.to_csv(f"{output_dir}/b_output.csv") + + +########################## Process Starts Here ########################## + +# Run List +run_list = pd.read_excel(run_path) +run_list = run_list[run_list["Batch"] == "3A"]["Filename"].unique().tolist() +print(f"Run List: {len(run_list)} files") + +# Input Dict +input_dict_3a = utils.read_input(local_path=input_path + "A") +input_dict_3b = utils.read_input(local_path=input_path + "B") + +input_dict = {**input_dict_3a, **input_dict_3b} +print(f"Input Dict: {len(input_dict)} files") + +# Filter Input Dict +input_dict = { + filename: contract_text + for filename, contract_text in input_dict.items() + if filename.split(".txt")[0] in run_list +} +print(f"Input Dict In Run List: {len(input_dict)} files") + +# Left to run +if "a" in config.FIELDS: + input_dict_ac2 = { + filename: contract_text + for filename, contract_text in input_dict.items() + if filename.split(".txt")[0] not in os.listdir(ac2_output_path) + } + print(f"Input Dict Left to Run (AC2): {len(input_dict_ac2)} files") + +# ABC Dicts +abc_df = pd.read_excel(clean_path) +abc_df.rename(columns={v: k for k, v in valid.B_MAPPING.items()}, inplace=True) +abc_df.rename(columns={v: k for k, v in valid.AC_MAPPING.items()}, inplace=True) +abc_df.rename(columns=combined_mapping, inplace=True) +abc_df.drop(["State from Ben", "Unnamed: 54"], axis=1, inplace=True) +abc_names = list(abc_df["Filename"].unique()) +abc_dicts = abc_df.to_dict(orient="records") +print(f"Clean abc : {len(abc_dicts)}") + +# Left to run +if "b" in config.FIELDS: + input_dict_b = { + filename: contract_text + for filename, contract_text in input_dict.items() + if filename.split(".txt")[0] in abc_df["Filename"].unique() + } + input_dict_b = { + filename: contract_text + for filename, contract_text in input_dict_b.items() + if filename.split(".txt")[0] not in os.listdir(b_output_path) + } + print(f"Input Dict Left to Run (B): {len(input_dict_b)} files") + +with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor: + if "a" in config.FIELDS and "c" in config.FIELDS: + futures = [ + executor.submit(process_ac2_adhoc, item) for item in input_dict_ac2.items() + ] + if "b" in config.FIELDS: + futures = [ + executor.submit(process_b_adhoc, item) for item in input_dict_b.items() + ] + + for future in concurrent.futures.as_completed(futures): + try: + result = future.result() + except Exception as e: + print(f"Error in future: {e}") + traceback.print_exc() + + +# for item in input_dict_b.items(): +# process_b_adhoc(item) diff --git a/fieldExtraction/scripts/adhoc/cnc_adhoc_stitching_1.py b/fieldExtraction/scripts/adhoc/cnc_adhoc_stitching_1.py new file mode 100644 index 0000000..e0e60d7 --- /dev/null +++ b/fieldExtraction/scripts/adhoc/cnc_adhoc_stitching_1.py @@ -0,0 +1,226 @@ +import os +import pandas as pd +import logging +import concurrent.futures + +logging.getLogger().setLevel("ERROR") + +import warnings + +warnings.filterwarnings("ignore") + +import postprocessing_funcs +import valid +import config + + +def b_postprocess(filename, combined_df, pages): + if combined_df.shape[0] > 0: + # Metadata fields + # combined_df['Contract Name'] = filename + # combined_df['Parent Agreement Code'] = postprocessing_funcs.get_parent_agreement_code(filename) + # combined_df['Pages'] = pages + + # Add Single Code, Multiple Rate + combined_df = postprocessing_funcs.add_scmr(combined_df) + # Filter Add Ons + combined_df = postprocessing_funcs.filter_add_ons(combined_df) + # Clean MSR + combined_df = postprocessing_funcs.clean_msr_lesser(combined_df) + # Clean Default + combined_df = postprocessing_funcs.clean_default_term(combined_df) + # Clean Prov 2 + combined_df = postprocessing_funcs.clean_prov_2(combined_df) + # Clean Lesser Of Rate + combined_df = postprocessing_funcs.clean_lesser_rate(combined_df) + # Clean LOB + combined_df = postprocessing_funcs.clean_lob(combined_df, filename) + # Rename and reorder + combined_df.rename(columns=valid.B_MAPPING, inplace=True) + column_order = [ + col for col in valid.B_MAPPING.values() if col in combined_df.columns + ] + final_df = combined_df[column_order] + return final_df + else: + return combined_df + + +def read_individual(filepath, filename): + df_list = [] + for file in os.listdir(filepath): + full_path = os.path.join(filepath, file) + df = pd.read_csv(os.path.join(full_path, filename)) + df_list.append(df) + return pd.concat(df_list, ignore_index=True) + + +all_column_mappings = valid.B_MAPPING.copy() +all_column_mappings.update(valid.AC_MAPPING) + +# For Original ABC - +# For Duplicates - +# For Non-B: All AC1 and AC2 Fields + +# TERM_CLAUSE, CONTRACT_EFFECTIVE_DATE, PROV_GROUP_TIN, PROV_GROUP_NPI, DEFAULT_TERM + [ALL PART 2 FIELDS] + + +# # ############################## READ - Clean ABC Part 1 + 3 New B Fields ############################## +abc = read_individual("output_individual/cnc_batch1_b", "b_output.csv") + +abc.drop(["Exclusions"], axis=1, inplace=True) +abc = abc[[col for col in abc.columns if "Unnamed" not in col]] +abc.rename(columns={v: k for k, v in all_column_mappings.items()}, inplace=True) +abc.rename( + columns={ + "If rate is % of Payer or MCR [STANDARD]": "RATE_STANDARD", + "If rate is % of Payer or MCR [STANDARD]_Short": "RATE_SHORT", + "Lesser of Logic Language, included (Y/N)": "LESSER", + "Flat Fee": "FLAT_FEE_STANDARD", + "Reimb. Methodology_short": "SHORT_METHODOLOGY", + }, + inplace=True, +) +abc["Filename"] = abc["Filename"].str.replace(".txt", "", regex=False) + +print("Clean ABC Part 1 + 3 New B Fields: ") +print(f"Filenames: {len(abc.Filename.unique())}", abc.Filename[0]) +print(list(abc.columns)) +print( + f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}" +) +print(abc.shape) + +# ############################## READ - AC2 For All Contracts ############################## +ac2 = read_individual("output_individual/cnc_batch1_ac", "ac_output.csv") +ac2 = ac2[[col for col in ac2.columns if "Unnamed" not in col]] +ac2.rename(columns={v: k for k, v in all_column_mappings.items()}, inplace=True) +ac2["Filename"] = ac2["Filename"].str.replace(".txt", "", regex=False) +print("AC2 For All Contracts: ") +print(f"Filenames: {len(ac2.Filename.unique())}", ac2.Filename[0]) +print(list(ac2.columns)) +print( + f"Invalid Cols: {[col for col in ac2.columns if col not in all_column_mappings.keys()]}" +) +print(ac2.shape) + +# ############################## READ - New AC1 (3 fields) For All Contracts ############################## +ac1 = read_individual("output_individual/cnc_batch1_ac1", "ac_output.csv") +# ac1.columns = ['Filename', 'Contract Effective Date', 'IRS #', 'NPI (10-digits)'] +ac1.rename(columns={v: k for k, v in all_column_mappings.items()}, inplace=True) +ac1["Filename"] = ac1["Filename"].str.replace(".txt", "", regex=False) +print("New AC1 For All Contracts: ") +print(f"Filenames: {len(ac1.Filename.unique())}", ac1.Filename[0]) +print(list(ac1.columns)) +print(ac1.shape) + +# ############################## READ - Original AC1 For All Contracts ############################## +ac = pd.read_csv("reference_files/CNC-1-AC.csv") +ac.rename(columns={v: k for k, v in all_column_mappings.items()}, inplace=True) +ac.rename( + columns={ + "Evergreen, Fixed or Hard Term": "CONTRACT_AUTO_RENEWAL_IND", + "Sequestration Reductions, included [Medicare only] (Y/N)": "SEQUESTRATION_REDUCTIONS_IND", + }, + inplace=True, +) +print("Original AC1 For Non-B Contracts: ") +print(f"Filenames: {len(ac.Filename.unique())}", ac.Filename[0]) +print(list(ac.columns)) +print( + f"Invalid Cols: {[col for col in ac.columns if col not in all_column_mappings.keys()]}" +) +print(ac.shape) + +############################## MERGE - Original AC1 + 3 New AC1 Fields ############################## +ac.drop( + ["CONTRACT_EFFECTIVE_DT", "PROV_GROUP_TIN", "PROV_GROUP_NPI"], axis=1, inplace=True +) +ac = pd.merge(ac, ac1, on="Filename", how="right") # .reset_index(drop=True) +# ac.rename(columns=all_column_mappings, inplace=True) +ac = ac[[col for col in ac.columns if "Unnamed" not in col]] +print("Final AC1 (with 3 replaced fields)") +print(f"Filenames: {len(ac.Filename.unique())}") +print(list(ac.columns)) +print(ac.shape) + +############################## FILTER - Only AC1 that are not already in Clean ABC ############################## +ac_new_only = ac[~ac["Filename"].isin(abc.Filename.unique())] +print("AC New Only (with 3 replaced fields)") +print(f"Filenames: {len(ac_new_only.Filename.unique())}") +print(list(ac_new_only.columns)) +print(ac_new_only.shape) + +############################## ADD - Only New AC1 to abc ############################## +abc = abc.reset_index(drop=True) +ac = ac.reset_index(drop=True) +abc = pd.concat([abc, ac_new_only], axis=0) +print("Updated ABC (With non-B AC1 Fields added)") +print(f"Filenames: {len(abc.Filename.unique())}") +print(list(abc.columns)) +print( + f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}" +) +print(abc.shape) + +############################## MERGE - AC2 to abc ############################## +abc = abc[ + [ + col + for col in abc.columns + if col not in [c for c in ac2.columns if c != "Filename"] + ] +] + +# Merge ac2 to abc, keeping the columns from ac2 when there is a conflict. Except for CREDENTIALING_APP_IND +# abc = pd.merge(abc, ac2, on='Filename', how='left') +abc = pd.merge(abc, ac2, on="Filename", how="left", suffixes=("", "_from_ac2")) +for column in abc.columns: + if column.endswith("_from_ac2"): + orig_column = column[:-10] # Remove the '_from_ac2' suffix + if orig_column != "CREDENTIALING_APP_IND": + abc[orig_column] = abc[column] + abc.drop(column, axis=1, inplace=True) + +print("Updated ABC (with Non-B AC1 fields, new AC2 field, new B fields)") +print(f"Filenames: {len(abc.Filename.unique())}") +print(list(abc.columns)) +print( + f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}" +) +print(abc.shape) +# abc.rename(columns=all_column_mappings, inplace=True) +# abc.to_csv('output_consolidated/CNC-Batch1-ABC-BeforePostprocessing.csv') + + +############################## POSTPROCESS - Final Postprocess Step ############################## +def postprocess_ad_hoc(combined_df): + # B Steps + print(combined_df.shape) + combined_df = postprocessing_funcs.add_scmr(combined_df) + print(combined_df.shape) + # combined_df = postprocessing_funcs.filter_add_ons(combined_df) # Removing rows + # print(combined_df.shape) + combined_df = postprocessing_funcs.clean_lesser_rate(combined_df) + print(combined_df.shape) + combined_df = postprocessing_funcs.clean_default_term(combined_df) + print(combined_df.shape) + + # combined_df = postprocessing_funcs.clean_msr_lesser(combined_df) # Modify for ad hoc + # combined_df = postprocessing_funcs.clean_prov_2(combined_df) # Modify for ad hoc + # combined_df = postprocessing_funcs.clean_lob(combined_df, "") + + # AC Steps + combined_df = postprocessing_funcs.clean_ac_fields(combined_df) + combined_df = combined_df.apply(postprocessing_funcs.derive_indicators, axis=1) + print(combined_df.shape) + return combined_df + + +abc_final = postprocess_ad_hoc(abc) +abc_final.rename(columns=all_column_mappings, inplace=True) +abc_final.to_csv("output_consolidated/CNC-1-RERUN-DRAFT4.csv") +print("Final ABC") +print(f"Filenames: {len(abc.Filename.unique())}") +print(list(abc_final.columns)) +print(abc_final.shape) diff --git a/fieldExtraction/scripts/adhoc/cnc_adhoc_stitching_2.py b/fieldExtraction/scripts/adhoc/cnc_adhoc_stitching_2.py new file mode 100644 index 0000000..7e22af5 --- /dev/null +++ b/fieldExtraction/scripts/adhoc/cnc_adhoc_stitching_2.py @@ -0,0 +1,267 @@ +import os +import pandas as pd +import logging +import concurrent.futures + +logging.getLogger().setLevel("ERROR") + +import warnings + +warnings.filterwarnings("ignore") + +import postprocessing_funcs +import valid +import config + + +def b_postprocess(filename, combined_df, pages): + if combined_df.shape[0] > 0: + # Metadata fields + # combined_df['Contract Name'] = filename + # combined_df['Parent Agreement Code'] = postprocessing_funcs.get_parent_agreement_code(filename) + # combined_df['Pages'] = pages + + # Add Single Code, Multiple Rate + combined_df = postprocessing_funcs.add_scmr(combined_df) + # Filter Add Ons + combined_df = postprocessing_funcs.filter_add_ons(combined_df) + # Clean MSR + combined_df = postprocessing_funcs.clean_msr_lesser(combined_df) + # Clean Default + combined_df = postprocessing_funcs.clean_default_term(combined_df) + # Clean Prov 2 + combined_df = postprocessing_funcs.clean_prov_2(combined_df) + # Clean Lesser Of Rate + combined_df = postprocessing_funcs.clean_lesser_rate(combined_df) + # Clean LOB + combined_df = postprocessing_funcs.clean_lob(combined_df, filename) + # Rename and reorder + combined_df.rename(columns=valid.B_MAPPING, inplace=True) + column_order = [ + col for col in valid.B_MAPPING.values() if col in combined_df.columns + ] + final_df = combined_df[column_order] + return final_df + else: + return combined_df + + +def read_individual(filepath, filename): + df_list = [] + for file in os.listdir(filepath): + try: + full_path = os.path.join(filepath, file) + df = pd.read_csv(os.path.join(full_path, filename)) + df_list.append(df) + except: + pass + return pd.concat(df_list, ignore_index=True) + + +all_column_mappings = valid.B_MAPPING.copy() +all_column_mappings.update(valid.AC_MAPPING) + + +# ############################## READ - Clean AC1 + B + 3 New B ############################## +abc = read_individual("output_individual/cnc_batch2_b", "b_output.csv") +abc.drop(["Exclusions"], axis=1, inplace=True) +abc = abc[[col for col in abc.columns if "Unnamed" not in col]] +abc.rename(columns={v: k for k, v in all_column_mappings.items()}, inplace=True) +abc.rename( + columns={ + "If rate is % of Payer or MCR [STANDARD]": "RATE_STANDARD", + "If rate is % of Payer or MCR [STANDARD]_Short": "RATE_SHORT", + "Lesser of Logic Language, included (Y/N)": "LESSER", + "Flat Fee": "FLAT_FEE_STANDARD", + "Reimb. Methodology_short": "SHORT_METHODOLOGY", + }, + inplace=True, +) +abc["Filename"] = abc["Filename"].str.replace(".txt", "", regex=False) +print("\n\nABC + New B: ") +print(abc.shape) +print(list(abc.columns)) +print( + f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}" +) + + +# ############################## READ - New AC2 + Smart Chunked ############################## +ac2 = read_individual("output_individual/cnc_batch2_ac", "ac_output.csv") +ac2 = ac2[[col for col in ac2.columns if "Unnamed" not in col]] +ac2.rename(columns={v: k for k, v in all_column_mappings.items()}, inplace=True) +ac2.drop(["TERM_CLAUSE"], axis=1, inplace=True) +ac2["Filename"] = ac2["Filename"].str.replace(".txt", "", regex=False) +print("\n\nNew AC2: ") +print(ac2.shape) +print(list(ac2.columns)) +print( + f"Invalid Cols: {[col for col in ac2.columns if col not in all_column_mappings.keys()]}" +) + +# ############################## READ - New AC1 (3 fields) ############################## +ac1 = read_individual("output_individual/cnc_batch2_ac1", "ac_output.csv") +# ac1.columns = ['Filename', 'Contract Effective Date', 'IRS #', 'NPI (10-digits)'] +ac1.rename(columns={v: k for k, v in all_column_mappings.items()}, inplace=True) +print("\n\nNew AC1: ") +print(ac1.shape) +print(list(ac1.columns)) + +# ############################## READ - Original AC1 ############################## +ac = pd.read_excel("reference_files/CNC-2.0-AC.xlsx") +ac.rename(columns={v: k for k, v in all_column_mappings.items()}, inplace=True) +ac.rename( + columns={ + "Evergreen, Fixed or Hard Term": "CONTRACT_AUTO_RENEWAL_IND", + "Sequestration Reductions, included [Medicare only] (Y/N)": "SEQUESTRATION_REDUCTIONS_IND", + }, + inplace=True, +) +ac["Filename"] = ac["Filename"].str.replace(".txt", "", regex=False) +ac = ac[ac["Filename"].isin(abc["Filename"])] + +print("\n\nOriginal AC1: ") +print(ac.shape) +print(list(ac.columns)) +print( + f"Invalid Cols: {[col for col in ac.columns if col not in all_column_mappings.keys()]}" +) + + +############################## READ - Missing ABC ############################## +base_path = "output_individual/cnc_batch2_b_missing" +ac_frames = [] +b_frames = [] +for subdir in os.listdir(base_path): + subdir_path = os.path.join(base_path, subdir) + + # Construct file paths + ac_file_path = os.path.join(subdir_path, "test_batch-AC.csv") + b_file_path = os.path.join(subdir_path, "test_batch-B.csv") + + # Check if files exist and then read them + if os.path.exists(ac_file_path): + ac_df = pd.read_csv(ac_file_path) + ac_df.rename(columns={v: k for k, v in valid.AC_MAPPING.items()}, inplace=True) + ac_frames.append(ac_df) + if os.path.exists(b_file_path): + b_df = pd.read_csv(b_file_path) + b_df.rename(columns={v: k for k, v in valid.B_MAPPING.items()}, inplace=True) + b_frames.append(b_df) + +ac_missing = pd.concat(ac_frames, ignore_index=True) +b_missing = pd.concat(b_frames, ignore_index=True) +b_missing = b_missing[[col for col in b_missing.columns if ".1" not in col]] + +abc_missing = pd.merge(ac_missing, b_missing, on="Filename", how="left") +abc_missing.rename(columns={v: k for k, v in all_column_mappings.items()}, inplace=True) +abc_missing["Filename"] = abc_missing["Filename"].str.replace(".txt", "", regex=False) + +print("\n\nABC missing: ") +print(abc_missing.shape) +print(list(abc_missing.columns)) +print( + f"Invalid Cols: {[col for col in abc_missing.columns if col not in all_column_mappings.keys()]}" +) + + +# ############################## MERGE - Original AC + New AC1 ############################## +ac.drop( + ["CONTRACT_EFFECTIVE_DT", "PROV_GROUP_TIN", "PROV_GROUP_NPI"], axis=1, inplace=True +) +ac = pd.merge(ac, ac1, on="Filename", how="right") # .reset_index(drop=True) +# ac.rename(columns=all_column_mappings, inplace=True) +ac = ac[[col for col in ac.columns if "Unnamed" not in col]] +print("\n\nFinal AC1 (with 3 replaced fields)") +print(ac.shape) +print(list(ac.columns)) + +############################## ADD - New AC to abc ############################## +# Only ones that don't have B fields +abc = abc.reset_index(drop=True) +ac = ac.reset_index(drop=True) +abc = pd.concat([abc, ac], axis=0) +print("\n\nUpdated ABC (With non-B AC1 Fields added)") +print(abc.shape) +print(list(abc.columns)) +print( + f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}" +) + + +############################## MERGE - AC2 to abc ############################## +abc = abc[ + [ + col + for col in abc.columns + if col not in [c for c in ac2.columns if c != "Filename"] + ] +] +abc = pd.merge(abc, ac2, on="Filename", how="left") +print("\n\nUpdated ABC (with Non-B AC1 fields, new AC2 field, new B fields)") +print(list(abc.columns)) +print( + f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}" +) +print(abc.shape) + +print(abc.TERM_CLAUSE.value_counts()) +# abc.rename(columns=all_column_mappings, inplace=True) +# abc.to_csv('output_consolidated/CNC-Batch1-ABC-BeforePostprocessing.csv') + +############################## ADD - ABC missing to abc ############################## +abc = pd.concat([abc, abc_missing], axis=0) +print( + "\n\nFinal ABC (with Non-B AC1 fields, new AC2 fields, new B fields, missing ABC files)" +) +print(abc.shape) +print(list(abc.columns)) +print( + f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}" +) + + +############################## POSTPROCESS - Final Postprocess Step ############################## +def postprocess_ad_hoc(combined_df): + # B Steps + print(combined_df.shape) + combined_df = postprocessing_funcs.add_scmr(combined_df) + print(combined_df.shape) + # combined_df = postprocessing_funcs.filter_add_ons(combined_df) # Removing rows + # print(combined_df.shape) + combined_df = postprocessing_funcs.clean_lesser_rate(combined_df) + print(combined_df.shape) + combined_df = postprocessing_funcs.clean_default_term(combined_df) + print(combined_df.shape) + + # combined_df = postprocessing_funcs.clean_msr_lesser(combined_df) # Modify for ad hoc + combined_df = postprocessing_funcs.clean_prov_2(combined_df) # Modify for ad hoc + # combined_df = postprocessing_funcs.clean_lob(combined_df, "") + + # AC Steps + combined_df = postprocessing_funcs.clean_ac_fields(combined_df) + combined_df = combined_df.apply(postprocessing_funcs.derive_indicators, axis=1) + print(combined_df.shape) + return combined_df + + +abc_final = postprocess_ad_hoc(abc) +abc_final.rename(columns=all_column_mappings, inplace=True) +print("\n\nFinal ABC") +print(abc_final.shape) +print(f"Unique files: {abc_final['Contract Name'].unique()}") +print(list(abc_final.columns)) +print( + f"Invalid Cols: {[col for col in abc_final.columns if col not in valid.ABC_COLUMNS]}" +) + +abc_final = abc_final[[col for col in valid.ABC_COLUMNS if col in abc_final.columns]] +print("\n\nFinal ABC") +print(abc_final.shape) +print(list(abc_final.columns)) +print( + f"Invalid Cols: {[col for col in abc_final.columns if col not in valid.ABC_COLUMNS]}" +) + + +abc_final.to_csv("output_consolidated/CNC-2-RERUN-DRAFT4.csv") diff --git a/fieldExtraction/scripts/adhoc/cnc_adhoc_stitching_3.py b/fieldExtraction/scripts/adhoc/cnc_adhoc_stitching_3.py new file mode 100644 index 0000000..78df07f --- /dev/null +++ b/fieldExtraction/scripts/adhoc/cnc_adhoc_stitching_3.py @@ -0,0 +1,228 @@ +import os +import pandas as pd + +pd.set_option("display.max_columns", None) +pd.set_option("display.max_rows", None) +import logging +import concurrent.futures + +logging.getLogger().setLevel("ERROR") +from concurrent.futures import ThreadPoolExecutor + +import warnings + +warnings.filterwarnings("ignore") + +import postprocessing_funcs +import valid +import config +import prompts + + +def b_postprocess(filename, combined_df, pages): + if combined_df.shape[0] > 0: + # Metadata fields + # combined_df['Contract Name'] = filename + # combined_df['Parent Agreement Code'] = postprocessing_funcs.get_parent_agreement_code(filename) + # combined_df['Pages'] = pages + + # Add Single Code, Multiple Rate + combined_df = postprocessing_funcs.add_scmr(combined_df) + # Filter Add Ons + combined_df = postprocessing_funcs.filter_add_ons(combined_df) + # Clean MSR + combined_df = postprocessing_funcs.clean_msr_lesser(combined_df) + # Clean Default + combined_df = postprocessing_funcs.clean_default_term(combined_df) + # Clean Prov 2 + combined_df = postprocessing_funcs.clean_prov_2(combined_df) + # Clean Lesser Of Rate + combined_df = postprocessing_funcs.clean_lesser_rate(combined_df) + # Clean LOB + combined_df = postprocessing_funcs.clean_lob(combined_df, filename) + # Rename and reorder + combined_df.rename(columns=valid.B_MAPPING, inplace=True) + column_order = [ + col for col in valid.B_MAPPING.values() if col in combined_df.columns + ] + final_df = combined_df[column_order] + return final_df + else: + return combined_df + + +def read_csv_file(full_path, filename): + """Helper function to read CSV file.""" + try: + df = pd.read_csv(os.path.join(full_path, filename)) + return df + except Exception as e: + # print(f"Failed to read {os.path.join(full_path, filename)}: {e}") + return None + + +def read_individual(filepath, filename): + df_list = [] + # Use ThreadPoolExecutor to handle file reading in parallel + with ThreadPoolExecutor(max_workers=1000) as executor: + # Create a list of future tasks + futures = [] + for file in os.listdir(filepath): + full_path = os.path.join(filepath, file) + # Schedule the read_csv_file function to be executed and store the future object + futures.append(executor.submit(read_csv_file, full_path, filename)) + + # As each future completes, get the result (a DataFrame or None) + for future in futures: + result = future.result() + if result is not None: + df_list.append(result) + + # Concatenate all the DataFrames in the list + if df_list: + return pd.concat(df_list, ignore_index=True) + + +all_column_mappings = valid.B_MAPPING.copy() +all_column_mappings.update(valid.AC_MAPPING) + +abc_path = "output_individual/cnc_3_b" +ac1_original_path = "output_consolidated/CNC-3.0-A-AC.csv" +ac1_new_path = "output_individual/cnc_3_ac1" +ac2_path = "output_individual/cnc_3_ac2" + +# Read New AC2 +# ac2 = read_individual(ac2_path, 'ac_output.csv') +# ac2.rename(columns={v : k for k, v in all_column_mappings.items()}, inplace=True) +# ac2.to_csv('reference_files/ac2_new.csv') +ac2 = pd.read_csv("reference_files/ac2_new.csv") +ac2 = ac2[[col for col in ac2.columns if "Unnamed" not in col]] +ac2["Filename"] = ac2["Filename"].str.replace(".txt", "", regex=False) +print("\n\nNew AC2") +print(f"Unique Filenames: {len(ac2['Filename'].unique())}") +print(ac2.shape) +print(list(ac2.columns)) + +# Read Original ABC + 3 new B +# abc = read_individual(abc_path, 'b_output.csv') +# abc = abc[[col for col in abc.columns if 'Unnamed' not in col]] +# abc.drop(['MEDICAL_NECESSITY_LANGUAGE_IND', 'MEDICAL_NECESSITY_LANGUAGE'], axis=1, inplace=True) +# abc.rename(columns={ +# 'Lesser of Logic Language, included (Y/N)' : 'LESSER', +# 'Reimb. Methodology_short' : 'SHORT_METHODOLOGY', +# }, inplace=True) +# abc['Filename'] = abc['Filename'].str.replace('.txt', '', regex=False) +# overlapping_columns = [col for col in abc.columns if col in ac2.columns and col != 'Filename'] +# abc.drop(overlapping_columns, axis=1, inplace=True) +# abc.to_csv('reference_files/abc_original.csv') +abc = pd.read_csv("reference_files/abc_original.csv") +abc = abc[[col for col in abc.columns if "Unnamed" not in col]] +print("\n\nOriginal A1B + 3 New B") +print(f"Unique Filenames: {len(abc.Filename.unique())}") +print(abc.shape) +print(list(abc.columns)) + +# Read Original AC1 +# ac1 = pd.read_csv(ac1_original_path) +# ac1.rename(columns={v : k for k, v in all_column_mappings.items()}, inplace=True) +# ac1.rename(columns={'Sequestration Reductions, included [Medicare only] (Y/N)' : "SEQUESTRATION_REDUCTIONS_IND", +# 'Evergreen, Fixed or Hard Term' : "CONTRACT_AUTO_RENEWAL_IND"}, inplace=True) +# ac1['Filename'] = ac1['Filename'].str.replace('.txt', '', regex=False) +# ac1 = ac1[~ac1['Filename'].isin(abc['Filename'])] +# overlapping_columns = [col for col in ac1.columns if col in ac2.columns and col != 'Filename'] +# ac1.drop(overlapping_columns, axis=1, inplace=True) +# ac1.to_csv('reference_files/ac1_original.csv') +ac1 = pd.read_csv("reference_files/ac1_original.csv") +ac1 = ac1[[col for col in ac1.columns if "Unnamed" not in col]] +print("\n\nOriginal AC1") +print(f"Unique Filenames: {len(ac1['Filename'].unique())}") +print(ac1.shape) +print(list(ac1.columns)) + +# Read 3 new AC1 +# ac1_new = read_individual(ac1_new_path, 'ac_output.csv') +# ac1_new.rename(columns={v : k for k, v in all_column_mappings.items()}, inplace=True) +# ac1_new['Filename'] = ac1_new['Filename'].str.replace('.txt', '', regex=False) +# ac1_new.to_csv('reference_files/ac1_new.csv') +ac1_new = pd.read_csv("reference_files/ac1_new.csv") +ac1_new = ac1_new[[col for col in ac1_new.columns if "Unnamed" not in col]] +print("\n\n3 New AC1") +print(f"Unique Filenames: {len(ac1_new.Filename.unique())}") +print(ac1_new.shape) +print(list(ac1_new.columns)) + +# Merge 3 new AC1 to original AC1 +ac1.drop( + ["CONTRACT_EFFECTIVE_DT", "PROV_GROUP_TIN", "PROV_GROUP_NPI"], axis=1, inplace=True +) +ac1_full = pd.merge(ac1, ac1_new, on="Filename", how="left") +print("\n\nAC 1 Full") +print(f"Unique Filenames: {len(ac1_full.Filename.unique())}") +print(ac1_full.shape) +print(list(ac1_full.columns)) + +# Append Full AC1 to Original ABC +abc = pd.concat([abc, ac1_full], axis=0) +abc = abc[ + [ + col + for col in abc.columns + if col not in [c for c in ac2.columns if c != "Filename"] + ] +] +print("\n\nFull ABC") +print(f"Unique Filenames: {len(abc['Filename'].unique())}") +print(abc.shape) +print(list(abc.columns)) + +# Merge AC2 +# Filenames in abc not in ac2 +print(abc[~abc["Filename"].isin(ac2["Filename"])]["Filename"].unique()) + +# Filenames in ac2 not in abc +print(ac2[~ac2["Filename"].isin(abc["Filename"])]["Filename"].unique()) + +abc = pd.merge(abc, ac2, on="Filename", how="left") +print("\n\nABC Full Final (before postprocessing)") +print(f"Unique Filenames: {len(abc.Filename.unique())}") +print(abc.shape) +print(list(abc.columns)) + + +#### Postprocess #### +def postprocess_ad_hoc(combined_df): + # B Steps + combined_df = postprocessing_funcs.add_scmr(combined_df) + print(combined_df.shape) + # combined_df = postprocessing_funcs.filter_add_ons(combined_df) # Removing rows + # print(combined_df.shape) + combined_df = postprocessing_funcs.clean_lesser_rate(combined_df) + print(combined_df.shape) + combined_df = postprocessing_funcs.clean_default_term(combined_df) + print(combined_df.shape) + + # combined_df = postprocessing_funcs.clean_msr_lesser(combined_df) # Modify for ad hoc + # combined_df = postprocessing_funcs.clean_prov_2(combined_df) # Modify for ad hoc + # combined_df = postprocessing_funcs.clean_lob(combined_df, "") + + # AC Steps + combined_df = postprocessing_funcs.clean_ac_fields(combined_df) + combined_df = combined_df.apply(postprocessing_funcs.derive_indicators, axis=1) + print(combined_df.shape) + return combined_df + + +abc_final = postprocess_ad_hoc(abc) +abc_final.rename(columns=all_column_mappings, inplace=True) +print("ABC Final (after postprocessing)") +print(f"Unique Filenames: {len(abc_final['Contract Name'].unique())}") +print(abc_final.shape) +print(list(abc_final.columns)) + +abc_final = abc_final[[col for col in valid.ABC_COLUMNS if col in abc_final.columns]] +print("ABC Full Final (after column renaming)") +print(f"Unique Filenames: {len(abc_final['Contract Name'].unique())}") +print(abc_final.shape) +print(list(abc_final.columns)) + +abc_final.to_csv("output_consolidated/CNC-3-RERUN-DRAFT6.csv") diff --git a/fieldExtraction/scripts/adhoc/cnc_prov2_effdate.py b/fieldExtraction/scripts/adhoc/cnc_prov2_effdate.py new file mode 100644 index 0000000..21fd0b1 --- /dev/null +++ b/fieldExtraction/scripts/adhoc/cnc_prov2_effdate.py @@ -0,0 +1,168 @@ +import pandas as pd + +pd.set_option("display.max_columns", None) +pd.set_option("display.max_rows", None) +import numpy as np +from concurrent.futures import ThreadPoolExecutor +import os +import time + +import utils +import postprocessing_funcs +import claude_funcs +import config +from utils import is_empty +import re +import valid + + +def clean_prov_2(df): + valid_types = valid.select_valid_prov_2(df["Provider Type"]) + target_rows = df[df["Provider Type - Level 2"].apply(is_empty)] + + def find_exact_match(text): + if pd.isna(text) or text == "": + return None + + words = re.findall(r"\b[\w/]+(?:[-\s][\w/]+)*\b", text) + for i in range(len(words)): + for j in range(i + 1, len(words) + 1): + phrase = " ".join(words[i:j]) + if phrase in valid_types: # Case-sensitive matching + return phrase + return None + + for index, row in target_rows.iterrows(): + match = None + + if not is_empty(row["Service Type"]): + match = find_exact_match(str(row["Service Type"])) + if match: + df.at[index, "Provider Type - Level 2"] = match + continue + + if not is_empty(row["Attachment/Exhibit"]): + match = find_exact_match(str(row["Attachment/Exhibit"])) + if match: + df.at[index, "Provider Type - Level 2"] = match + continue + + exhibit_rows = df[df["Attachment/Exhibit"] == row["Attachment/Exhibit"]] + if not exhibit_rows.empty: + for _, exhibit_row in exhibit_rows.iterrows(): + if not is_empty(exhibit_row["Provider Type - Level 2"]): + match = find_exact_match( + str(exhibit_row["Provider Type - Level 2"]) + ) + if match: + df.at[index, "Provider Type - Level 2"] = match + break + elif not is_empty(exhibit_row["Service Type"]): + match = find_exact_match(str(exhibit_row["Service Type"])) + if match: + df.at[index, "Provider Type - Level 2"] = match + break + + # Final check to ensure no invalid values were assigned + invalid_assignments = df[ + (~df["Provider Type - Level 2"].isin(valid_types)) + & (~df["Provider Type - Level 2"].apply(is_empty)) + ] + if not invalid_assignments.empty: + df.loc[invalid_assignments.index, "Provider Type - Level 2"] = "" + + return df + + +def get_new_effective_date(unique_contract_names): + new_dates = {} + for contract_name in unique_contract_names: + if contract_name + ".txt" in input_dict.keys(): + contract_text = input_dict[contract_name + ".txt"] + + try: + prompt = f"""### Contract Start ### {contract_text} ### Contract End ### + Above is a contract. What is the contract effective date mentioned in any of the following locations: the signatory section, the preamble of the agreement, or the start of the amendment? Look for phrases such as /'This amendment is effective/'. + If there is no clear effective date, return the date from the signature page. + Return the date converted to YYYY-MM-DD format, with no other commentary or explanation. + """ + date_answer = claude_funcs.invoke_claude( + prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 128 + ) + print(date_answer) + new_dates[contract_name] = date_answer + except: + prompt = f"""### Contract Start ### {contract_text[0:200000]} ### Contract End ### + Above is a contract. What is the contract effective date mentioned in any of the following locations: the signatory section, the preamble of the agreement, or the start of the amendment? Look for phrases such as /'This amendment is effective/'. + If there is no clear effective date, return the date from the signature page. + Return the date converted to YYYY-MM-DD format, with no other commentary or explanation. + """ + date_answer = claude_funcs.invoke_claude( + prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 128 + ) + new_dates[contract_name] = date_answer + return new_dates + + +#################### Process Starts Here ################### +abc = pd.read_csv("output_consolidated/CNC-3-RERUN-DRAFT6.csv") +# null_counts = abc.isnull().sum() +# print(null_counts) +# quit() + +input_dict = utils.read_input("data_cnc/batch3A") + +# Clean Prov 2 +abc_grouped = abc.groupby("Contract Name") +abc_clean = pd.concat([clean_prov_2(group) for name, group in abc_grouped]) + +# Effective Date +contains_meridian = abc_clean["PAYER NAME"].str.contains( + "meridian", case=False, na=False +) + +# Use your utility function to identify empty dates +empty_dates = abc_clean["Contract Effective Date"].apply(utils.is_empty) + +# Combine filters to find the relevant 'Contract Names' +relevant_contracts = abc_clean[contains_meridian & empty_dates][ + "Contract Name" +].unique() + +# Get new effective dates for these contracts +new_effective_dates = get_new_effective_date(relevant_contracts) + +# Apply the new effective dates to the DataFrame +for contract_name, new_date in new_effective_dates.items(): + # Find rows with this 'Contract Name' where dates need replacing + condition = ( + (abc_clean["Contract Name"] == contract_name) & contains_meridian & empty_dates + ) + abc_clean.loc[condition, "Contract Effective Date"] = new_date + + +abc_clean.to_csv("output_consolidated/CNC-3-RERUN-DRAFT7.csv") + +print("ABC Full Final (after column renaming)") +print(f"Unique Filenames: {len(abc_clean['Contract Name'].unique())}") +print(abc_clean.shape) +print(list(abc_clean.columns)) + + +# abc = pd.read_excel('output_consolidated/CNC-3-RERUN-DRAFT6.csv') +# print(abc.shape) +# print(len(abc['Contract Effective Date'].unique())) + +# date_mapping = pd.read_csv('output_consolidated/CNC-1-RERUN-DRAFT5.csv') +# unique_date_mapping = date_mapping.drop_duplicates(subset='Contract Name', keep='first') +# contract_dates_dict = dict(zip(unique_date_mapping['Contract Name'], unique_date_mapping['Contract Effective Date'])) +# print(contract_dates_dict) + +# abc_postprocessed = clean_prov_2(abc) +# abc_postprocessed['Contract Effective Date'] = abc_postprocessed['Contract Name'].map(contract_dates_dict) + +# print(abc_postprocessed.shape) +# print(len(abc_postprocessed['Contract Effective Date'].unique())) +# print(list(abc_postprocessed['Contract Effective Date'].unique())) + +# abc_postprocessed.to_csv('output_consolidated/CNC-1-RERUN-DRAFT7.csv') diff --git a/fieldExtraction/scripts/adhoc/combine_ac2_and_b.py b/fieldExtraction/scripts/adhoc/combine_ac2_and_b.py new file mode 100644 index 0000000..31d673d --- /dev/null +++ b/fieldExtraction/scripts/adhoc/combine_ac2_and_b.py @@ -0,0 +1,214 @@ +import pandas as pd +import os + +import valid +from postprocess import ac_postprocess +import postprocessing_funcs +import logging + +logging.getLogger().setLevel("ERROR") + +import warnings + +warnings.filterwarnings("ignore") + +""" +This script consolidates a&c fields. Its data sources are `reference_files/CNC-Batch [1/2] Output_File1.xlsx` +as well as .csv files from `output_individual/cnc_batch[1/2]_ac/[name of provider]/ac_output.csv. +""" + + +def b_postprocess(combined_df, filename): + if combined_df.shape[0] > 0: + # Add Single Code, Multiple Rate + combined_df = postprocessing_funcs.add_scmr(combined_df) + + # Filter Add Ons - commented out bc there's no "ADD_ON_REIMBURSEMENT_LANGUAGE" in the Bs we get" + try: + combined_df = postprocessing_funcs.filter_add_ons(combined_df) + except Exception as e: + pass + + # Clean MSR + combined_df = postprocessing_funcs.clean_msr_lesser(combined_df) + + # Clean Default - commented out bc DEFAULT_TERM is full of NAs + try: + combined_df = postprocessing_funcs.clean_default_term(combined_df) + except Exception as e: + pass + + # Clean Prov 2 + combined_df = postprocessing_funcs.clean_prov_2(combined_df) + + # Clean Lesser Of Rate + try: + combined_df = postprocessing_funcs.clean_lesser_rate(combined_df) + except Exception as e: + pass + + # Clean LOB - commented out until we're ready + # combined_df = postprocessing_funcs.clean_lob(combined_df, filename) + + # Rename and reorder + combined_df.rename(columns=valid.B_MAPPING, inplace=True) + column_order = [ + col for col in valid.B_MAPPING.values() if col in combined_df.columns + ] + final_df = combined_df[column_order] + return final_df + else: + return combined_df + + +##### Batch 1 ##### +# New B Outputs +print("Starting B") +dir_batch_1_individual_b_outputs = "output_individual/cnc_batch1_b/" + +fn_b_batch_1_outputs_list = [ + dir_batch_1_individual_b_outputs + f + "/b_output.csv" + for f in os.listdir(dir_batch_1_individual_b_outputs) +] +pre_concat_list_b_batch_1 = [] +print("starting B concatenation for batch 1") +print(str(len(fn_b_batch_1_outputs_list)) + " B files pre-CSV reading") + +for fn in fn_b_batch_1_outputs_list: + # try: + file_b_df = pd.read_csv(fn) + # for colname in file_b_df.columns: + # print(colname) + # if colname == 'DEFAULT_TERM': + # print(file_b_df[colname].info()) + # print(fn) + file_b_df = b_postprocess(file_b_df, fn) + pre_concat_list_b_batch_1.append(file_b_df) +# except Exception as e: +# print(f"Appending failed on {fn} with exception {e}") + +print(str(len(pre_concat_list_b_batch_1)) + " B files post-CSV reading") +abc = pd.concat(pre_concat_list_b_batch_1, axis=1) + +abc.to_csv("src/scratchfiles/b_concat_df.csv", index=False) + + +abc.drop(["Exclusions", "Unnamed: 0"], axis=1, inplace=True) +# post = b_postprocess(abc, "") +# print(post.shape) +# print(post.columns) + +# print(post.head()) + +quit() +# # Original AC1 Outputs - All files +# df_batch_1_ac1 = pd.read_csv('reference_files/CNC-1-AC.csv') +# print(df_batch_1_ac1.shape) +# print(df_batch_1_ac1.columns) + +# # quit() + +# # Cleaned ABC original +# fn_batch_1_clean_abc = 'reference_files/CNC-Batch 1 Output_File1.xlsx' +# df_batch_1_clean_abc = pd.read_excel(fn_batch_1_clean_abc) +# print("Head of df_batch_1_clean_abc['Contract_Name']:") +# print(df_batch_1_clean_abc['Contract_Name'].head()) +# print("Batch 1 clean shape: " + str(df_batch_1_clean_abc.shape)) + + +# New AC2 Outputs +dir_batch_1_individual_ac2_outputs = "output_individual/cnc_batch1_ac/" +fn_ac2_batch_1_outputs_list = [ + dir_batch_1_individual_ac2_outputs + f + "/ac_output.csv" + for f in os.listdir(dir_batch_1_individual_ac2_outputs) +] +pre_concat_list_batch_1 = [] +print("starting AC2 concatenation for batch 1") +print(str(len(fn_ac2_batch_1_outputs_list)) + " files pre-CSV reading") +for fn in fn_ac2_batch_1_outputs_list: + try: + pre_concat_list_batch_1.append(pd.read_csv(fn)) + except Exception as e: + print(f"Appending failed on {fn} with exception {e}") +print(str(len(pre_concat_list_batch_1)) + " AC2 files post-CSV reading") +df_concat_batch_1_indiv_outputs = ac_postprocess(pd.concat(pre_concat_list_batch_1)) + +print(df_concat_batch_1_indiv_outputs.shape) + + +""" +1. Read B - create abc - remove old columns (Exclusions, Add-On, MedNes) +2. Updae AC 1 column names - Add AC1 original to the end of abc +3. Merge AC2 to abc +4. Postprocess + + +Just doing: +1. Read B +2. Postprocess as we read them in before we concatenate them ac_postprocess and b_postprocess (for b postprocess we comment out contract name, parent agreement, pages) +3. Just merge new AC2 outputs + +""" + + +# print("starting merging for batch 1") +# df_batch_1_clean_abc['Contract_Name'] = df_batch_1_clean_abc['Contract_Name'] + '.txt' +# batch_1_merged_ac = pd.merge(df_batch_1_clean_abc, df_concat_batch_1_indiv_outputs, how='left', left_on='Contract_Name', +# right_on='Contract Name', indicator=True) + +# # batch_1_merged.to_csv("output_consolidated/CNC-1-AC2-Added.csv", index=False) + +# print("Batch1 AC2 merge completed. Merge status counts:") +# print(batch_1_merged['_merge'].value_counts()) + + +# batch_1_abc = pd.merge(batch_1_merged_ac, df_concat_batch_1_indiv_b_outputs, how='left', left_on='Contract_Name', +# right_on='Contract Name', indicator=True) +# print(batch_1_abc.shape) + +# print("Batch 1 merged and written to CSV") + +# batch_1_abc.to_csv("batch_1_merged_abc.csv", index=False) + + +##### Batch 2 ##### + +# fn_batch_2_clean_abc = 'reference_files/CNC-Batch 2 Output_File1.xlsx' +# df_batch_2_clean_abc = pd.read_excel(fn_batch_2_clean_abc) + +# df_batch_2_clean_abc['Contract_Name'] = df_batch_2_clean_abc['Contract Name'] +# print("Head of df_batch_2_clean_abc['Contract_Name']:") +# print(df_batch_2_clean_abc['Contract_Name'].head()) + + +# print("Batch 2 clean shape: " + str(df_batch_2_clean_abc.shape)) + + +# dir_batch_2_individual_ac2_outputs = 'output_individual/cnc_batch2_ac/' +# fn_ac2_batch_2_outputs_list = [dir_batch_2_individual_ac2_outputs + f + "/ac_output.csv" +# for f in os.listdir(dir_batch_2_individual_ac2_outputs)] + +# pre_concat_list_batch_2 = [] + +# print("starting concatenation for batch 2") +# print(str(len(fn_ac2_batch_2_outputs_list)) + " files pre-CSV reading") +# for fn in fn_ac2_batch_2_outputs_list: +# try: +# pre_concat_list_batch_2.append(pd.read_csv(fn)) +# except Exception as e: +# print(f"Appending failed on {fn} with exception {e}") + +# print(str(len(pre_concat_list_batch_2)) + " files post-CSV reading") +# df_concat_batch_2_indiv_outputs = pd.concat(pre_concat_list_batch_2) + +# print("batch 2 concatenation complete") + + +# print("starting merging for batch 2") +# df_batch_2_clean_abc['Contract_Name'] = df_batch_2_clean_abc['Contract_Name'] + '.txt' +# batch_2_merged = pd.merge(df_batch_2_clean_abc, df_concat_batch_2_indiv_outputs, how='left', left_on='Contract_Name', +# right_on='Contract Name', indicator=True) +# batch_2_merged.to_csv("batch_2_merged.csv", index=False) +# print("batch 2 merge completed. Merge status counts:") +# print(batch_2_merged['_merge'].value_counts()) +# print("Batch 2 merged and written to CSV") diff --git a/fieldExtraction/scripts/adhoc/consolidate_ac2.py b/fieldExtraction/scripts/adhoc/consolidate_ac2.py new file mode 100644 index 0000000..9fb5828 --- /dev/null +++ b/fieldExtraction/scripts/adhoc/consolidate_ac2.py @@ -0,0 +1,196 @@ +import pandas as pd +import os + +import valid +from postprocess import ac_postprocess +import postprocessing_funcs + +""" +This script consolidates a&c fields. Its data sources are `reference_files/CNC-Batch [1/2] Output_File1.xlsx` +as well as .csv files from `output_individual/cnc_batch[1/2]_ac/[name of provider]/ac_output.csv. +""" + + +def b_postprocess(combined_df, filename): + if combined_df.shape[0] > 0: + # Add Single Code, Multiple Rate + combined_df = postprocessing_funcs.add_scmr(combined_df) + + # Filter Add Ons + combined_df = postprocessing_funcs.filter_add_ons(combined_df) + + # Clean MSR + combined_df = postprocessing_funcs.clean_msr_lesser(combined_df) + + # Clean Default + combined_df = postprocessing_funcs.clean_default_term(combined_df) + + # Clean Prov 2 + combined_df = postprocessing_funcs.clean_prov_2(combined_df) + + # Clean Lesser Of Rate + combined_df = postprocessing_funcs.clean_lesser_rate(combined_df) + + # Clean LOB + combined_df = postprocessing_funcs.clean_lob(combined_df, filename) + + # Rename and reorder + combined_df.rename(columns=valid.B_MAPPING, inplace=True) + column_order = [ + col for col in valid.B_MAPPING.values() if col in combined_df.columns + ] + final_df = combined_df[column_order] + return final_df + else: + return combined_df + + +##### Batch 1 ##### +# New B Outputs +print("Starting B") +dir_batch_1_individual_b_outputs = "output_individual/cnc_batch1_b/" + +fn_b_batch_1_outputs_list = [ + dir_batch_1_individual_b_outputs + f + "/b_output.csv" + for f in os.listdir(dir_batch_1_individual_b_outputs) +] +pre_concat_list_b_batch_1 = [] +print("starting B concatenation for batch 1") +print(str(len(fn_b_batch_1_outputs_list)) + " B files pre-CSV reading") + +for fn in fn_b_batch_1_outputs_list: + try: + file_b_df = pd.read_csv(fn) + file_b_df = b_postprocess(file_b_df, fn) + pre_concat_list_b_batch_1.append(file_b_df) + except Exception as e: + print(f"Appending failed on {fn} with exception {e}") + +print(str(len(pre_concat_list_b_batch_1)) + " B files post-CSV reading") +abc = pd.concat(pre_concat_list_b_batch_1) + +abc.drop(["Exclusions", "Unnamed: 0"], axis=1, inplace=True) +print(abc.shape) +print(abc.columns) + +# Original AC1 Outputs - All files +df_batch_1_ac1 = pd.read_csv("reference_files/CNC-1-AC.csv") +print(df_batch_1_ac1.shape) +print(df_batch_1_ac1.columns) + +# quit() + +# Cleaned ABC original +fn_batch_1_clean_abc = "reference_files/CNC-Batch 1 Output_File1.xlsx" +df_batch_1_clean_abc = pd.read_excel(fn_batch_1_clean_abc) +print("Head of df_batch_1_clean_abc['Contract_Name']:") +print(df_batch_1_clean_abc["Contract_Name"].head()) +print("Batch 1 clean shape: " + str(df_batch_1_clean_abc.shape)) + + +# New AC2 Outputs +dir_batch_1_individual_ac2_outputs = "output_individual/cnc_batch1_ac/" +fn_ac2_batch_1_outputs_list = [ + dir_batch_1_individual_ac2_outputs + f + "/ac_output.csv" + for f in os.listdir(dir_batch_1_individual_ac2_outputs) +] +pre_concat_list_batch_1 = [] +print("starting AC2 concatenation for batch 1") +print(str(len(fn_ac2_batch_1_outputs_list)) + " files pre-CSV reading") +for fn in fn_ac2_batch_1_outputs_list: + try: + pre_concat_list_batch_1.append(pd.read_csv(fn)) + except Exception as e: + print(f"Appending failed on {fn} with exception {e}") +print(str(len(pre_concat_list_batch_1)) + " AC2 files post-CSV reading") +df_concat_batch_1_indiv_outputs = ac_postprocess(pd.concat(pre_concat_list_batch_1)) + + +""" +1. Read B - create abc - remove old columns (Exclusions, Add-On, MedNes) +2. Updae AC 1 column names - Add AC1 original to the end of abc +3. Merge AC2 to abc +4. Postprocess + + +Just doing: +1. Read B +2. Postprocess as we read them in before we concatenate them ac_postprocess and b_postprocess (for b postprocess we comment out contract name, parent agreement, pages) +3. Just merge new AC2 outputs + +""" + + +print("starting merging for batch 1") +df_batch_1_clean_abc["Contract_Name"] = df_batch_1_clean_abc["Contract_Name"] + ".txt" +batch_1_merged_ac = pd.merge( + df_batch_1_clean_abc, + df_concat_batch_1_indiv_outputs, + how="left", + left_on="Contract_Name", + right_on="Contract Name", + indicator=True, +) + +# batch_1_merged.to_csv("output_consolidated/CNC-1-AC2-Added.csv", index=False) + +print("Batch1 AC2 merge completed. Merge status counts:") +print(batch_1_merged["_merge"].value_counts()) + + +batch_1_abc = pd.merge( + batch_1_merged_ac, + df_concat_batch_1_indiv_b_outputs, + how="left", + left_on="Contract_Name", + right_on="Contract Name", + indicator=True, +) +print(batch_1_abc.shape) + +print("Batch 1 merged and written to CSV") + +batch_1_abc.to_csv("batch_1_merged_abc.csv", index=False) + + +##### Batch 2 ##### + +# fn_batch_2_clean_abc = 'reference_files/CNC-Batch 2 Output_File1.xlsx' +# df_batch_2_clean_abc = pd.read_excel(fn_batch_2_clean_abc) + +# df_batch_2_clean_abc['Contract_Name'] = df_batch_2_clean_abc['Contract Name'] +# print("Head of df_batch_2_clean_abc['Contract_Name']:") +# print(df_batch_2_clean_abc['Contract_Name'].head()) + + +# print("Batch 2 clean shape: " + str(df_batch_2_clean_abc.shape)) + + +# dir_batch_2_individual_ac2_outputs = 'output_individual/cnc_batch2_ac/' +# fn_ac2_batch_2_outputs_list = [dir_batch_2_individual_ac2_outputs + f + "/ac_output.csv" +# for f in os.listdir(dir_batch_2_individual_ac2_outputs)] + +# pre_concat_list_batch_2 = [] + +# print("starting concatenation for batch 2") +# print(str(len(fn_ac2_batch_2_outputs_list)) + " files pre-CSV reading") +# for fn in fn_ac2_batch_2_outputs_list: +# try: +# pre_concat_list_batch_2.append(pd.read_csv(fn)) +# except Exception as e: +# print(f"Appending failed on {fn} with exception {e}") + +# print(str(len(pre_concat_list_batch_2)) + " files post-CSV reading") +# df_concat_batch_2_indiv_outputs = pd.concat(pre_concat_list_batch_2) + +# print("batch 2 concatenation complete") + + +# print("starting merging for batch 2") +# df_batch_2_clean_abc['Contract_Name'] = df_batch_2_clean_abc['Contract_Name'] + '.txt' +# batch_2_merged = pd.merge(df_batch_2_clean_abc, df_concat_batch_2_indiv_outputs, how='left', left_on='Contract_Name', +# right_on='Contract Name', indicator=True) +# batch_2_merged.to_csv("batch_2_merged.csv", index=False) +# print("batch 2 merge completed. Merge status counts:") +# print(batch_2_merged['_merge'].value_counts()) +# print("Batch 2 merged and written to CSV") diff --git a/fieldExtraction/scripts/adhoc/tx_ac_no_reimbursement.py b/fieldExtraction/scripts/adhoc/tx_ac_no_reimbursement.py new file mode 100644 index 0000000..b9a4039 --- /dev/null +++ b/fieldExtraction/scripts/adhoc/tx_ac_no_reimbursement.py @@ -0,0 +1,121 @@ +import concurrent.futures +import traceback +import os +import csv +import time +import sys +import random +import pandas as pd +import random + +random.seed(42) + +import utils +import config +import claude_funcs +import file_processing +import tracking +import consolidate_output + + +def process_b(item): + filename, contract_text = item + if utils.contains_reimbursement(contract_text, 0): + try: + results = file_processing.run_b_prompts(item) + if results is not None: + tracking.update_results_csv(item, results) + return item, results + except Exception as e: + print(f"Error processing item {filename}: {e}") + traceback.print_exc() + else: + print(f"No reimbursement information found in {filename}. Skipping processing.") + + +def process_ac(item): + filename, contract_text = item + try: + results = file_processing.run_ac_prompts(item) + if results is not None: + tracking.update_results_csv(item, results) + return item, results + except Exception as e: + print(f"Error processing item {filename}: {e}") + traceback.print_exc() + + +def main(): + if config.TEST: + print( + claude_funcs.invoke_claude( + "Write 'test', nothing more.", + model_id=config.MODEL_ID_CLAUDE2, + filename="test", + max_tokens=10, + ) + ) + else: + input_dict = ( + utils.read_input() + ) # keys are contract names, values are full contract text + total_files = len(input_dict) + + print(f"Input Files : {len(input_dict)}") + + # Filter no reimbursement + input_dict = {k: input_dict[k] for k in list(input_dict)[:100]} + input_dict = { + k: v for k, v in input_dict.items() if utils.contains_reimbursement(str(v)) + } # Filter out non-contracts + print( + f"Input Files after reimbursement filter: {len(input_dict)} | {total_files-len(input_dict)} files removed" + ) + + # Filter already processed + if config.FILTER_ALREADY_PROCESSED: + input_dict = utils.filter_already_processed(input_dict) + print(f"Input Files left to be processed : {len(input_dict)}") + + batch_results = [] + processed_count = 0 + + with concurrent.futures.ThreadPoolExecutor( + max_workers=config.MAX_WORKERS + ) as executor: + if "b" in config.FIELDS: + futures = [ + executor.submit(process_b, item) for item in input_dict.items() + ] + if "a" in config.FIELDS and "c" in config.FIELDS: + futures = [ + executor.submit(process_ac, item) for item in input_dict.items() + ] + + for future in concurrent.futures.as_completed(futures): + try: + result = future.result() + if result: + batch_results.append(result) + processed_count += 1 + + if processed_count % 5 == 0: + tracking.write_batch_results(batch_results) + batch_results = [] + except Exception as e: + print(f"Error in future: {e}") + traceback.print_exc() + + if batch_results: + tracking.write_batch_results(batch_results) + + tracking.write_stats_to_csv() + print("\nIndividual Processing Complete") + + print("Consolidation starting...") + consolidate_output.consolidate_output() + print("Consolidation complete...") + + +if __name__ == "__main__": + main() diff --git a/fieldExtraction/scripts/adhoc/unit_test.py b/fieldExtraction/scripts/adhoc/unit_test.py new file mode 100644 index 0000000..906c908 --- /dev/null +++ b/fieldExtraction/scripts/adhoc/unit_test.py @@ -0,0 +1,34 @@ +# """ +# Run N iterations of a single prompt, count unique outputs +# """ + +# import pandas as pd + +# import prompts +# import claude_funcs +# import config +# import dict_operations + + +# N = 1 + +# # Paste contract text here +# text = """The maximum compensation for Covered Services rendered to a Covered Person shall be the "Allowed Amount." +# The Allowed Amount for Covered Services is the lesser of: (i) Allowable Charges; or (ii) as applicable, +# (a) 100% of the Payor's Medicaid fee schedule in effect on the date of service, +# or (b) for those Covered Services included in the Service Categories in Table 1 below, +# the applicable "Contracted Rate" set forth in Table 1 below. Physical, Occupational and Speech Therapy | 80% of the Medicaid Rate in effect on the date of service. +# """ + +# # Change prompt below +# prompt = prompts.BOTTOM_UP_METHODOLOGY_BREAKOUT({'FULL_METHODOLOGY' : text}) + +# all_dicts = [] +# for i in range(N): +# answer = claude_funcs.invoke_claude(prompt, config.MODEL_ID_CLAUDE35_SONNET, "", 512) +# answer_dict = dict_operations.secondary_string_to_dict(answer, "") +# print(answer_dict) +# all_dicts.append(answer_dict) + +# # answer_df = pd.DataFrame(all_dicts) +# # answer_df.to_csv('full_methodology_unit_test.csv') diff --git a/fieldExtraction/scripts/look_at_columns_batches_1_and_2.py b/fieldExtraction/scripts/look_at_columns_batches_1_and_2.py new file mode 100644 index 0000000..8ffd643 --- /dev/null +++ b/fieldExtraction/scripts/look_at_columns_batches_1_and_2.py @@ -0,0 +1,106 @@ +import pandas as pd +import os + +fn_batch_1_clean_abc = "reference_files/CNC-Batch 1 Output_File1.xlsx" +df_batch_1_clean_abc = pd.read_excel(fn_batch_1_clean_abc) + +print("Head of df_batch_1_clean_abc['Contract_Name']:") +print(df_batch_1_clean_abc["Contract_Name"].head()) + +fn_batch_2_clean_abc = "reference_files/CNC-Batch 2 Output_File1.xlsx" +df_batch_2_clean_abc = pd.read_excel(fn_batch_2_clean_abc) + +df_batch_2_clean_abc["Contract_Name"] = df_batch_2_clean_abc["Contract Name"] +print("Head of df_batch_2_clean_abc['Contract_Name']:") +print(df_batch_2_clean_abc["Contract_Name"].head()) + +print("Batch 1 clean shape: " + str(df_batch_1_clean_abc.shape)) +print("Batch 2 clean shape: " + str(df_batch_2_clean_abc.shape)) + +print("batch 1 columns: ") +for column in df_batch_1_clean_abc.columns: + print(column) + +print("\nbatch 2 columns: ") +for column in df_batch_2_clean_abc.columns: + print(column) + +print("\n columns in batch 2 not in batch 1: ") +b1cols = set(df_batch_1_clean_abc.columns) +b2cols = set(df_batch_2_clean_abc.columns) +print(b2cols.difference(b1cols)) + +quit() + +dir_batch_1_individual_ac2_outputs = "output_individual/cnc_batch1_ac/" +fn_ac2_batch_1_outputs_list = [ + dir_batch_1_individual_ac2_outputs + f + "/ac_output.csv" + for f in os.listdir(dir_batch_1_individual_ac2_outputs) +] + +dir_batch_2_individual_ac2_outputs = "output_individual/cnc_batch2_ac/" +fn_ac2_batch_2_outputs_list = [ + dir_batch_2_individual_ac2_outputs + f + "/ac_output.csv" + for f in os.listdir(dir_batch_2_individual_ac2_outputs) +] + + +pre_concat_list_batch_1 = [] +pre_concat_list_batch_2 = [] +print("starting concatenation for batch 1") +print(str(len(fn_ac2_batch_1_outputs_list)) + " files pre-CSV reading") + +for fn in fn_ac2_batch_1_outputs_list: + try: + pre_concat_list_batch_1.append(pd.read_csv(fn)) + except Exception as e: + print(f"Appending failed on {fn} with exception {e}") + +print(str(len(pre_concat_list_batch_1)) + " files post-CSV reading") +df_concat_batch_1_indiv_outputs = pd.concat(pre_concat_list_batch_1) + +print("batch 1 concatenation complete") + +print("starting concatenation for batch 2") +print(str(len(fn_ac2_batch_2_outputs_list)) + " files pre-CSV reading") +for fn in fn_ac2_batch_2_outputs_list: + try: + pre_concat_list_batch_2.append(pd.read_csv(fn)) + except Exception as e: + print(f"Appending failed on {fn} with exception {e}") + +print(str(len(pre_concat_list_batch_2)) + " files post-CSV reading") +df_concat_batch_2_indiv_outputs = pd.concat(pre_concat_list_batch_2) + +print("batch 2 concatenation complete") + +print("starting merging for batch 1") + +df_batch_1_clean_abc["Contract_Name"] = df_batch_1_clean_abc["Contract_Name"] + ".txt" +batch_1_merged = pd.merge( + df_batch_1_clean_abc, + df_concat_batch_1_indiv_outputs, + how="left", + left_on="Contract_Name", + right_on="Contract Name", + indicator=True, +) +batch_1_merged.to_csv("batch_1_merged.csv", index=False) +print("batch 1 merge completed. Merge status counts:") +print(batch_1_merged["_merge"].value_counts()) +print("Batch 1 merged and written to CSV") + +print("starting merging for batch 2") +df_batch_2_clean_abc["Contract_Name"] = df_batch_2_clean_abc["Contract_Name"] + ".txt" +batch_2_merged = pd.merge( + df_batch_2_clean_abc, + df_concat_batch_2_indiv_outputs, + how="left", + left_on="Contract_Name", + right_on="Contract Name", + indicator=True, +) +batch_2_merged.to_csv("batch_2_merged.csv", index=False) +print("batch 2 merge completed. Merge status counts:") +print(batch_2_merged["_merge"].value_counts()) +print("Batch 2 merged and written to CSV") diff --git a/fieldExtraction/scripts/qa_qc.py b/fieldExtraction/scripts/qa_qc.py new file mode 100644 index 0000000..633a0eb --- /dev/null +++ b/fieldExtraction/scripts/qa_qc.py @@ -0,0 +1,508 @@ +import pandas as pd +import os +import sys +from datetime import datetime +import re +from openpyxl import Workbook +from openpyxl.utils.dataframe import dataframe_to_rows +import argparse +import boto3 +from botocore.exceptions import ClientError + +sys.path.append("src") +import utils + +file_name_column = "Contract Name" + +"""python scripts/qa_qc.py --input_dir ______ --output_dir _______ --ac_df ________ --b_df ______ +You should have both input and output (ouput is just an empty folder) and then one of ac_df or b_df or both +The final report will be saved in the parent directory of when you are running under reports/ +Run this from the base folder of the repo.""" + + +def parse_arguments(): + parser = argparse.ArgumentParser(description="QC/QA Integration Script") + parser.add_argument("--input_dir", help="Input directory (local path or S3 URI)") + parser.add_argument("--output_dir", help="Output directory") + parser.add_argument("--ac_df", help="Path to AC DataFrame CSV") + parser.add_argument("--b_df", help="Path to B DataFrame CSV") + return parser.parse_args() + + +def is_s3_path(path): + return path.startswith("s3://") + + +def read_s3_file(bucket, key): + s3 = boto3.client("s3") + try: + response = s3.get_object(Bucket=bucket, Key=key) + return response["Body"].read().decode("utf-8") + except ClientError as e: + print(f"Error reading S3 file {key} from bucket {bucket}: {e}") + return None + + +def read_input_files(input_dir): + if not input_dir: + return {} + + if is_s3_path(input_dir): + s3_parts = input_dir[5:].split("/", 1) + bucket = s3_parts[0] + prefix = s3_parts[1] if len(s3_parts) > 1 else "" + s3 = boto3.client("s3") + input_dict = {} + try: + for obj in s3.list_objects_v2(Bucket=bucket, Prefix=prefix)["Contents"]: + key = obj["Key"] + if key.endswith(".txt"): + content = read_s3_file(bucket, key) + if content: + input_dict[os.path.basename(key)] = content + except ClientError as e: + print(f"Error listing S3 objects: {e}") + else: + input_dict = {} + for filename in os.listdir(input_dir): + if filename.endswith(".txt"): + with open( + os.path.join(input_dir, filename), "r", encoding="utf-8" + ) as file: + input_dict[filename] = file.read() + return input_dict + + +def remove_txt_extension(filename): + if isinstance(filename, str) and filename.lower().endswith(".txt"): + return filename[:-4] + return filename + + +def detect_anomalies(series, field_name, is_merged=False): + total_count = len(series) + unique_values = series.fillna("").nunique() + + date_fields = [ + "CONTRACT_BASE_EFFECTIVE_DT", + "CONTRACT_EFFECTIVE_DT", + "CONTRACT_EFFECTIVE_DT_SIGNATORY", + "CONTRACT_TERMINATION_DT", + "CONTRACT_SIGNATORY_DT", + ] + date_pattern = r"(\d{1,2}[/-]\d{1,2}[/-]\d{2,4}|\d{2,4}[/-]\d{1,2}[/-]\d{1,2})" + + date_like_count = 0 + + if pd.api.types.is_string_dtype(series) or pd.api.types.is_object_dtype(series): + series = series.astype(str) + if field_name in date_fields: + date_like_count = series.str.count(date_pattern).sum() + else: + date_like_count = series.str.count(r"\d{1,2}/\d{1,2}/\d{2,4}").sum() + + data_type = "float64" if pd.api.types.is_float_dtype(series) else "string" + + dataset = "Merged DataFrame" if is_merged else "Original DataFrame" + result = { + f"Total entries ({dataset})": total_count, + f"Unique values ({dataset})": unique_values, + f"Date-like entries ({dataset})": date_like_count, + } + + if is_merged: + result[f"Data type ({dataset})"] = data_type + + return result + + +def perform_qc_qa_tests_both(ac_df, b_df, merged_df, input_dict, processed_files): + total_input_files = len(input_dict) if input_dict else 0 + processed_ac_files = len( + [f for f in processed_files if f in ac_df[file_name_column].values] + ) + processed_b_files = len( + [f for f in processed_files if f in b_df[file_name_column].values] + ) + failed_ac_files = ( + total_input_files - processed_ac_files if total_input_files > 0 else 0 + ) + failed_b_files = ( + total_input_files - processed_b_files if total_input_files > 0 else 0 + ) + + general_stats = { + "Total Number of Contracts processed": total_input_files, + "# of contracts successfully processed for AC fields": processed_ac_files, + "# of contracts successfully processed for B fields": processed_b_files, + "# of contracts failed for AC fields": failed_ac_files, + "# of contracts failed for B fields": failed_b_files, + "# of rows for AC run": len(ac_df), + "# of rows for B run": len(b_df), + "Total rows in merged DataFrame": len(merged_df), + "Unique Filenames in AC DataFrame": ac_df[file_name_column].nunique(), + "Unique Filenames in B DataFrame": b_df[file_name_column].nunique(), + "Unique Filenames in merged DataFrame": merged_df[file_name_column].nunique(), + "Contracts with all AC fields missing": merged_df[ + [col for col in ac_df.columns if col != file_name_column] + ] + .isnull() + .all(axis=1) + .sum(), + "Contracts with all B fields missing": merged_df[ + [col for col in b_df.columns if col != file_name_column] + ] + .isnull() + .all(axis=1) + .sum(), + "# of duplicate rows for AC run": ac_df.duplicated().sum(), + "# of duplicate rows for B run": b_df.duplicated().sum(), + "Number of duplicate rows in merged DataFrame": merged_df.duplicated().sum(), + } + + ac_stats = {} + for field in [col for col in ac_df.columns if col != file_name_column]: + missing_contracts = merged_df[merged_df[field].isnull()][ + file_name_column + ].nunique() + ac_stats[field] = { + "Contracts missing this field (Merged DataFrame)": missing_contracts, + **detect_anomalies(ac_df[field], field, is_merged=False), + **detect_anomalies(merged_df[field], field, is_merged=True), + } + + b_stats = {} + for field in [col for col in b_df.columns if col != file_name_column]: + missing_contracts = ( + merged_df.groupby(file_name_column)[field] + .apply(lambda x: x.isnull().all()) + .sum() + ) + contracts_with_missing = ( + merged_df.groupby(file_name_column)[field] + .apply(lambda x: x.isnull().any()) + .sum() + ) + total_missing = merged_df[field].isnull().sum() + b_stats[field] = { + "Contracts missing all entries (Merged DataFrame)": missing_contracts, + "Contracts with at least one missing entry (Merged DataFrame)": contracts_with_missing, + "Total missing values (Merged DataFrame)": total_missing, + **detect_anomalies(b_df[field], field, is_merged=False), + **detect_anomalies(merged_df[field], field, is_merged=True), + } + + return general_stats, ac_stats, b_stats + + +def perform_qc_qa_tests_ac(ac_df, input_dict, processed_files): + total_input_files = len(input_dict) if input_dict else 0 + processed_ac_files = len( + [f for f in processed_files if f in ac_df[file_name_column].values] + ) + failed_ac_files = ( + total_input_files - processed_ac_files if total_input_files > 0 else 0 + ) + + general_stats = { + "Total Number of Contracts processed": total_input_files, + "# of contracts successfully processed for AC fields": processed_ac_files, + "# of contracts failed for AC fields": failed_ac_files, + "# of rows for AC run": len(ac_df), + "Unique Filenames in AC DataFrame": ac_df[file_name_column].nunique(), + "Contracts with all AC fields missing": ac_df[ + [col for col in ac_df.columns if col != file_name_column] + ] + .isnull() + .all(axis=1) + .sum(), + "# of duplicate rows for AC run": ac_df.duplicated().sum(), + } + + ac_stats = {} + for field in [col for col in ac_df.columns if col != file_name_column]: + missing_contracts = ac_df[ac_df[field].isnull()][file_name_column].nunique() + ac_stats[field] = { + "Contracts missing this field (Merged DataFrame)": missing_contracts, + **detect_anomalies(ac_df[field], field, is_merged=False), + } + + return general_stats, ac_stats + + +def perform_qc_qa_tests_b(b_df, input_dict, processed_files): + total_input_files = len(input_dict) if input_dict else 0 + processed_b_files = len( + [f for f in processed_files if f in b_df[file_name_column].values] + ) + failed_b_files = ( + total_input_files - processed_b_files if total_input_files > 0 else 0 + ) + + general_stats = { + "Total Number of Contracts processed": total_input_files, + "# of contracts successfully processed for B fields": processed_b_files, + "# of contracts failed for B fields": failed_b_files, + "# of rows for B run": len(b_df), + "Unique Filenames in B DataFrame": b_df[file_name_column].nunique(), + "Contracts with all B fields missing": b_df[ + [col for col in b_df.columns if col != file_name_column] + ] + .isnull() + .all(axis=1) + .sum(), + "# of duplicate rows for B run": b_df.duplicated().sum(), + } + + b_stats = {} + for field in [col for col in b_df.columns if col != file_name_column]: + missing_contracts = ( + b_df.groupby(file_name_column)[field] + .apply(lambda x: x.isnull().all()) + .sum() + ) + contracts_with_missing = ( + b_df.groupby(file_name_column)[field] + .apply(lambda x: x.isnull().any()) + .sum() + ) + total_missing = b_df[field].isnull().sum() + b_stats[field] = { + "Contracts missing all entries (B DataFrame)": missing_contracts, + "Contracts with at least one missing entry (B DataFrame)": contracts_with_missing, + "Total missing values (Merged DataFrame)": total_missing, + **detect_anomalies(b_df[field], field, is_merged=False), + } + + return general_stats, b_stats + + +def split_into_pages(text): + text_list = text.split("Start of Page No. = ") + text_dict = {page.split()[0]: page for page in text_list[1:]} + return text_dict + + +def get_table_stats(text_dict): + table_count, num_table_pages, rate_count = 0, 0, 0 + table_pages, rate_pages, rates_more_than_10 = [], [], [] + for page_num, page_text in text_dict.items(): + if utils.contains_reimbursement(page_text, page_num): + rate_pages.append(page_num) + rates_on_page = sum(1 for char in page_text if char in {"$", "%"}) + rate_count += rates_on_page + if rates_on_page >= 10: + rates_more_than_10.append(page_num) + + table_texts = re.findall( + r"-------Table Start--------(.*?)-------Table End--------", + page_text, + re.DOTALL, + ) + table_page_count = len(table_texts) + if table_page_count > 0: + table_count += table_page_count + num_table_pages += 1 + table_pages.append(page_num) + + return { + "Table Count": table_count, + "Table Page Count": num_table_pages, + "Table Pages": table_pages, + "Rate Count": rate_count, + "Rate Pages": rate_pages, + "Num Rate Pages": len(rate_pages), + ">=10 Rate Pages": rates_more_than_10, + "Num >=10 Rates": len(rates_more_than_10), + } + + +def main(): + args = parse_arguments() + + script_dir = os.path.dirname(os.path.abspath(__file__)) + parent_dir = os.path.dirname(script_dir) + + ac_df_path = ( + os.path.abspath(os.path.join(parent_dir, args.ac_df)) if args.ac_df else None + ) + b_df_path = ( + os.path.abspath(os.path.join(parent_dir, args.b_df)) if args.b_df else None + ) + + input_dir = ( + os.path.abspath(os.path.join(parent_dir, args.input_dir)) + if args.input_dir + else None + ) + output_dir = ( + os.path.abspath(os.path.join(parent_dir, args.output_dir)) + if args.output_dir + else parent_dir + ) + + reports_dir = os.path.join(parent_dir, "reports") + os.makedirs(reports_dir, exist_ok=True) + + if ac_df_path: + print(f"AC DataFrame: {ac_df_path}") + if b_df_path: + print(f"B DataFrame: {b_df_path}") + if input_dir: + print(f"Input Dir: {input_dir}") + print(f"Output Dir: {output_dir}") + print(f"Reports Dir: {reports_dir}") + + if not input_dir: + print("Needs an Input Directory!") + return + if not ac_df_path and not b_df_path: + print("Needs at least one of AC Outputs and B Outputs!") + return + + input_dict = read_input_files(input_dir) + total_files = len(input_dict) + print(f"Total Input Files: {total_files}") + + all_input_filenames = ( + [remove_txt_extension(f.lower()) for f in input_dict.keys()] + if input_dict + else [] + ) + + ac_df = ( + pd.read_csv(ac_df_path).drop(columns=["Unnamed: 0"], errors="ignore") + if ac_df_path + else None + ) + b_df = ( + pd.read_csv(b_df_path).drop(columns=["Unnamed: 0"], errors="ignore") + if b_df_path + else None + ) + + if ac_df_path: + ac_df[file_name_column] = ac_df[file_name_column].apply( + lambda x: remove_txt_extension(str(x)).lower() + ) + ac_filenames = set(ac_df[file_name_column]) + if b_df_path: + b_filenames = set(b_df[file_name_column]) + b_df[file_name_column] = b_df[file_name_column].apply( + lambda x: remove_txt_extension(str(x)).lower() + ) + + if ac_df_path and b_df_path: + unmatched_b_filenames = b_filenames - ac_filenames + unmatched_b_df = pd.DataFrame({file_name_column: list(unmatched_b_filenames)}) + unmatched_b_output_path = os.path.join(output_dir, "unmatched_B_filenames.csv") + unmatched_b_df.to_csv(unmatched_b_output_path, index=False) + print(f"Unmatched B filenames saved to '{unmatched_b_output_path}'") + + # Perform an inner merge to keep only matching rows for the main output + merged_df = pd.merge(ac_df, b_df, on=file_name_column, how="inner") + merged_df = merged_df.drop(columns=["Unnamed: 0"], errors="ignore") + ac_rows_before = len(ac_df) + b_rows_before = len(b_df) + merged_rows = len(merged_df) + unmatched_b_count = len(unmatched_b_filenames) + + print(f"Rows in AC DataFrame before merge: {ac_rows_before}") + print(f"Rows in B DataFrame before merge: {b_rows_before}") + print(f"Rows in merged DataFrame: {merged_rows}") + print(f"Unique filenames in B not found in AC: {unmatched_b_count}") + + merged_output_path = os.path.join(output_dir, "DOCZYAI_merged_results.csv") + merged_df.to_csv(merged_output_path, index=False) + print(f"Merged data saved to '{merged_output_path}'") + + processed_files = list( + set(ac_df[file_name_column].tolist() + b_df[file_name_column].tolist()) + ) + general_stats, ac_stats, b_stats = perform_qc_qa_tests_both( + ac_df, b_df, merged_df, input_dict, processed_files + ) + + elif ac_df_path: + processed_files = list(set(ac_df[file_name_column].tolist())) + general_stats, ac_stats = perform_qc_qa_tests_ac( + ac_df, input_dict, processed_files + ) + else: + processed_files = list(set(b_df[file_name_column].tolist())) + general_stats, b_stats = perform_qc_qa_tests_b( + b_df, input_dict, processed_files + ) + + table_stats = [] + if input_dict: + for filename, content in input_dict.items(): + filename_without_ext = remove_txt_extension(filename).lower() + text_dict = split_into_pages(content) + stats = get_table_stats(text_dict) + table_stats.append( + { + "Contract Name": filename, + "Has B output": ( + filename_without_ext in b_df[file_name_column].values + if b_df_path + else "No B Output Given" + ), + "Has AC output": ( + filename_without_ext in ac_df[file_name_column].values + if ac_df_path + else "No AC Output Given" + ), + "# Tables": stats["Table Count"], + "# Reimb rates": stats["Rate Count"], + "Table Pages": ", ".join(map(str, stats["Table Pages"])), + "Rate Pages": ", ".join(map(str, stats["Rate Pages"])), + ">=10 Rate Pages": ", ".join(map(str, stats[">=10 Rate Pages"])), + } + ) + + wb = Workbook() + ws_general = wb.active + ws_general.title = "General Stats" + + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + general_stats_with_metadata = { + "Report Generation Time": timestamp, + "AC File": args.ac_df if ac_df_path else "No AC Output Given", + "B File": args.b_df if b_df_path else "No B Output Given", + **general_stats, + } + + general_df = pd.DataFrame.from_dict( + general_stats_with_metadata, orient="index", columns=["Value"] + ) + for r in dataframe_to_rows(general_df, index=True, header=True): + ws_general.append(r) + + if ac_df_path: + ws_ac = wb.create_sheet("AC Fields Stats") + ac_stats_df = pd.DataFrame(ac_stats).T + for r in dataframe_to_rows(ac_stats_df, index=True, header=True): + ws_ac.append(r) + + if b_df_path: + ws_b = wb.create_sheet("B Fields Stats") + b_stats_df = pd.DataFrame(b_stats).T + for r in dataframe_to_rows(b_stats_df, index=True, header=True): + ws_b.append(r) + + if table_stats: + ws_table_stats = wb.create_sheet("Table and Reimbursement Stats") + table_stats_df = pd.DataFrame(table_stats) + for r in dataframe_to_rows(table_stats_df, index=False, header=True): + ws_table_stats.append(r) + + excel_report_filename = os.path.join( + reports_dir, f'QC_QA_Report_{datetime.now().strftime("%Y%m%d_%H%M%S")}.xlsx' + ) + wb.save(excel_report_filename) + print(f"Excel QC/QA Report saved to {excel_report_filename}") + + +if __name__ == "__main__": + main() diff --git a/fieldExtraction/src/.gitignore b/fieldExtraction/src/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/fieldExtraction/src/ac_funcs.py b/fieldExtraction/src/ac_funcs.py new file mode 100644 index 0000000..1a29b51 --- /dev/null +++ b/fieldExtraction/src/ac_funcs.py @@ -0,0 +1,316 @@ +import json +import boto3 +import pandas as pd +import numpy as np +from datetime import datetime +import os +import anthropic +import re +import logging +from urllib.parse import unquote_plus +from botocore.exceptions import ClientError +from botocore.config import Config +import traceback +import time +import sys + +import claude_funcs +import dict_operations +import config + + +def get_ac_answer(prompt, filename, fields): + response_text = claude_funcs.invoke_claude( + prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 8192 + ) + response_text = response_text.replace("_DATE", "_DT") + # response_dict = dict_operations.secondary_string_to_dict(response_text, filename) + response_dict = json_parsing_search(response_text, fields) + return response_dict + + +def get_ac_answer_single_field_simple(prompt, filename, field): + try: + response_text = claude_funcs.invoke_claude( + prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 8192 + ) + + # Clean the response + clean_response = response_text.strip() + + # Try to find a match within curly braces + brace_match = re.search(r"\{([^}]+)\}", clean_response) + if brace_match: + brace_content = brace_match.group(1).strip() + # Try to find the field within the braced content + field_pattern = r'"{}"\s*:\s*"?([^"}}]+)"?'.format(re.escape(field)) + field_match = re.search(field_pattern, brace_content, re.IGNORECASE) + if field_match: + return {field: field_match.group(1).strip()} + + # If no match in braces, try to find field in the format "Field: Value" + field_match = re.search( + r"{}: (.+?)(?:\n|$)".format(re.escape(field)), clean_response, re.IGNORECASE + ) + if field_match: + return {field: field_match.group(1).strip()} + + # Try to find field in the format "Field Value" + field_match = re.search( + r"{}\s+(.+?)(?:\n|$)".format(re.escape(field)), + clean_response, + re.IGNORECASE, + ) + if field_match: + return {field: field_match.group(1).strip()} + + # If no specific format is found, return the entire response + return {field: clean_response} + + except Exception as e: + print( + f"Error in get_ac_answer_single_field_simple for {filename} - {field}: {str(e)}" + ) + return {field: f"Error: {str(e)}"} + + +def get_filename_from_path(full_path): + return os.path.basename(full_path) + + +def select_context(text_dict, field): + if (field == "RELATIONSHIP_OF_PARTIES_IND") or ( + field == "RELATIONSHIP_OF_PARTIES_LANGUAGE" + ): + keywords = ["Relationship of Parties"] + if field == "REGULATORY_REQUIREMENTS": + keywords = ["Complience with Regulatory Requirements"] + if (field == "DISPARAGEMENT_PROHIBITION_IND") or ( + field == "DISPARAGEMENT_PROHIBITION_LANGUAGE" + ): + keywords = ["Disparagement Prohibition"] + if field == "ELIGIBILITY_VERIFICATION": + keywords = ["Eligibility Determination"] + if field == "INSURANCE_REQUIREMENT": + keywords = ["Insurance"] + if field == "PARTICIPATION_IN_PRODUCTS": + keywords = ["Participation in Products"] + if field == "POLICIES_AND_PROCEDURES": + keywords = ["Provider Manual", "Policies and Procedures"] + if field == "ARBITRATION_AND_DISPUTES": + keywords = ["Informal Dispute Resolution", "Arbitration"] + print(field) + page_list = [] + print(keywords) + for keyword in keywords: + for page in text_dict: + if keyword.lower() in text_dict[page].lower(): + if page > 1: + if page < len(text_dict): + page_list.extend([page - 1, page, page + 1]) + else: + page_list.extend([page - 1, page]) + else: + page_list.extend([page, page + 1]) + + page_list = list(set(page_list)) + context = [text_dict[page] for page in page_list] + context = "\n".join(context) + + return context, page_list + + +def create_prompt(context, question): + return f""" + + Human: Use the following pieces of context to provide a concise answer to the questions at the end. You must answer in correct JSON format. + + {context} + + Question: {question} + + Assistant: Answer in JSON format: {{""" + + +def json_parsing(response_text, context): + print("Parsing JSON response.") + try: + if response_text.split("{", 1)[1].strip()[0] == '"': + response_text = "{" + response_text.split("{", 1)[1] + else: + response_text = "{" + response_text + except: + response_text = "{" + response_text + + if len(response_text.split("}", 1)) > 1: + if response_text.rsplit("}", 1)[0].strip()[-1] == '"': + response_text = response_text.rsplit("}", 1)[0] + "}" + elif response_text.rsplit("}", 1)[0].strip()[-1] == "}": + response_text = response_text.rsplit("}", 1)[0] + else: + response_text = response_text.rstrip(",") + "}" + + try: + response_dict = json.loads(response_text) + except: + response_dict = {"Test field": response_text} + + field_l = list(response_dict.keys()) + answer_l = list(response_dict.values()) + field_dict = {k: v for k, v in response_dict.items() if not k.endswith("_PG")} + page_dict = {k: v for k, v in response_dict.items() if k.endswith("_PG")} + page_dict = {k[:-3]: v for k, v in response_dict.items()} + field_l = list(field_dict.keys()) + answer_l = list(field_dict.values()) + page_no_l = [page_dict.get(x, "") for x in field_l] + + answer_for_snippet = [ + a.lower() if isinstance(a, str) else str(a) if isinstance(a, int) else "" + for a in answer_l + ] + + answer_for_snippet = [ + ( + a[:4] + if "_DT" in f + else ( + "renew" + if f + in ["CONTRACT_AUTO_RENEWAL_IND", "CONTRACT_AUTO_RENEWAL_TERM_LENGTH"] + else ( + "signature" + if f == "CONTRACT_SIGNATORY_IND" + else ( + "hospital" + if f == "CONTRACT_TYPE" + else ( + a[:10] + if f in ["PROV_NPI_OTHER", "PROV_TIN_OTHER"] + else ( + "participat" + if f == "PROV_PARTICIPATION_STATUS" + else ( + "master" + if f == "CONTRACT_CHARGEMASTER_PROTECTION_LANGUAGE" + else a + ) + ) + ) + ) + ) + ) + ) + for a, f in zip(answer_for_snippet, field_l) + ] + + try: + location_l = [ + ( + context.lower().find(a, context.find("Start of Page No. = " + str(p))) + if isinstance(a, str) and a != "" + else -1 + ) + for a, p in zip(answer_for_snippet, page_no_l) + ] + except: + location_l = [ + ( + context.lower().find(answer) + if isinstance(answer, str) and answer != "" + else -1 + ) + for answer in answer_for_snippet + ] + + words_before = [ + " ".join(context[:location].split()[-25:]) if location != -1 else " " + for location in location_l + ] + words_after = [ + " ".join(context[location:].split()[:30]) if location != -1 else " " + for location in location_l + ] + snippet_l = [ + ( + " ".join(context[: location - len(wb) - 1].split(".")[-1:]) + + " " + + wb + + " " + + wa + + " " + + " ".join(context[location + len(wa) + 1 :].split(".")[:1]) + + "." + if location != -1 + else " " + ) + for location, wb, wa in zip(location_l, words_before, words_after) + ] + snippet_l = [snippet.strip() for snippet in snippet_l] + + print(f"Parsed fields: {field_l}") + return field_l, answer_l, page_no_l, location_l, snippet_l + + +def json_parsing_search(response_text, field_list): + try: + if response_text.split("{", 1)[1].strip()[0] == '"': + response_text = "{" + response_text.split("{", 1)[1] + else: + response_text = "{" + response_text + except: + response_text = "{" + response_text + + if len(response_text.split("}", 1)) > 1: + if response_text.rsplit("}", 1)[0].strip()[-1] == '"': + response_text = response_text.rsplit("}", 1)[0] + "}" + elif response_text.rsplit("}", 1)[0].strip()[-1] == "}": + response_text = response_text.rsplit("}", 1)[0] + else: + response_text = response_text.rstrip(",") + "}" + field_l = [] + answer_l = [] + position_dict = {} + for f in field_list: + location = response_text.find('"' + f + '"') + if location != -1: + position_dict[location] = f + field_list = list(dict(sorted(position_dict.items())).values()) + for f in field_list: + if f in response_text: + field_l.append(f) + value = response_text.split('"' + f + '"')[0] + response_text = response_text.split('"' + f + '"')[1] + if f != field_list[0] and f != field_list[-1]: + answer_l.append( + value.strip("\n") + .strip('"') + .strip(":") + .strip(" ") + .strip("\n") + .strip(" ") + .strip(",") + .strip('"') + ) + elif f == field_list[-1]: + answer_l.append( + value.strip("\n") + .strip('"') + .strip(":") + .strip(" ") + .strip("\n") + .strip(" ") + .strip(",") + .strip('"') + ) + answer_l.append( + response_text.strip("\n") + .strip('"') + .strip(":") + .strip(" ") + .strip("}") + .strip("\n") + .strip(" ") + .strip('"') + ) + + return dict(zip(field_l, answer_l)) diff --git a/fieldExtraction/src/ac_smart_chunking.py b/fieldExtraction/src/ac_smart_chunking.py new file mode 100644 index 0000000..49a62c4 --- /dev/null +++ b/fieldExtraction/src/ac_smart_chunking.py @@ -0,0 +1,39 @@ +def chunk_hierarchical(text_dict, key_dict): + # Look for the keywords, one-by-one, and take the first that satisfies the keyword + page_list = [] + for keyword in key_dict["keywords"]: + if len(page_list) == 0: + for page in text_dict.keys(): + if keyword.lower() in text_dict[page].lower(): + if str(int(page) - 1) in text_dict.keys(): + page_list.append(str(int(page) - 1)) + + page_list.append(page) + + if str(int(page) + 1) in text_dict.keys(): + page_list.append(str(int(page) + 1)) + + page_list_sorted = sorted([int(page_str) for page_str in set(page_list)]) + return page_list_sorted + + +def chunk_or(text_dict, key_dict): + # Look for any of the keywords and add any pages with any keywords to the chunk + # When we add a page, we add the previous, current, and next page + page_list = [] + # for page in text_dict.values(): + for page in text_dict.keys(): + for keyword in key_dict["keywords"]: + if keyword.lower() in text_dict[page].lower(): + # Add buffer of previous and next page, if we're not at the beginning nor end + # If we are at the beginning (or the end) just take the next (or previous) page + if str(int(page) - 1) in text_dict.keys(): + page_list.append(str(int(page) - 1)) + + page_list.append(page) + + if str(int(page) + 1) in text_dict.keys(): + page_list.append(str(int(page) + 1)) + + page_list_sorted = sorted([int(page_str) for page_str in set(page_list)]) + return page_list_sorted diff --git a/fieldExtraction/src/bottom_up_funcs.py b/fieldExtraction/src/bottom_up_funcs.py new file mode 100644 index 0000000..72821c1 --- /dev/null +++ b/fieldExtraction/src/bottom_up_funcs.py @@ -0,0 +1,157 @@ +import dict_operations +import postprocessing_funcs +import prompts +import config +import claude_funcs +import utils + +import difflib +import re + + +def run_bottom_up(filename: str, text_dict: dict) -> list[dict]: + """Processes the text of a document using a two-tiered Bottom Up approach to extract key financial and operational information. + + This function first runs BOTTOM_UP_PRIMARY and BOTTOM_UP_SECONDARY prompts, then processes these initial results to further + refine and structure them into a dictionary form. + + Args: + filename (str): The name of the file being processed, used to tag output data. + text_dict (dict): A dictionary of text keyed by page numbers that contains the content to be analyzed. + + Returns: + list[dict]: A list of dictionaries with each dictionary containing refined and structured information + from both primary and secondary Bottom Up analyses, all tagged with the filename. + """ + + # Bottom Up Primary - FULL_SERVICE, FULL_METHODOLOGY + bu_primary_results = run_bottom_up_primary(text_dict, 8000, filename) + + # Bottom Up Secondary + bu_secondary_results = run_bottom_up_secondary( + bu_primary_results, text_dict, 8000, filename + ) + + for d in bu_secondary_results: + d["Filename"] = filename + return bu_secondary_results # List of dictionaries + + +def run_bottom_up_primary( + text_dict: dict, tokens: int, filename: str +) -> dict[str, str]: + """Executes the primary Bottom Up processing for pages that contain reimbursement terms. + + This function scans through a dictionary of page texts, identifies pages containing reimbursement terms, + and processes those pages using Claude to extract relevant information. + + Args: + text_dict (dict): A dictionary containing text content of documents keyed by page numbers. + tokens (int): The maximum number of tokens to use in the language model invocation for generating the response. Deprecated. + filename (str): The filename of the contract in question. Used for logging purposes. + + Returns: + dict[str, str]: A dictionary where each key is a page number (string) and the value is the response from the language model. + """ + + # chunk_dict = preprocess.chunk_text(text_dict) + answer_strings = {} + # for page_number in chunk_dict.keys(): + # if '%' in chunk_dict[page_number] or '$' in chunk_dict[page_number]: + # prompt = prompts.BOTTOM_UP_PRIMARY(chunk_dict[page_number], config.CLIENT_NAME) + for page_number in text_dict.keys(): + if utils.contains_reimbursement(text_dict, page_number): + # Run Primary + prompt = prompts.BOTTOM_UP_PRIMARY( + text_dict[page_number], config.CLIENT_NAME + ) + answer = claude_funcs.invoke_claude( + prompt, config.MODEL_ID_CLAUDE35_SONNET, filename + ) + answer_strings[page_number] = answer + + answer_dicts = dict_operations.primary_string_to_dict(answer_strings, filename) + + answer_dicts = postprocessing_funcs.consolidate_subheader(answer_dicts) + answer_dicts = postprocessing_funcs.filter_service_column( + answer_dicts + ) # Filter on entire Service + Subheader + answer_dicts = postprocessing_funcs.filter_methodology_column(answer_dicts) + + return answer_dicts + + +def run_bottom_up_secondary( + answer_dicts: list[dict], text_dict: dict, tokens: int, filename: str +) -> list[dict]: + """Executes the secondary Bottom Up processing phase on the results obtained from the primary Bottom Up analysis. + + This function enhances the primary results with additional analyses based on configured conditions. Invokes + Claude 3 with tailored prompts to generate structured information that complements the initial results. + + Each piece of data processed possibly undergoes several rounds of checks and transformations, ensuring detailed and + comprehensive output. + + Args: + answer_dicts (list[dict]): Initial processed data from the primary Bottom Up analysis. + text_dict (dict): Dictionary containing the original document text keyed by page numbers. + tokens (int): Token limit for language model invocations. + filename (str): The filename of the contract in question. Used for logging purposes. + + Returns: + list[dict]: A list of dictionaries containing enriched and finalized structured data from both the primary and + secondary analyses. + """ + # Add additional fields to each row + final_dicts = [] + for d in answer_dicts: + if d is not None: + page_num = d["page_num"] + + # if '%' in d['FULL_METHODOLOGY'] or '$' in d['FULL_METHODOLOGY']: + prompt = prompts.BOTTOM_UP_METHODOLOGY_BREAKOUT(d) + answer = claude_funcs.invoke_claude( + prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, max_tokens=512 + ) + answer_dict = dict_operations.secondary_string_to_dict(answer, filename) + d.update(answer_dict) + + # Ensure all bottom up methodology keys are present + for key in [ + "LESSER", + "RATE_STANDARD", + "RATE_SHORT", + "FLAT_FEE_STANDARD", + "LESSER_RATE", + "NOT_TO_EXCEED", + ]: + if key not in d.keys(): + d[key] = "" + + # SHORT_METHODOLOGY + if "%" in d["RATE_STANDARD"]: + d["SHORT_METHODOLOGY"] = get_short_methodology(d["RATE_STANDARD"]) + elif "$" in d["FLAT_FEE_STANDARD"]: + d["SHORT_METHODOLOGY"] = "Flat Fee" + else: + d["SHORT_METHODOLOGY"] = "N/A" + + # Escalator + prompt = prompts.BOTTOM_UP_ESCALATOR(d, text_dict[page_num]) + escalator_answer = claude_funcs.invoke_claude( + prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, max_tokens=1024 + ) + escalator_dict = dict_operations.secondary_string_to_dict( + escalator_answer, filename + ) + d.update(escalator_dict) + + final_dicts.append(d) + + return final_dicts + + +def get_short_methodology(s): + pattern = r"\d+%" + short_string = re.sub(pattern, "%", s) + return short_string diff --git a/fieldExtraction/src/claude_funcs.py b/fieldExtraction/src/claude_funcs.py new file mode 100644 index 0000000..0ec4bc2 --- /dev/null +++ b/fieldExtraction/src/claude_funcs.py @@ -0,0 +1,336 @@ +### EC2 ### + +import json +import anthropic +import config +from botocore.exceptions import ReadTimeoutError, ClientError +import random +import time + +from botocore.exceptions import ClientError +import config + + +def update_stats(filename, tokens_sent, tokens_received, elapsed_time, model_type): + # Update per-file statistics + config.MODEL_STATS[filename]["requests"] += 1 + config.MODEL_STATS[filename]["tokens_sent"] += tokens_sent + config.MODEL_STATS[filename]["tokens_received"] += tokens_received + config.MODEL_STATS[filename]["total_time"] += elapsed_time + + # Update per-file model-specific request count + model_key = f"{model_type}_requests" + config.MODEL_STATS[filename][model_key] = ( + config.MODEL_STATS[filename].get(model_key, 0) + 1 + ) + + # Update global statistics + config.GLOBAL_STATS["total_requests"] = ( + config.GLOBAL_STATS.get("total_requests", 0) + 1 + ) + config.GLOBAL_STATS["total_tokens_sent"] = ( + config.GLOBAL_STATS.get("total_tokens_sent", 0) + tokens_sent + ) + config.GLOBAL_STATS["total_tokens_received"] = ( + config.GLOBAL_STATS.get("total_tokens_received", 0) + tokens_received + ) + config.GLOBAL_STATS["total_time"] = ( + config.GLOBAL_STATS.get("total_time", 0) + elapsed_time + ) + + # Update global model-specific request count + global_model_key = f"total_{model_type}_requests" + config.GLOBAL_STATS[global_model_key] = ( + config.GLOBAL_STATS.get(global_model_key, 0) + 1 + ) + + +def invoke_claude(prompt, model_id, filename, max_tokens=4096): + if config.RUN_MODE == "local": + if model_id == config.MODEL_ID_CLAUDE2: + return local_claude_2(prompt, model_id, filename, max_tokens) + elif model_id == config.MODEL_ID_CLAUDE3_HAIKU: + return local_claude_3(prompt, model_id, filename, max_tokens) + elif model_id == config.MODEL_ID_CLAUDE35_SONNET: + return local_claude_3(prompt, model_id, filename, max_tokens) + elif config.RUN_MODE == "ec2": + if model_id == config.MODEL_ID_CLAUDE2: + return ec2_claude2(prompt, model_id, filename, max_tokens) + elif model_id == config.MODEL_ID_CLAUDE3_HAIKU: + return ec2_claude_3(prompt, model_id, filename, max_tokens) + elif model_id == config.MODEL_ID_CLAUDE35_SONNET: + return ec2_claude_35(prompt, model_id, filename, max_tokens) + else: + print("Usage: python local_main.py <-local> OR python main.py ") + raise + + +# Claude calls +def local_claude_2(prompt, model_id, filename, max_tokens): + + if config.ENABLE_ANSWER_TRACKING and prompt in config.ANSWER_TRACKING_DICT.keys(): + response_text = config.ANSWER_TRACKING_DICT[prompt] + + else: + body = json.dumps( + { + "prompt": anthropic.HUMAN_PROMPT + prompt + anthropic.AI_PROMPT, + "max_tokens_to_sample": max_tokens, + "temperature": 0.0, + "top_p": 1, + "top_k": 250, + "stop_sequences": [anthropic.HUMAN_PROMPT], + } + ) + response = config.BEDROCK_RUNTIME.invoke_model( + body=body, + modelId=model_id, + accept="application/json", + contentType="application/json", + ) + response_body = json.loads(response.get("body").read()) + response_text = response_body["completion"] + + if config.ENABLE_ANSWER_TRACKING: + config.ANSWER_TRACKING_DICT[prompt] = response_text + + return response_text + + +def local_claude_3( + prompt, model_id, filename, max_tokens, max_retries=5, initial_delay=1 +): + + if config.ENABLE_ANSWER_TRACKING and prompt in config.ANSWER_TRACKING_DICT.keys(): + return config.ANSWER_TRACKING_DICT[prompt] + + body = json.dumps( + { + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": max_tokens, + "temperature": 0.0, + "messages": [ + {"role": "user", "content": [{"type": "text", "text": prompt}]} + ], + } + ) + + for attempt in range(max_retries): + try: + response = config.BEDROCK_RUNTIME.invoke_model( + body=body, + modelId=model_id, + accept="application/json", + contentType="application/json", + ) + response_body = json.loads(response.get("body").read()) + response_text = response_body["content"][0]["text"] + + if config.ENABLE_ANSWER_TRACKING: + config.ANSWER_TRACKING_DICT[prompt] = response_text + + return response_text + + except ClientError as e: + if e.response["Error"]["Code"] in [ + "ThrottlingException", + "TooManyRequestsException", + ]: + if attempt < max_retries - 1: + delay = (2**attempt + random.uniform(0, 1)) * initial_delay + time.sleep(delay) + else: + raise + else: + raise + + raise Exception("Max retries exceeded") + + +def ec2_claude2(prompt, model_id, filename, max_tokens): + start_time = time.time() + body = json.dumps( + { + "prompt": anthropic.HUMAN_PROMPT + prompt + anthropic.AI_PROMPT, + "max_tokens_to_sample": max_tokens, + "temperature": 0.0, + "top_p": 1, + "top_k": 250, + "stop_sequences": [anthropic.HUMAN_PROMPT], + } + ) + response = config.BEDROCK_RUNTIME.invoke_model( + body=body, + modelId=config.MODEL_ID_CLAUDE2, + accept="application/json", + contentType="application/json", + ) + response_body = json.loads(response.get("body").read()) + response_text = response_body["completion"] + elapsed_time = time.time() - start_time + tokens_sent = len(prompt.split()) + tokens_received = len(response_text.split()) + + update_stats(filename, tokens_sent, tokens_received, elapsed_time, "claude2") + return response_text + + +def ec2_claude_3( + prompt, model_id, filename, max_tokens, max_retries=5, initial_delay=1 +): + start_time = time.time() + body = json.dumps( + { + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": max_tokens, + "temperature": 0.0, + "messages": [ + {"role": "user", "content": [{"type": "text", "text": prompt}]} + ], + } + ) + dev_runtime = config.assume_role_and_get_runtime( + config.dev_account_id, config.dev_role_name + ) + prod_runtime = config.assume_role_and_get_runtime( + config.prod_account_id, config.prod_role_name + ) + runtimes = { + "UAT": config.BEDROCK_RUNTIME, # This is the default runtime (UAT) + "DEV": dev_runtime, + "PROD": prod_runtime, + } + for runtime_key in runtimes.keys(): + current_runtime = runtimes[runtime_key] + for attempt in range(max_retries): + try: + response = current_runtime.invoke_model( + body=body, + modelId=model_id, + accept="application/json", + contentType="application/json", + ) + response_body = json.loads(response.get("body").read()) + response_text = response_body["content"][0]["text"] + + elapsed_time = time.time() - start_time + tokens_sent = len(prompt.split()) + tokens_received = len(response_text.split()) + + if filename: + if model_id == config.MODEL_ID_CLAUDE35_SONNET: + update_stats( + filename, + tokens_sent, + tokens_received, + elapsed_time, + "claude35", + ) + elif model_id == config.MODEL_ID_CLAUDE3_HAIKU: + update_stats( + filename, + tokens_sent, + tokens_received, + elapsed_time, + "claude3", + ) + + return response_text + + except ClientError as e: + error_code = e.response["Error"]["Code"] + if error_code in ["ThrottlingException", "TooManyRequestsException"]: + if attempt < max_retries - 1: + delay = (2**attempt + random.uniform(0, 1)) * initial_delay + print( + f"Attempt {attempt + 1} failed. Retrying in {delay:.2f} seconds..." + ) + time.sleep(delay) + else: + print( + f"Switching to next runtime due to {error_code}: {str(e)}." + ) + break # Switch to the next runtime + else: + print(f"Unexpected error: {str(e)}") + raise # Reraise other exceptions + + except Exception as e: + print(f"Unexpected error: {str(e)}") + raise + + raise Exception("Max retries reached. Unable to get a response from the model.") + + +def ec2_claude_35( + prompt, model_id, filename, max_tokens, max_retries=5, initial_delay=1 +): + start_time = time.time() + body = json.dumps( + { + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": max_tokens, + "temperature": 0.0, + "messages": [ + {"role": "user", "content": [{"type": "text", "text": prompt}]} + ], + } + ) + dev_runtime = config.assume_role_and_get_runtime( + config.dev_account_id, config.dev_role_name + ) + prod_runtime = config.assume_role_and_get_runtime( + config.prod_account_id, config.prod_role_name + ) + runtimes = { + "UAT": config.BEDROCK_RUNTIME, # This is the default runtime (UAT) + "DEV": dev_runtime, + "PROD": prod_runtime, + } + for runtime_key in runtimes.keys(): + current_runtime = runtimes[runtime_key] + for attempt in range(max_retries): + try: + response = current_runtime.invoke_model( + body=body, + modelId=model_id, + accept="application/json", + contentType="application/json", + ) + response_body = json.loads(response.get("body").read()) + response_text = response_body["content"][0]["text"] + + elapsed_time = time.time() - start_time + tokens_sent = len(prompt.split()) + tokens_received = len(response_text.split()) + + if filename: + update_stats( + filename, tokens_sent, tokens_received, elapsed_time, "claude35" + ) + + return response_text + + except (ReadTimeoutError, ClientError) as e: + error_code = e.response["Error"]["Code"] + if error_code in ["ThrottlingException", "TooManyRequestsException"]: + if attempt < max_retries - 1: + delay = (2**attempt + random.uniform(0, 1)) * initial_delay + print( + f"Attempt {attempt + 1} failed. Retrying in {delay:.2f} seconds..." + ) + time.sleep(delay) + else: + print( + f"Switching to next runtime due to {error_code}: {str(e)}." + ) + break # Switch to the next runtime + else: + print(f"Unexpected error: {str(e)}") + raise # Reraise other exceptions + + except Exception as e: + print(f"Unexpected error: {str(e)}") + raise + + raise Exception("Max retries reached. Unable to get a response from the model.") diff --git a/fieldExtraction/src/compare_output.py b/fieldExtraction/src/compare_output.py new file mode 100644 index 0000000..eae50e2 --- /dev/null +++ b/fieldExtraction/src/compare_output.py @@ -0,0 +1,49 @@ +import pandas as pd + +import os + + +def read_and_compare_csvs(csv_file1, csv_file2): + # Read CSV files into pandas DataFrames + df1 = pd.read_csv(csv_file1) + df2 = pd.read_csv(csv_file2) + + df2 = df2[df1.columns] + + # Check for any differences + diff = (df1 != df2) & df1.notnull() & df2.notnull() + + # Create a writer for Excel + writer = pd.ExcelWriter("data/differences.xlsx", engine="xlsxwriter") + df2.to_excel(writer, sheet_name="Differences", index=False) + + # Get the xlsxwriter workbook and worksheet objects + workbook = writer.book + worksheet = writer.sheets["Differences"] + + # Define a format for changed cells + highlight_fmt = workbook.add_format({"font_color": "red", "bg_color": "yellow"}) + + # Apply formatting to the changed cells + for col in df1.columns: + col_idx = ( + df1.columns.get_loc(col) + 1 + ) # Excel index starts from 1, and there is an index column + for row in diff.index[diff[col]]: + value = df2.at[row, col] + if pd.isna(value): + value = ( + "NaN" # Change this if you want a different representation for NaN + ) + worksheet.write(row + 1, col_idx, value, highlight_fmt) + + # Close the Pandas Excel writer and output the Excel file + writer.close() + + +# Example usage +base_path = "output/57-2883440-Cordia Anderson-Hopkins LCSW-ICMProviderAgreement_221673" +read_and_compare_csvs( + os.path.join(base_path, "combined_results_post_processed.csv"), + os.path.join(base_path, "test_3.5.csv"), +) diff --git a/fieldExtraction/src/conditional_funcs.py b/fieldExtraction/src/conditional_funcs.py new file mode 100644 index 0000000..f0611a0 --- /dev/null +++ b/fieldExtraction/src/conditional_funcs.py @@ -0,0 +1,108 @@ +import prompts +import claude_funcs +import dict_operations +import config + + +def run_conditional(combined_results, text_dict, filename): + + dsh_dict, rbrvs_dict, sl_dict = {}, {}, {} + + for d in combined_results: + if d is not None: + page_num = d["page_num"] + + # LOB/Program Check + if "," in d["CONTRACT_LOB"]: + prompt = prompts.CONDITIONAL_LOB_CHECK(d, text_dict[page_num]) + lob_answer = claude_funcs.invoke_claude( + prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, max_tokens=128 + ) + d["CONTRACT_LOB"] = lob_answer + # lob_dict = dict_operations.secondary_string_to_dict(lob_answer, filename) + # d.update(lob_dict) + + # Prov Type Level 2 + prompt = prompts.CONDITIONAL_PROV_2(d, text_dict[page_num]) + prov2_answer = claude_funcs.invoke_claude( + prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, max_tokens=128 + ) + d["PROV_TYPE_LEVEL_2"] = prov2_answer.strip() + + # # Inclusion of essential RBRVS "Fee Source" Language (Y/N) - Top Down approach to be used for non-NY + if "NY" not in config.STATE: + prompt = prompts.CONDITIONAL_RBRVS(text_dict[page_num]) + rbrvs_answer = claude_funcs.invoke_claude( + prompt, config.MODEL_ID_CLAUDE2, filename, max_tokens=16 + ) + # print(rbrvs_answer) + d['Inclusion of essential RBRVS "Fee Source" Language (Y/N)'] = ( + rbrvs_answer.strip() + ) + rbrvs_dict[page_num] = rbrvs_answer + else: + if "RBRVS" in d["FULL_METHODOLOGY"]: + d['Inclusion of essential RBRVS "Fee Source" Language (Y/N)'] = "Y" + else: + d['Inclusion of essential RBRVS "Fee Source" Language (Y/N)'] = "N" + + # IP_OP + if "FACILITY" in d["PROV_TYPE"].upper(): + prompt = prompts.CONDITIONAL_IP_OP_SMALL( + d["FULL_SERVICE"], d["FULL_METHODOLOGY"] + ) + ip_op_answer = claude_funcs.invoke_claude( + prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, max_tokens=16 + ) + if ip_op_answer.strip() in ["IP", "OP"]: + d["IP_OP"] = ip_op_answer.strip() + else: + d["IP_OP"] = "N/A" + + if "IP" in ip_op_answer: + # IP - DSH/IME/UC, included (Y/N) + if page_num in dsh_dict.keys(): + d["IP - DSH/IME/UC, included (Y/N)"] = dsh_dict[page_num] + else: + prompt = prompts.CONDITIONAL_DSH_IME_UC(text_dict[page_num]) + dsh_answer = claude_funcs.invoke_claude( + prompt, config.MODEL_ID_CLAUDE2, filename, max_tokens=16 + ) + d["IP - DSH/IME/UC, included (Y/N)"] = dsh_answer.strip() + dsh_dict[page_num] = dsh_answer + + # Stop-Loss Catastrophic Threshold + if page_num in sl_dict.keys(): + d["IP - Stoploss Catastrophic Threshold"] = sl_dict[page_num] + else: + if any( + [ + i.upper() in text_dict[page_num].upper() + for i in ["stop loss", "stop-loss", "stoploss"] + ] + ): + prompt = prompts.CONDITIONAL_STOP_LOSS(text_dict[page_num]) + sl_answer = claude_funcs.invoke_claude( + prompt, + config.MODEL_ID_CLAUDE3_HAIKU, + filename, + max_tokens=256, + ) + else: + sl_answer = "N/A" + d["IP - Stoploss Catastrophic Threshold"] = sl_answer + sl_dict[page_num] = sl_answer + else: + d["IP - Stoploss Catastrophic Threshold"] = "N/A" + d["IP - DSH/IME/UC, included (Y/N)"] = "N/A" + else: + d["IP - Stoploss Catastrophic Threshold"] = "N/A" + d["IP - DSH/IME/UC, included (Y/N)"] = "N/A" + d["IP_OP"] = "N/A" + else: + d["IP_OP"] = "N/A" + d["IP - DSH/IME/UC, included (Y/N)"] = "N/A" + d['Inclusion of essential RBRVS "Fee Source" Language (Y/N)'] = "N/A" + d["IP - Stoploss Catastrophic Threshold"] = "N/A" + + return combined_results diff --git a/fieldExtraction/src/config.py b/fieldExtraction/src/config.py new file mode 100644 index 0000000..541b211 --- /dev/null +++ b/fieldExtraction/src/config.py @@ -0,0 +1,202 @@ +from datetime import datetime +import boto3 +from boto3.session import Session +from collections import defaultdict + +from botocore.config import Config + +import sys + +import requests +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry + + +######################################## GENERAL SETTINGS ######################################## +TEST = False # True to run test prompt - just for testing model connection +TODAY = datetime.now().strftime("%Y%m%d") + +######################################## SYSTEM ARGUMENTS ######################################## +# read_mode, run_mode, input_dir, output_dir, consolidated_output_dir + + +def get_arg_value(arg_name, default): + args = [item for item in sys.argv if arg_name == item.split("=")[0]] + if len(args) == 0: + return default + elif len(args) == 1: + return args[0].split("=")[1] + else: + raise Exception(f"Too many argument values: {args}") + + +# Run config args +RUN_MODE = get_arg_value("run_mode", "ec2") # Valid: local or ec2 +READ_MODE = get_arg_value("read_mode", "local") # Valid: local or s3 +BATCH_ID = get_arg_value("batch_id", "test_batch") + +# Field args +FIELDS = get_arg_value("fields", "abc") +AC_FIELD_GROUP = get_arg_value("ac_fields", "all") # Valid: all, part1, part2 + +# Client-specific args +STATE = get_arg_value("state", "None").upper() + +# File IO args +LOCAL_PATH = get_arg_value( + "input_dir", "data/test" +) # Valid: any valid path that contains txt files +S3_PREFIX = get_arg_value("s3_prefix", "") +S3_BUCKET = get_arg_value("s3_bucket", "") +OUTPUT_DIRECTORY = get_arg_value( + "output_dir", "output_individual" +) # Valid: any valid directory +TRACKING_FILE = get_arg_value("tracking_file", f"{BATCH_ID}_tracking.csv") + +# Filtering args +FILTER_ALREADY_PROCESSED = ( + get_arg_value("filter", False) == "True" +) # Valid: True or False +FILTER_ON_CSV = get_arg_value("filter_csv", False) == "True" # Valid: True or False + +# Multithreading args +MAX_WORKERS = int(get_arg_value("max_workers", 2)) # Valid: any integer + + +######################################## CLIENT SETTINGS ######################################## +CLIENT_NAME = "" +PROV_TYPE = None + +######################################## I/O SETTINGS ######################################## +# Input Settings +# LOCAL_PATH = '../priority_non_complex' +# FILTER_ALREADY_PROCESSED = True + +# Output Settings +WRITE_OUTPUT = ( + True # True writes csvs, False prints result in console but no output written +) +# OUTPUT_DIRECTORY = 'output_tx' +CONSOLIDATED_OUTPUT_DIRECTORY = "output_consolidated" +REPORTING_OUTPUT_DIRECTORY = "output_reports" + +UNPROCESSED_RESULTS_NAME = f"{BATCH_ID}-B-Unprocessed.csv" +AC_RESULTS_NAME = f"{BATCH_ID}-AC.csv" +B_RESULTS_NAME = f"{BATCH_ID}-B.csv" +TABLE_ANALYSIS_NAME = f"{BATCH_ID}-Table-Analysis.csv" + + +######################################## AWS SETTINGS ######################################### + +# Bedrock +if RUN_MODE == "local": + AWS_ACCESS_KEY_ID = "ASIA6GBMBVWOF5KYPM7C" + AWS_SECRET_ACCESS_KEY = "OWHMFAJrFSvRXuU0PwrhRSSNFkIfjSOoC9QrtBsj" + AWS_SESSION_TOKEN = "IQoJb3JpZ2luX2VjEB8aCXVzLWVhc3QtMiJHMEUCIQCEVnH/pPAu1c9QI1hqqtd1PQf/p+X8vanzEBfpi4O2XQIgZ7ttYH1o9kFg5WqlMmuH8e6HFgqIW3Bf/9zZLb7zy2IqgQMIeBAAGgw5NzUwNDk5NjA4NjAiDArUh+YuvXdNFA5cSCreAszOXTN5VEblU89ALc5fV10rFF/qPxP0DWuT4zZ69WWCatGfrjV/57/8XYH8Srjmo8Vlr158VBrQza81vhR1PSHVo6gy8RUQTZ40Ma6miOFHCBIIvwyUtTZi2ovaZhVn8foB3H1AS8Os4GIRbKojjp6brw5j3cHmLVNnYFm/O+tXeXu5XGhlr4ZN6Kx53foi8aiXgBNtSgGSqDoEWVL4UKJYZDbt5snhC43LktllWJVXWNsZY3uT5IytVcwFibeDiW9rn0wdJiU7ettJlOiq53Lkn2tgdJnv/gWcFJ2PeLkqoxRPqujaNK15ho6sHz1xgn76aWwTjkHlz30FfRZPVh1nAPbgWBc4xrSUs3K9aHSswqxOmvR94KWzY456LyrDEXXIzPQ0/89VjJSbLTi5mn9TfLt056Sfm5DOUVKxz5h+d62gp8ZK+7twqjz/hsurMHG1pBGMROpOz1e2oCoBMJHNn7gGOqYBcp0cW+KolXEHzFwxIv9vruphUn0ssQnk3DltSCv0BGy2sRlSFppsoSDvCXAS0D0ZZ6LIj6XWrDcfBasUVefsexuNM63sstXH/6MKIlKypso5GrNwwwcLGXwNNwHdv61n4SW/06coz14l+6y23xFjQbck+Scm4LyArpTlPici3m1SspgTfhHbcpUvh7Pp1JEnbY5diVA62Spo9ISBg7E87y1IG0OgYw==" + config = Config(read_timeout=2000) + BEDROCK_RUNTIME = boto3.client( + service_name="bedrock-runtime", + region_name="us-east-1", + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + aws_session_token=AWS_SESSION_TOKEN, + config=config, + ) + +elif RUN_MODE == "ec2": + config = Config(read_timeout=2000, max_pool_connections=60) + BEDROCK_RUNTIME = boto3.client( + service_name="bedrock-runtime", region_name="us-east-1", config=config + ) + + # S3 + S3_CLIENT = boto3.client("s3", region_name="us-east-2") + + +# Claude model IDs +MODEL_ID_CLAUDE3_HAIKU = "anthropic.claude-3-haiku-20240307-v1:0" +MODEL_ID_CLAUDE35_SONNET = "anthropic.claude-3-5-sonnet-20240620-v1:0" +MODEL_ID_CLAUDE2 = "anthropic.claude-instant-v1" + +######################################## CLAUDE AND PROMPT TRACKING ######################################## + +ENABLE_ANSWER_TRACKING = False +ANSWER_TRACKING_DICT = {} + + +######################################## MULTITHREAD SETTINGS ######################################## +# MAX_WORKERS = 10 + +######################################## DEBUG SETTINGS ######################################## +RUN_PRIMARY = True # Always True +RUN_LOB = True +RUN_LESSER = True +RUN_METHODOLOGY = True +RUN_FS = True +RUN_EXCEPTION = True +RUN_CODES = True + +######################################## POSTPROCESSING SETTINGS ######################################## +FUZZY_MATCH_THRESHOLD = 0.8 + +######################################## COMPLEX FLAG SETTINGS ######################################## +COMPLEX_MAX_WORKERS = 50 +RATE_THRESHOLD = 10 +TABLE_THRESHOLD = 2 +COMPLEX_OUTPUT_PATH = "complex_flag_output" +COMPLEX_OUTPUT_FILENAME = f"{TODAY}_complex_flag_test.csv" + +######################################## MESSAGING SETTINGS ######################################## +FINALIZATION_MESSAGE = f""" +""" + + +######################################## EC2 ######################################## +# BEDROCK_RUNTIME = boto3.client(service_name="bedrock-runtime", region_name="us-east-1", config = config) + +MODEL_STATS = defaultdict( + lambda: { + "requests": 0, + "tokens_sent": 0, + "tokens_received": 0, + "total_time": 0, + "claude2_requests": 0, + "claude3_requests": 0, + "claude35_requests": 0, + } +) + +GLOBAL_STATS = { + "total_requests": 0, + "total_tokens_sent": 0, + "total_tokens_received": 0, + "total_time": 0, + "total_claude2_requests": 0, + "total_claude3_requests": 0, + "total_claude35_requests": 0, +} + + +def assume_role_and_get_runtime(account_id, role_name): + sts_client = boto3.client("sts") + role_arn = f"arn:aws:iam::{account_id}:role/{role_name}" + assumed_role = sts_client.assume_role( + RoleArn=role_arn, RoleSessionName=f"UATLambdaSession" + ) + credentials = assumed_role["Credentials"] + runtime = boto3.client( + service_name="bedrock-runtime", + region_name="us-east-1", + aws_access_key_id=credentials["AccessKeyId"], + aws_secret_access_key=credentials["SecretAccessKey"], + aws_session_token=credentials["SessionToken"], + config=config, + ) + return runtime + + +dev_account_id = "660131068782" +dev_role_name = "doczy-dev-bedrock-lambda-role" + +prod_account_id = "211125720533" +prod_role_name = "doczy-prod-bedrock-lambda-role" diff --git a/fieldExtraction/src/consolidate_output.py b/fieldExtraction/src/consolidate_output.py new file mode 100644 index 0000000..97e9ed3 --- /dev/null +++ b/fieldExtraction/src/consolidate_output.py @@ -0,0 +1,58 @@ +import os +import pandas as pd + +import config +import valid + + +def consolidate_output(): + + # If AC only + if "a" in config.FIELDS and "c" in config.FIELDS: + ac_dfs = [] + for subdir, dirs, files in os.walk(config.OUTPUT_DIRECTORY): + ac_path = os.path.join(subdir, config.AC_RESULTS_NAME) + if os.path.exists(ac_path): + try: + ac_df = pd.read_csv(ac_path) + ac_dfs.append(ac_df) + except: + pass + ac_final_df = pd.concat(ac_dfs, ignore_index=True) + ac_final_df.to_csv( + os.path.join( + config.CONSOLIDATED_OUTPUT_DIRECTORY, f"{config.BATCH_ID}-AC.csv" + ) + ) + + # If B only + if "b" in config.FIELDS: + b_dfs = [] + for subdir, dirs, files in os.walk(config.OUTPUT_DIRECTORY): + b_path = os.path.join(subdir, config.B_RESULTS_NAME) + if os.path.exists(b_path): + try: + b_df = pd.read_csv(b_path) + b_dfs.append(b_df) + except: + pass + b_final_df = pd.concat(b_dfs, ignore_index=True) + b_final_df.to_csv( + os.path.join( + config.CONSOLIDATED_OUTPUT_DIRECTORY, f"{config.BATCH_ID}-B.csv" + ) + ) + + # If ABC + if "a" in config.FIELDS and "c" in config.FIELDS and "b" in config.FIELDS: + abc_path = os.path.join( + config.CONSOLIDATED_OUTPUT_DIRECTORY, f"{config.BATCH_ID}-ABC.csv" + ) + abc_final_df = pd.merge( + b_final_df, ac_final_df, on="Contract Name", how="right" + ) + abc_final_df = abc_final_df[ + [col for col in valid.ABC_COLUMNS if col in abc_final_df] + ] + abc_final_df.to_csv(abc_path) + print(f"Consolidation complete. Results written to {abc_path}") diff --git a/fieldExtraction/src/detect_complex.py b/fieldExtraction/src/detect_complex.py new file mode 100644 index 0000000..b334efc --- /dev/null +++ b/fieldExtraction/src/detect_complex.py @@ -0,0 +1,99 @@ +import concurrent.futures +import traceback +import re +import os +import pandas as pd + +import utils +import config +import preprocessing_funcs + +COMPLEX_LIST = [] + + +def count_reimbursements(page_text: str) -> int: + """Count number of reimbursement items found on a page text, through use of a + regex that catches either '$' or '%'. + + Args: + page_text (str): Input page text, as a string + + Returns: + int: Number of reimbursement items + """ + pattern = r"[%$]" + matches = re.findall(pattern, page_text) + return len(matches) + + +def detect_complex(filename, contract_text): + print(f"Processing {filename}") + contract_text = preprocessing_funcs.clean_newlines(contract_text) + contract_text = preprocessing_funcs.clean_law_symbols(contract_text) + text_dict = preprocessing_funcs.split_text(contract_text) + rate_dict = {} + table_dict = {} + for page_num, page_text in text_dict.items(): + rate_dict[page_num] = count_reimbursements(page_text) + table_dict[page_num] = "-------Table Start--------" in page_text + + page_count, consecutive_page_count, consecutive_rate_count = 0, 0, 0 + + for page_num in table_dict.keys(): + if table_dict[page_num] and rate_dict[page_num] > 0: + consecutive_rate_count += rate_dict[page_num] + consecutive_page_count += 1 + else: + if consecutive_rate_count >= config.RATE_THRESHOLD: + page_count += consecutive_page_count + consecutive_page_count, consecutive_rate_count = 0, 0 + + COMPLEX_LIST.append( + {"Filename": filename, "COMPLEX_FLAG": page_count >= config.TABLE_THRESHOLD} + ) + + +def process_item(contract): + data = s3_client.get_object(Bucket=config.BUCKET, Key=contract) + contents = data["Body"].read() + contract_text = contents.decode("utf-8") + + detect_complex(contract, contract_text) + + +s3_client = config.S3_CLIENT + +file_list = [] +paginator = s3_client.get_paginator("list_objects_v2") +pages = paginator.paginate(Bucket=config.BUCKET, Prefix=config.PREFIX) +for page in pages: + for obj in page["Contents"]: + if not obj["Key"].endswith("/"): + file_list.append(obj["Key"]) + +contract_list = sorted(file_list) +print(len(contract_list)) +files = {} + +with concurrent.futures.ThreadPoolExecutor( + max_workers=config.COMPLEX_MAX_WORKERS +) as executor: + # Process each item individually + futures = [executor.submit(process_item, contract) for contract in contract_list] + + # Wait for all futures to complete + for future in concurrent.futures.as_completed(futures): + # Retrieve any exceptions raised by the thread + try: + future.result() + except Exception as e: + print(f"Error: {e}") + +complex_df = pd.DataFrame(COMPLEX_LIST) +# complex_df.to_csv(os.path.join(config.COMPLEX_OUTPUT_PATH, config.COMPLEX_OUTPUT_FILENAME)) + +complex_df.to_csv( + os.path.join( + config.COMPLEX_OUTPUT_PATH, "TX_priority_tin_complex_categorization.csv" + ) +) diff --git a/fieldExtraction/src/dict_operations.py b/fieldExtraction/src/dict_operations.py new file mode 100644 index 0000000..d36a1f4 --- /dev/null +++ b/fieldExtraction/src/dict_operations.py @@ -0,0 +1,120 @@ +import re +import json +from collections import defaultdict + +import config +import prompts +import claude_funcs + + +def secondary_string_to_dict(dict_string, filename): + """ + Converts a string representation of a dictionary into an actual dictionary object, handling potential formatting issues. + + This function cleans the string by removing specific unwanted characters and markers that may interfere with JSON parsing. + It then locates the substring that correctly forms a dictionary format, attempts to parse it as JSON, and handles common + parsing errors by replacing problematic single quotes with double quotes before re-parsing. + + Parameters: + dict_string (str): The string containing the dictionary-like content, potentially surrounded by extra text or characters. + + Returns: + dict: The dictionary obtained from parsing the cleaned and corrected string. + """ + start_index = dict_string.find("{") + end_index = dict_string.rfind("}") + 1 + dict_substring = dict_string[start_index:end_index] + try: + result_dict = json.loads(dict_substring) + except: + try: + dict_substring = dict_substring.replace("'", '"') + result_dict = json.loads(dict_substring) + except: + try: + prompt = prompts.FIX_JSON(dict_substring) + dict_substring = claude_funcs.invoke_claude( + prompt, config.MODEL_ID_CLAUDE3_HAIKU, filename + ) + result_dict = json.loads(dict_substring) + except Exception as e: + print(e) + result_dict = {} + return result_dict + + +def primary_string_to_dict(string_dict, filename): + """ + Converts a dictionary of strings, where each string represents multiple dictionary entries, into a list of dictionaries, + augmenting each with metadata such as page number and filename. + + This function iterates over each page number's string data, extracting and converting string representations of + dictionaries into actual dictionary objects. It handles and cleans the string format to properly parse it into dictionaries. + All dictionaries are then augmented with their respective page number and the filename before being compiled into a list. + + Parameters: + string_dict (dict): A dictionary where keys are page numbers and values are strings containing multiple dictionary entries. + filename (str): The filename associated with these entries, used to tag each resulting dictionary. + + Returns: + list of dict: A list of dictionaries, each representing data extracted and converted from the input string, tagged with + their page number and filename. + """ + data = [] + pattern = r"\{.*?\}" + for page_num in string_dict.keys(): + primary_list = string_dict[page_num] + dicts = primary_list.split("[")[1] # Strip front + dicts = dicts.split("]")[0] # Strip back + dicts = dicts.replace("\n", "") # Remove new lines + dicts = dicts.replace("<<<", "") + dicts = dicts.replace(">>>", "") + dicts = re.sub(r"(? 1: + previous_page_num = str(int(page_num) - 1) + previous_page_td_dict = find_td_dict(td_results, previous_page_num) + # If there was already a td dict generated for that page + if previous_page_td_dict: + return get_value( + previous_page_td_dict, + bu_dict, + key, + previous_page_num, + text_dict, + td_results, + ) + else: # If a new td prompt is needed + previous_page_td_dict = top_down_funcs.run_top_down_primary( + text_dict, previous_page_num, bu_dict["Filename"] + ) + td_results.append(previous_page_td_dict) + return get_value( + previous_page_td_dict, + bu_dict, + key, + previous_page_num, + text_dict, + td_results, + ) + else: + return "N/A" # Base case if no previous pages exist + else: + value = td_dict.get(key, "N/A") + if value == "N/A" or value == "": + previous_page_num = str(int(page_num) - 1) + if previous_page_num in text_dict.keys(): + previous_page_td_dict = find_td_dict(td_results, previous_page_num) + # If there was already a td dict generated for that page + if previous_page_td_dict: + return get_value( + previous_page_td_dict, + bu_dict, + key, + previous_page_num, + text_dict, + td_results, + ) + else: # If a new td prompt is needed + previous_page_td_dict = top_down_funcs.run_top_down_primary( + text_dict, previous_page_num, bu_dict["Filename"] + ) + td_results.append(previous_page_td_dict) + return get_value( + previous_page_td_dict, + bu_dict, + key, + previous_page_num, + text_dict, + td_results, + ) + else: + return "N/A" + elif "," in value: + # If this is the first recursion, then SERVICE and METHODOLOGY are on the page. Run same page select + if bu_dict["page_num"] == page_num: + prompt = prompts.MERGE_SELECT_SAME_PAGE( + key, value, bu_dict, text_dict[page_num] + ) + answer = claude_funcs.invoke_claude_3(prompt) + return answer + else: # If this is a recursive call, then SERVICE and METHODOLOGY are not on the page. Run different page select + prompt = prompts.MERGE_SELECT_DIFFERENT_PAGE( + key, value, text_dict[page_num] + ) + answer = claude_funcs.invoke_claude_3(prompt) + return answer + else: + return value + + +def merge_results(td_results, bu_results, text_dict): + merged_results = [] + for bu_dict in bu_results: + page_num = bu_dict["page_num"] + td_dict = find_td_dict(td_results, page_num) + + merged_dict = bu_dict.copy() + + if td_dict: + for key in [k for k in td_dict.keys() if k not in ["page_num", "Filename"]]: + merged_dict[key] = get_value( + td_dict, bu_dict, key, page_num, text_dict, td_results + ) + merged_results.append(merged_dict) + return merged_results diff --git a/fieldExtraction/src/merge_issue_regex.py b/fieldExtraction/src/merge_issue_regex.py new file mode 100644 index 0000000..24015e9 --- /dev/null +++ b/fieldExtraction/src/merge_issue_regex.py @@ -0,0 +1,146 @@ +import os +import re +import csv + + +def split_text(text): + text_list = text.split("Start of Page No. = ") + text_dict = {page.split()[0]: page for page in text_list[1:]} + return text_dict + + +def contains_reimbursement(text, page): + if isinstance(text, dict): + return page.isdigit() and ( + "%" in text[page] or "$" in text[page] or "percent " in text[page].lower() + ) + elif isinstance(text, str): + return "%" in text or "$" in text or "percent " in text.lower() + else: + print("contains_reimbursement - Invalid data type") + return False + + +def search_files(directory): + pattern = re.compile(r"'',\s*'',") + results = [] + total_files = 0 + files_with_pattern = 0 + + for root, dirs, files in os.walk(directory): + for file in files: + if file.endswith(".txt"): + total_files += 1 + file_path = os.path.join(root, file) + try: + with open(file_path, "r", encoding="utf-8") as f: + content = f.read() + pages = split_text(content) + + reimbursement_pages = [ + page + for page in pages + if contains_reimbursement(pages, page) + ] + table_pages = [ + page + for page in reimbursement_pages + if "-------Table Start--------" in pages[page] + ] + + matches = [] + for page in table_pages: + matches.extend(pattern.findall(pages[page])) + + if matches: + files_with_pattern += 1 + results.append( + ( + file_path, + len(matches), + True, + len(reimbursement_pages), + len(table_pages), + ) + ) + else: + results.append( + ( + file_path, + 0, + False, + len(reimbursement_pages), + len(table_pages), + ) + ) + except Exception as e: + print(f"Error reading {file_path}: {str(e)}") + results.append((file_path, 0, False, 0, 0)) + + return results, total_files, files_with_pattern + + +def write_to_csv(results, total_files, files_with_pattern, output_file): + with open(output_file, "w", newline="", encoding="utf-8") as csvfile: + writer = csv.writer(csvfile) + writer.writerow( + [ + "File Name", + "Match Count", + "Contains Pattern", + "Reimbursement Pages", + "Table Pages", + "Percentage of Total", + ] + ) + + for ( + file_path, + match_count, + contains_pattern, + reimbursement_pages, + table_pages, + ) in results: + percentage = (match_count / total_files) * 100 if total_files > 0 else 0 + writer.writerow( + [ + file_path, + match_count, + contains_pattern, + reimbursement_pages, + table_pages, + f"{percentage:.2f}%", + ] + ) + + percentage_with_pattern = ( + (files_with_pattern / total_files) * 100 if total_files > 0 else 0 + ) + writer.writerow(["", "", "", "", "", ""]) + writer.writerow(["Summary", "", "", "", "", ""]) + writer.writerow(["Total Files", total_files, "", "", "", ""]) + writer.writerow( + [ + "Files with Pattern", + files_with_pattern, + "", + "", + "", + f"{percentage_with_pattern:.2f}%", + ] + ) + + +if __name__ == "__main__": + + current_dir = os.path.dirname(os.path.abspath(__file__)) + + facility_dir = "batch1_priority" + + if os.path.isdir(facility_dir): + results, total_files, files_with_pattern = search_files(facility_dir) + output_file = os.path.join(current_dir, "contract_analysis_results.csv") + write_to_csv(results, total_files, files_with_pattern, output_file) + print(f"Results have been written to {output_file}") + else: + print(f"Error: The directory {facility_dir} does not exist.") diff --git a/fieldExtraction/src/postprocess.py b/fieldExtraction/src/postprocess.py new file mode 100644 index 0000000..3c05023 --- /dev/null +++ b/fieldExtraction/src/postprocess.py @@ -0,0 +1,68 @@ +import pandas as pd +import re +import os +import numpy as np + +import postprocessing_funcs +import valid +import prompts +import claude_funcs +import config +import utils + + +def b_postprocess(filename, df, pages): + if df.shape[0] > 0: + # Metadata fields + df["Contract Name"] = filename + df["Parent Agreement Code"] = postprocessing_funcs.get_parent_agreement_code( + filename + ) + df["Pages"] = pages + + # Add Single Code, Multiple Rate + df = postprocessing_funcs.add_scmr(df) + + # Filter Add Ons + df = postprocessing_funcs.filter_add_ons(df) + + # Clean MSR + df = postprocessing_funcs.clean_msr_lesser(df) + + # Clean Default + df = postprocessing_funcs.clean_default_term(df) + + # Clean Prov 2 + df = postprocessing_funcs.clean_prov_2(df) + + # Clean Lesser Of Rate + df = postprocessing_funcs.clean_lesser_rate(df) + + # Clean LOB + df = postprocessing_funcs.clean_lob(df, filename) + + # Rename and reorder + df.rename(columns=valid.B_MAPPING, inplace=True) + column_order = [col for col in valid.B_MAPPING.values() if col in df.columns] + final_df = df[column_order] + return final_df + else: + return df + + +def ac_postprocess(df, pages): + if df.shape[0] > 0: + if "b" not in config.FIELDS: + df["Pages"] = pages + + df = postprocessing_funcs.clean_ac_fields(df) + df = df.apply(postprocessing_funcs.derive_indicators, axis=1) + df = df.rename(columns=valid.AC_MAPPING) + + column_order = [col for col in valid.ABC_COLUMNS if col in df.columns] + [ + col for col in df.columns if col not in valid.ABC_COLUMNS + ] + final_df = df[column_order] + return final_df + else: + return df diff --git a/fieldExtraction/src/postprocessing_funcs.py b/fieldExtraction/src/postprocessing_funcs.py new file mode 100644 index 0000000..2bfe622 --- /dev/null +++ b/fieldExtraction/src/postprocessing_funcs.py @@ -0,0 +1,502 @@ +import pandas as pd +import numpy as np +import re +import difflib + +import config +import prompts +import valid +from valid import DERIVED_INDICATOR_FIELDS +import claude_funcs +from utils import is_empty + +import logging + +logging.basicConfig( + level=logging.ERROR, format="%(asctime)s - %(levelname)s - %(message)s" +) + + +def sanitize_value(value): + try: + if isinstance(value, list): + return ", ".join(str(v) for v in value) + elif isinstance(value, str): + value = value.strip("[]") + return ", ".join([item.strip(" '") for item in value.split(",")]) + elif pd.isna(value): + return "N/A" + except: + return value + + +def exact_match(val, valid_values): + val = val.strip().upper() + for valid_val in valid_values: + if val == valid_val.upper(): + return valid_val + return None + + +def get_closest_match(val, valid_values, similarity_threshold=0.7): + if pd.isna(val): + return None + val = val.strip().upper() + matches = difflib.get_close_matches( + val, [v.upper() for v in valid_values], n=1, cutoff=similarity_threshold + ) + return matches[0] if matches else None + + +def correct_misplaced_values(df, columns, valid_values_dict): + for index, row in df.iterrows(): + for col in columns: + if pd.notna(row[col]): + terms = row[col].split(",") + for term in terms: + term = term.strip() + for target_col, valid_values in valid_values_dict.items(): + if target_col != col: + match = exact_match(term, valid_values) + if match: + if ( + pd.isna(row[target_col]) + or not row[target_col].strip() + ): + df.at[index, target_col] = match + df.at[index, col] = None + else: + current_value = row[target_col].strip() + if ( + get_closest_match( + match, + [current_value], + config.FUZZY_MATCH_THRESHOLD, + ) + is None + ): + df.at[index, "Corrected_" + target_col] = ( + f"Found {term} in {col} cell" + ) + df.at[index, col] = None + return df + + +def filter_service_column(answer_dicts): + filtered_list = [] + for d in answer_dicts: + clean_dict = True + for keyword in valid.SERVICE_FILTER: + if ( + keyword.upper() in d["FULL_SERVICE"].upper() + or d["FULL_SERVICE"].upper() in keyword.upper() + ): + clean_dict = False + if clean_dict: + filtered_list.append(d) + return filtered_list + + +def filter_methodology_column(answer_dicts): + filtered_list = [] + for d in answer_dicts: + clean_dict = True + for keyword in valid.METHODOLOGY_FILTER: + if ( + keyword.upper() in d["FULL_METHODOLOGY"].upper() + or d["FULL_METHODOLOGY"].upper() in keyword.upper() + ): + clean_dict = False + if clean_dict: + filtered_list.append(d) + return filtered_list + + +def clean_td(td): + td_clean = [] + for d in td: + new_d = {} + for k, v in d.items(): + if "DATE" in k: + new_d[k] = v if isinstance(v, list) else [v] + elif k not in ["page_num", "Filename"]: + if isinstance(v, str) and "," in v: + new_d[k] = [item.strip() for item in v.split(",")] + elif v == "N/A": + new_d[k] = [] + else: + new_d[k] = [v] if isinstance(v, str) else v + else: + new_d[k] = v + td_clean.append(new_d) + return td_clean + + +def get_parent_agreement_code(filename): + try: + filename = filename.split(".txt")[0] + match = re.search(r"([^\sa-zA-Z]+)(?=\.\w+$|$)", filename) + end = [i for i in match.group(1).split("_") if i] + return end[0] + except: + return "N/A" + + +def consolidate_subheader(dict_list): + modified_list = [] + for d in dict_list: + if "FULL_SERVICE" in d and "SUBHEADER" in d: + if d["SUBHEADER"] != "N/A": + d["FULL_SERVICE"] = d["SUBHEADER"] + " - " + d["FULL_SERVICE"] + # Remove the 'SUBHEADER' key + del d["SUBHEADER"] + modified_list.append(d) + return modified_list + + +def clean_msr_lesser(df, ls=valid.VALID_MSR): + pattern = "|".join(re.escape(item) for item in ls) + mask = df["FULL_SERVICE"].str.contains(pattern, case=False, na=False) + target_rows = df[mask] + + # Iterate over these rows + for index, row in target_rows.iterrows(): + # Get all rows with the same 'EXHIBIT' value + exhibit_rows = df[df["EXHIBIT"] == row["EXHIBIT"]] + + # Check the 'LESSER' values of these rows + if (exhibit_rows["LESSER"] == "Y").any(): + # If any row has 'LESSER' == 'Y', set the 'LESSER' value of the original row to 'Y' + df.at[index, "LESSER"] = "Y" + + # Move first LESSER_RATE where LESSER==Y to MSR row + lesser_rate = exhibit_rows[exhibit_rows["LESSER"] == "Y"][ + "LESSER_RATE" + ].iloc[0] + df.at[index, "LESSER_RATE"] = lesser_rate + return df + + +def clean_default_term(df, ls=valid.INVALID_DEFAULT): + # Create a case-insensitive regex pattern that matches any of the values in ls + pattern = "|".join(re.escape(term) for term in ls) + + # Find rows where the DEFAULT_TERM column contains any of the terms from ls + df["DEFAULT_TERM"] = df["DEFAULT_TERM"].astype(str) + mask = df["DEFAULT_TERM"].str.contains(pattern, case=False, na=False) + + # Replace these values with 'N/A' + df.loc[mask, "DEFAULT_TERM"] = "N/A" + df.loc[mask, "DEFAULT_RATE"] = "N/A" + + return df + + +def clean_lesser_rate(df): + for index, row in df.iterrows(): + lesser = str(row["LESSER"]) if not pd.isna(row["LESSER"]) else "" + lesser_rate = row["LESSER_RATE"] + rate_standard = row["RATE_STANDARD"] + + if "Y" in lesser and is_empty(lesser_rate) and not is_empty(rate_standard): + df.at[index, "LESSER_RATE"] = row["RATE_STANDARD"] + df.at[index, "RATE_STANDARD"] = "N/A" + return df + + +def clean_prov_2(df): + valid_types = valid.select_valid_prov_2(df["PROV_TYPE"]) + target_rows = df[df["PROV_TYPE_LEVEL_2"].apply(is_empty)] + + def find_exact_match(text): + if pd.isna(text) or text == "": + return None + + words = re.findall(r"\b[\w/]+(?:[-\s][\w/]+)*\b", text) + for i in range(len(words)): + for j in range(i + 1, len(words) + 1): + phrase = " ".join(words[i:j]) + if phrase in valid_types: # Case-sensitive matching + return phrase + return None + + for index, row in target_rows.iterrows(): + match = None + + if not is_empty(row["FULL_SERVICE"]): + match = find_exact_match(str(row["FULL_SERVICE"])) + if match: + logging.info(f"Row {index}: Matched in FULL_SERVICE: {match}") + df.at[index, "PROV_TYPE_LEVEL_2"] = match + continue + + if not is_empty(row["EXHIBIT"]): + match = find_exact_match(str(row["EXHIBIT"])) + if match: + logging.info(f"Row {index}: Matched in EXHIBIT: {match}") + df.at[index, "PROV_TYPE_LEVEL_2"] = match + continue + + exhibit_rows = df[df["EXHIBIT"] == row["EXHIBIT"]] + if not exhibit_rows.empty: + for _, exhibit_row in exhibit_rows.iterrows(): + if not is_empty(exhibit_row["PROV_TYPE_LEVEL_2"]): + match = find_exact_match(str(exhibit_row["PROV_TYPE_LEVEL_2"])) + if match: + logging.info( + f"Row {index}: Matched in other row's PROV_TYPE_LEVEL_2: {match}" + ) + df.at[index, "PROV_TYPE_LEVEL_2"] = match + break + elif not is_empty(exhibit_row["FULL_SERVICE"]): + match = find_exact_match(str(exhibit_row["FULL_SERVICE"])) + if match: + logging.info( + f"Row {index}: Matched in other row's FULL_SERVICE: {match}" + ) + df.at[index, "PROV_TYPE_LEVEL_2"] = match + break + + if match is None: + logging.warning(f"Row {index}: No valid match found") + + # Final check to ensure no invalid values were assigned + invalid_assignments = df[ + (~df["PROV_TYPE_LEVEL_2"].isin(valid_types)) + & (~df["PROV_TYPE_LEVEL_2"].apply(is_empty)) + ] + if not invalid_assignments.empty: + logging.warning("Invalid assignments found:") + for idx, row in invalid_assignments.iterrows(): + logging.warning( + f"Row {idx}: Invalid PROV_TYPE_LEVEL_2: {row['PROV_TYPE_LEVEL_2']}" + ) + logging.warning(f" FULL_SERVICE: {row['FULL_SERVICE']}") + logging.warning(f" EXHIBIT: {row['EXHIBIT']}") + + df.loc[invalid_assignments.index, "PROV_TYPE_LEVEL_2"] = "" + + return df + + +def clean_ac_fields(final_df): + # additional post-processing + final_df = final_df.fillna("") + # final_df.loc[final_df['CREDENTIALING_APP_IND'] != 'Yes', 'CREDENTIALING_APP_IND'] = "No" + # final_df.loc[final_df['AFFILIATION_CLAUSE_IND'] != 'Yes', 'AFFILIATION_CLAUSE_IND'] = "No" + # final_df.loc[final_df['ASSIGNMENTS_CLAUSE_IND'] != 'Yes', 'ASSIGNMENTS_CLAUSE_IND'] = "No" + if "TERM_CLAUSE" in final_df: + final_df.loc[ + final_df["TERM_CLAUSE"].str.startswith("IL-4 Termination"), "TERM_CLAUSE" + ] = "N/A" + final_df.loc[ + final_df["TERM_CLAUSE"].str.startswith("bonus payment shall be effective"), + "TERM_CLAUSE", + ] = "N/A" + final_df.loc[ + final_df["TERM_CLAUSE"].str.contains("does not contain"), "TERM_CLAUSE" + ] = "N/A" + final_df.loc[ + final_df["TERM_CLAUSE"].str.contains("No term or termination"), + "TERM_CLAUSE", + ] = "N/A" + # if 'CONTRACT_SIGNATORY_IND' in final_df and 'PROV_PARTICIPATION_STATUS' in final_df: + # final_df.loc[final_df['CONTRACT_SIGNATORY_IND'] == 'Yes', 'PROV_PARTICIPATION_STATUS'] = "Yes" + if ( + "CONTRACT_AUTO_RENEWAL_IND" in final_df + and "CONTRACT_TERMINATION_DT" in final_df + ): + final_df.loc[ + final_df["CONTRACT_AUTO_RENEWAL_IND"] == "Yes", "CONTRACT_TERMINATION_DT" + ] = np.nan + + # check if NPI has 10 digits + if "PROV_GROUP_NPI" in final_df: + final_df["PROV_GROUP_NPI"] = final_df["PROV_GROUP_NPI"].map( + lambda x: x if sum(c.isdigit() for c in str(x) + " ") == 10 else "" + ) + # NETWORK_ACCESS_FEES_IND + if "NETWORK_ACCESS_FEES_IND" in final_df: + final_df.loc[ + ~final_df["NETWORK_ACCESS_FEES_IND"].isin(["N/A", "No", "", " "]), + "NETWORK_ACCESS_FEES_IND", + ] = "Yes" + + if "PAYER_NAME" in final_df and "HEALTH_PLAN_STATE" in final_df: + final_df.loc[ + final_df["PAYER_NAME"].str.startswith("Illini"), "HEALTH_PLAN_STATE" + ] = "Illinois" + if "HEALTH_PLAN_STATE" in final_df: + final_df["HEALTH_PLAN_STATE"] = ( + final_df["HEALTH_PLAN_STATE"] + .map(valid.STATE_MAP) + .fillna(final_df["HEALTH_PLAN_STATE"]) + ) + + if "NOTICE_PROVIDER_NAME" in final_df and "NOTICE_PROVIDER_ADDRESS" in final_df: + final_df.loc[ + final_df["NOTICE_PROVIDER_NAME"].str.contains( + "Superior HealthPlan", na=False + ), + "NOTICE_PROVIDER_NAME", + ] = np.nan + final_df.loc[ + final_df["NOTICE_PROVIDER_NAME"].str.contains( + "Superior HealthPlan", na=False + ), + "NOTICE_PROVIDER_ADDRESS", + ] = np.nan + + if "Contract Name" in final_df.columns: + final_df["temp_filename"] = ( + final_df["Contract Name"].str[:10].str.replace("-", "") + ) + if "PROV_GROUP_TIN" in final_df and "Contract Name" in final_df: + final_df.loc[ + (final_df["PROV_GROUP_TIN"].isin([" ", ""])) + & (final_df["temp_filename"].str.isnumeric()), + "PROV_GROUP_TIN", + ] = final_df["temp_filename"] + final_df.drop(columns=["temp_filename"], inplace=True) + elif "Filename" in final_df.columns: + final_df["temp_filename"] = final_df["Filename"].str[:10].str.replace("-", "") + if "PROV_GROUP_TIN" in final_df and "Filename" in final_df: + final_df.loc[ + (final_df["PROV_GROUP_TIN"].isin([" ", ""])) + & (final_df["temp_filename"].str.isnumeric()), + "PROV_GROUP_TIN", + ] = final_df["temp_filename"] + final_df.drop(columns=["temp_filename"], inplace=True) + + # For POLICIES_AND_PROCEDURES, filter out anything without either "policies" or "procedures". + if "POLICIES_AND_PROCEDURES" in final_df: + final_df.loc[ + ~final_df["POLICIES_AND_PROCEDURES"].str.contains( + "polic", flags=re.IGNORECASE + ) + & ~final_df["POLICIES_AND_PROCEDURES"].str.contains( + "procedure", flags=re.IGNORECASE + ), + "POLICIES_AND_PROCEDURES", + ] = "N/A" + + final_df = final_df.apply(lambda x: x.map(replace_null_terms)) + final_df = final_df.apply(lambda x: x.map(replace_quotes)) + + return final_df + + +def replace_quotes(value): + try: + return str(value).replace(r"\"", '"') + except: + return value + + +def replace_null_terms(value): + # Convert value to string and check if any term from NULL_ANSWER_TERMS is in the value + if any(term.lower() in str(value).lower() for term in valid.NULL_ANSWER_TERMS): + return "N/A" + return value + + +def filter_dict(d, pattern): + final_dict = {} + for page_num, answer in d.items(): + if not pattern.search(answer): + final_dict[page_num] = answer + return final_dict + + +def filter_add_ons(df): + pattern = r"(in no event).+(includ.?)|(forward).+(payments)" + + mask = df["ADD_ON_REIMBURSEMENT_LANGUAGE"].str.contains( + pattern, case=False, na=False, regex=True + ) + df.loc[mask, "ADD_ON_REIMBURSEMENT_LANGUAGE"] = "N/A" + df.loc[mask, "ADD_ON_REIMBURSEMENT_IND"] = "N" + + def check_add_on(group): + # Check if any 'SERVICE_TYPE' contains 'Add-on' or 'Add On' + if ( + group["FULL_SERVICE"] + .str.contains("Add-on|Add On", regex=True, case=False, na=False) + .any() + ): + group["ADD_ON_REIMBURSEMENT_LANGUAGE"] = ( + "N/A" # Set 'ADD_ON' to 'N/A' for the whole group + ) + group["ADD_ON_REIMBURSEMENT_IND"] = "N" + return group + + # Group by 'Contract Name' and 'EXHIBIT', then apply the check_add_on function + if "Contract Name" in df.columns: + df = df.groupby(["Contract Name", "EXHIBIT"]).apply(check_add_on) + elif "Filename" in df.columns: + df = df.groupby(["Filename", "EXHIBIT"]).apply(check_add_on) + + # TODO : If original add on value is actually an Exclusion, and the Exclusion value for the row is invalid, then move the Add On value to the Exclusions column + return df + + +def add_scmr(df): + def count_dollar_values(s): + return str(s).count("$") + str(s).count("%") + + if "FLAT_FEE_STANDARD" in df.columns: + col = "FLAT_FEE_STANDARD" + elif "FLAT_FEE" in df.columns: + col = "FLAT_FEE" + elif "FLAT FEE" in df.columns: + col = "FLAT FEE" + + df["dollar_count"] = df[col].apply(count_dollar_values) + + df["Single Code Multiple Rates (Language)"] = df.apply( + lambda row: row[col] if row["dollar_count"] > 1 else "N/A", axis=1 + ) + + df["Single Code Multiple Rates (Y/N)"] = df["dollar_count"].apply( + lambda x: "Y" if x > 1 else "N" + ) + df.drop("dollar_count", axis=1, inplace=True) + + return df + + +def clean_lob(df, filename): + def update_contract_lob(row): + try: + # Check if more than one valid lob is present in the 'FULL_SERVICE' column + if ( + sum( + val.lower() in row["FULL_SERVICE"].lower() + for val in valid.VALID_LOBS + ) + > 1 + ): + prompt = prompts.LOB_SWEEPER(row["EXHIBIT"]) + answer = claude_funcs.invoke_claude( + prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 256 + ) + return answer # Return the value from 'EXHIBIT' if condition is met + return row["CONTRACT_LOB"] + except: + return row["CONTRACT_LOB"] + + df["CONTRACT_LOB"] = df.apply(update_contract_lob, axis=1) + return df + + +def derive_indicators(results): + + for field in DERIVED_INDICATOR_FIELDS: + indicator_field = field + "_IND" + if field in results and results[field] and not is_empty(results[field]): + print(f"Y - {indicator_field}") + results[indicator_field] = "Y" + else: + print(f"N - {indicator_field}") + results[indicator_field] = "N" + return results diff --git a/fieldExtraction/src/preprocess.py b/fieldExtraction/src/preprocess.py new file mode 100644 index 0000000..2c44dd7 --- /dev/null +++ b/fieldExtraction/src/preprocess.py @@ -0,0 +1,30 @@ +import preprocessing_funcs +import top_down_funcs +import table_funcs +import config + + +def preprocess(contract_text, filename, fields=config.FIELDS): + + contract_text = preprocessing_funcs.clean_newlines(contract_text) + contract_text = preprocessing_funcs.clean_law_symbols(contract_text) + + text_dict = preprocessing_funcs.split_text( + contract_text + ) # return a dictionary with keys - page_num (str), values as the page_text + + num_pages = len(text_dict.keys()) + text_dict = preprocessing_funcs.filter_quick_review(text_dict) + + if fields == "b": + exhibit_pages = top_down_funcs.get_exhibit_pages( + text_dict, filename + ) # All pages with exhibit headers + text_dict = table_funcs.align_and_format_tables(text_dict, filename) + text_dict = preprocessing_funcs.chunk_consecutive(text_dict, exhibit_pages) + ac_chunks, full_context = "", "" + if fields == "ac": + exhibit_pages = [] + ac_chunks = preprocessing_funcs.smart_chunk_ac(text_dict) + + return text_dict, exhibit_pages, num_pages, ac_chunks diff --git a/fieldExtraction/src/preprocessing_funcs.py b/fieldExtraction/src/preprocessing_funcs.py new file mode 100644 index 0000000..5546a51 --- /dev/null +++ b/fieldExtraction/src/preprocessing_funcs.py @@ -0,0 +1,231 @@ +from itertools import groupby, count +import re +from collections import defaultdict +import utils +import valid +from valid import DERIVED_INDICATOR_FIELDS +from keywords import KEYWORD_MAPPINGS +from collections import defaultdict +import ac_smart_chunking + + +def clean_newlines(contract_text: str) -> str: + """Clean textract output by removing newlines and page number indicators. + + This function processes input text to: + 1. Remove all newline characters. + 2. Remove lines that indicate page numbers (e.g. 'Page 1 of 10') + + Args: + contract_text (str): Raw text output from Textract to be cleaned + + Returns: + str: cleaned text with newlines and page number indicators removed + """ + cleaned_text = re.sub(r"(? dict[str, str]: + """Split text on pages by the string `Start of Page No. = ' + + Args: + text (str): Raw text output from Textract to be split + + Returns: + dict[str, str]: A dictionary, keyed by the string page number and valued by the page text. + """ + temp_list = text.split("Start of Page No. = ") + text_list = re.split(r"Start of Page No. = [0-9]+\n", text) + + text_dict = {} + for i in range(len(text_list)): + text_dict[temp_list[i].split()[0]] = text_list[i] + + return {k: v for k, v in text_dict.items() if k != "Document"} + + +def highlight_rates(text_dict): + for page, text in text_dict.items(): + words = text.split() + highlighted_words = [] + for word in words: + if "%" in word or "$" in word: + word = f">>>{word}<<<" + highlighted_words.append(word) + text_dict[page] = " ".join(highlighted_words) + return text_dict + + +def clean_law_symbols(contract_text): + contract_text = contract_text.replace("$$", "$") + + contract_text = re.sub(r"(U\.?S\.?C\.?) \$", r"\1§", contract_text) + contract_text = re.sub(r"(C\.?F\.?R\.?) \$", r"\1§", contract_text) + + # Second correction: Replace '$' with '§' when followed by a number with three decimal places + contract_text = re.sub(r"\$(?=\d+\.\d{3})", "§", contract_text) + + return contract_text + + +# ORIGINAL +# def chunk_consecutive(text_dict, exhibit_pages): +# # If needed - extend page 1 to page 1 and 2, then cut chunking off after (edge case: compensation terms not found on first page of exhibit) + +# reimbursement_pages = [page_num for page_num in text_dict.keys() if utils.contains_reimbursement(text_dict, page_num)] + +# page_dict = {} +# current_exhibit = None +# for page_num in text_dict.keys(): + +# # Page is Reimbursement AND Exhibit +# if page_num in reimbursement_pages and page_num in exhibit_pages: +# current_exhibit = page_num +# page_dict[page_num] = [page_num] + +# # Page is Reimbursement NOT Exhibit +# elif page_num in reimbursement_pages and page_num not in exhibit_pages: +# if current_exhibit: +# page_dict[current_exhibit].append(page_num) +# else: +# page_dict[page_num] = [page_num] + +# # Page is Exhibit NOT Reimbursement +# elif page_num in exhibit_pages and page_num not in reimbursement_pages: +# current_exhibit = page_num +# page_dict[page_num] = [page_num] + +# # Page is NOT Exhibit NOT Reimbursment +# elif page_num not in exhibit_pages and page_num not in reimbursement_pages: +# current_exhibit = None +# page_dict[page_num] = [page_num] + +# final_dict = {page_num : '' for page_num in page_dict.keys()} +# for page_num in page_dict.keys(): +# for p in page_dict[page_num]: +# final_dict[page_num] += text_dict[p] +# return final_dict + + +def chunk_consecutive(text_dict, exhibit_pages): + exhibit_pages = set(str(page) for page in exhibit_pages) + reimbursement_pages = { + page_num + for page_num in text_dict.keys() + if utils.contains_reimbursement(text_dict, page_num) + } + + page_dict = {} + current_chunk_start = None + last_exhibit = None + in_exhibit = False + + def word_count(text): + return len(text.split()) + + for page_str in sorted(text_dict.keys(), key=int): + page_num = int(page_str) + + if page_str in exhibit_pages: + current_chunk_start = page_str + last_exhibit = page_str + in_exhibit = True + page_dict[current_chunk_start] = [page_str] + # print(f"found page {page_num} in exhibit, starting new chunk") + + if page_str in reimbursement_pages: + if not in_exhibit or current_chunk_start is None: + # Check if this page has less than 200 words and should be added to the previous chunk + if current_chunk_start and word_count(text_dict[page_str]) < 200: + page_dict[current_chunk_start].append(page_str) + # print(f"found page {page_num} in reimbursement with less than 200 words, adding to previous chunk") + else: + current_chunk_start = page_str + page_dict[current_chunk_start] = [page_str] + # print(f"found page {page_num} in reimbursement, starting new chunk") + else: + page_dict[current_chunk_start].append(page_str) + # print(f"found page {page_num} in reimbursement, adding to current chunk") + else: + if in_exhibit and current_chunk_start is not None: + page_dict[current_chunk_start].append(page_str) + # print(f"found non-reimbursement page {page_num} in exhibit, adding to current chunk") + else: + in_exhibit = False + + # Check next page if it's non-reimbursement and not in exhibit + next_page_str = str(page_num + 1) + if ( + next_page_str in text_dict + and next_page_str not in reimbursement_pages + and next_page_str not in exhibit_pages + ): + if current_chunk_start is not None: + page_dict[current_chunk_start].append(next_page_str) + # print(f'next page {next_page_str} was found to be a non-reimbursement and added') + + # If we've moved past the last exhibit page, reset in_exhibit + if in_exhibit and int(page_str) > int(last_exhibit): + in_exhibit = False + + # Remove duplicates and sort page numbers in each chunk + for key in page_dict: + page_dict[key] = sorted(list(set(page_dict[key])), key=int) + + final_dict = {page_num: "" for page_num in page_dict.keys()} + for page_num in page_dict.keys(): + for p in page_dict[page_num]: + final_dict[page_num] += text_dict[p] + + # print("Final page_dict:", {k: v for k, v in page_dict.items()}) + return final_dict + + +def smart_chunk_ac(text_dict: dict, keyword_mappings: dict = KEYWORD_MAPPINGS) -> dict: + """Performs a keyword search on textract outputs to retrieve relevant chunks for a given field. + + Args: + text_dict (dict): Dictionary keyed by string page-number and valued by actual textract output page + keyword_mappings (dict, optional): Dictionary keyed by field, with sub-dictionary with keys `methodology` and `keywords` + `methodology` must be either `or` or `hierarchy. Defaults to keywords.KEYWORD_MAPPINGS + + Raises: + ValueError: When the `key_dict` from keyword_mappings has a methodology that is not implemented. + Current options are `or` and `hierarchy`. + + Returns: + dict: A dictionary keyed by field (defined by the keys in keyword_mappings) valued by + corresponding chunks of text (input strings delimited by newlines). + """ + field_pages = defaultdict(set) # TODO: remove this, it's unused + chunks = defaultdict(str) + + full_context = "\n".join( + [f"Page {page}:\n{content}" for page, content in text_dict.items()] + ) + # print(f"Full context word count: {len(full_context.split())}") + + for field, key_dict in keyword_mappings.items(): + if key_dict["methodology"] == "or": + page_list = ac_smart_chunking.chunk_or(text_dict, key_dict) + elif key_dict["methodology"] == "hierarchy": + page_list = ac_smart_chunking.chunk_hierarchical(text_dict, key_dict) + else: + page_list = list(text_dict.keys()) + raise ValueError( + "methodology must be 'hierarchy' or 'or' - using entire contract as chunk" + ) + chunks[field] = "\n".join([text_dict[str(page_num)] for page_num in page_list]) + return chunks + + +def filter_quick_review(text_dict): + return { + page_num: page_text + for page_num, page_text in text_dict.items() + if "QUICK REVIEW" not in page_text.upper() + and "TOP SHEET" not in page_text.upper() + and "COVER SHEET" not in page_text.upper() + } diff --git a/fieldExtraction/src/prerun.py b/fieldExtraction/src/prerun.py new file mode 100644 index 0000000..163f8cd --- /dev/null +++ b/fieldExtraction/src/prerun.py @@ -0,0 +1,101 @@ +import table_funcs +import config +import utils +import preprocessing_funcs +import concurrent.futures +import pandas as pd +import os + +input_dict = utils.read_input() +table_analysis_dicts = [] + + +def table_analysis(file_object): + filename, contract_text = file_object + text_dict = preprocessing_funcs.split_text(contract_text) + file_dict = {"Filename": filename, "Filename_pdf": filename.replace(".txt", ".pdf")} + + contains_reimbursement = utils.contains_reimbursement(contract_text) + file_dict["Contains Reimbursement"] = contains_reimbursement + + # Determine if the file should be present in the output + file_dict["Present in Output"] = contains_reimbursement + + stats_dict = table_funcs.get_table_stats(text_dict) + file_dict.update(stats_dict) + table_analysis_dicts.append(file_dict) + + +with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor: + futures = [executor.submit(table_analysis, item) for item in input_dict.items()] + for future in concurrent.futures.as_completed(futures): + try: + future.result() + except Exception as e: + print(f"Error: {e}") + +# Create the main table analysis DataFrame +table_analysis_df = pd.DataFrame(table_analysis_dicts) +table_analysis_df.to_csv( + os.path.join(config.REPORTING_OUTPUT_DIRECTORY, config.TABLE_ANALYSIS_NAME) +) + +# Define the columns we want in our report +report_columns = [ + "Filename", + "Contains Reimbursement", + "Present in Output", + "Table Count", + "Table Page Count", + "Rate Count", + "Num Rate Pages", + "Num >=10 Rates", +] + +# Create the new report DataFrame with available columns +report_df = table_analysis_df[ + [col for col in report_columns if col in table_analysis_df.columns] +] + + +# Function to safely create summary columns +def create_summary_column(df, source_col, target_col): + if source_col in df.columns: + df[target_col] = df[source_col].apply( + lambda x: ", ".join(map(str, eval(x))) if x != "[]" else "None" + ) + else: + df[target_col] = "N/A" + + +# Add summary columns if the data is available +create_summary_column(report_df, "Table Pages", "Table Pages Summary") +create_summary_column(report_df, "Rate Pages", "Rate Pages Summary") +create_summary_column(report_df, ">=10 Rate Pages", ">=10 Rate Pages Summary") + +# Define the final column order +final_columns = [ + "Filename", + "Contains Reimbursement", + "Present in Output", + "Table Count", + "Table Page Count", + "Table Pages Summary", + "Rate Count", + "Num Rate Pages", + "Rate Pages Summary", + "Num >=10 Rates", + ">=10 Rate Pages Summary", +] + +# Reorder columns, including only those that exist +report_df = report_df[[col for col in final_columns if col in report_df.columns]] + +# Save the new report to a CSV file +report_output_path = os.path.join( + config.REPORTING_OUTPUT_DIRECTORY, "enhanced_reimbursement_report.csv" +) +report_df.to_csv(report_output_path, index=False) + +print(f"Enhanced reimbursement report saved to: {report_output_path}") +print("Columns in the report:", ", ".join(report_df.columns)) diff --git a/fieldExtraction/src/prompts.py b/fieldExtraction/src/prompts.py new file mode 100644 index 0000000..ea6caf1 --- /dev/null +++ b/fieldExtraction/src/prompts.py @@ -0,0 +1,494 @@ +import config +import valid + + +##################################################################################### +##################################### BOTTOM UP ##################################### +##################################################################################### + + +def BOTTOM_UP_PRIMARY(page, payer): + return f""" +### PAGE START ### {page} ### PAGE END + +The preceding text is one page of a contract between a Payer {payer} and a provider in their network. Your job is to extract attributes related to the reimbursement of different services and specialties. Ensure every compensation term (with a % or $ value) is captured. + +Some values are found in sections explicitly identified as examples - these must be omitted from output. Other values might be related to liability - these must also be excluded. + +Return at least one json object for each combination of attributes seen. It is possible that a page will not have any attributes. When this is the case, return an empty list. + +Here are the attributes to be included in each dictionary, and instructions on how to correctly answer: +'FULL_SERVICE' : What is the Service that is being reimbursed? Include the full Service, including any distinguishing context. Include any codes that are part of the service, including revenue codes and procedure codes. Use only exact text from the document. Do not leave this field N/A. +'SUBHEADER' : What subheader or subsection name does the FULL_SERVICE belong to? This may be include crucial additional detail about the Service. Note that this is not a primary header like that of a compensation schedule or exhibit. If there is no subheader, write 'N/A'. +'FULL_METHODOLOGY' : Write the full sentence or paragraph in the text describing the reimbursement methodology. Ensure any relevant detail is included, including rates in tables if applicable. Include any 'Lesser of' statement that applies to the reimbursement. The 'Lesser of' statement might not be found in immediate proximity to the reimbursement term and may instead be found in a paragraph above. If the methodology is presented in a table, concatenate any relevant lesser of statement that applies to the table with the portion of the methodology found in the table. + +Here are some examples of language with additional context to be included in the FULL_SERVICE: +Text: 'Covered Services that are Medicare Covered Services and are not Medicaid Covered Services', +Text: 'Covered Services rendered to Covered Person who are eligible for Medicare and enrolled in a MMP Plan that may include coverage for both Medicare and Medicaid Covered Services' + +Here are some examples of SUBHEADER values: +Text: 'Where Payor is the Payor for both Medicare Covered Services and Medicaid Covered Services:' +Text: 'Outpatient Services' + +Here is an example of text with a Lesser of statement and a table that should be included as a FULL_METHODOLOGY answer: +Text: 'Covered Services is the lesser of: (i) Allowable Charges; or (ii) the "Contracted Rate" percentage found in Table 1. Table 1 - OB and Anesthesia Services : 100% of the Payor's Oregon Health Plan DMAP fee schedule. Radiology Services : 110% of Medicare Fee Schedule' +Answer: "[{{'FULL_SERVICE' : 'OB and Anesthesia Services', 'SUBHEADER' : 'N/A', 'FULL_METHODOLOGY' : 'lesser of (i) Allowable Charges; or (ii) 100% of the Payor's Oregon Health Plan DMAP fee schedule'}}, +{{'FULL_SERVICE' : 'Radiology Services', 'SUBHEADER' : 'N/A', 'FULL_METHODOLOGY' : 'lesser of (i) Allowable Charges; or (ii) 110% of Medicare Fee Schedule'}}]" + +If there is no clear FULL_SERVICE, then fill in 'Covered Services' for the FULL_SERVICE value. + +Ensure that you do not forget to extract reimbursement items. MAKE SURE THAT YOU DON'T MISS ANY REIMBURSEMENT ITEMS AND RETURN ALL NECESSARY AND RELEVANT JSON OBJECTS. +Only return the list, with no other commentary or explanation. Ensure you abide by proper JSON formatting. +""" + + +def BOTTOM_UP_METHODOLOGY_BREAKOUT(d): + return f"""Analyze the full methodology text listed below: +{d['FULL_METHODOLOGY']} + +Populate a dictionary with the following values: + +LESSER : If the methodology contains the words 'lesser of' or similar, write 'Y'. Otherwise, write 'N'. +RATE_STANDARD : If there is only one rate in the methodology, write that rate here. Use the format 'X% of Y'. Do not include Flat Fees here. If there is no rate in the methodology, write 'N/A'. This value MUST be different than the LESSER_RATE and NOT_TO_EXCEED values. +RATE_SHORT : Return only the percentage from RATE_STANDARD in decimal form, with no other text. For example, 100% -> 1. +FLAT_FEE_STANDARD : If any part of the methodology is a flat fee (e.g. $X), write that value here. Do not include percentages here. If there is a 'Per' statement like 'Per Diem', 'Per Unit', etc, inclue that here as well. If there is no dollar value in the methodology, leave this field N/A. This value MUST be different than the LESSER_RATE and NOT_TO_EXCEED values. +LESSER_RATE : If the methodology contains the words 'lesser of' or similar, what is the rate listed that is NOT the RATE_STANDARD or the FLAT_FEE_STANDARD? Answer in the format 'X% of Y', where X is a numeric value. If there is no 'lesser of' in the text, write 'N/A'. +NOT_TO_EXCEED : Only if the methodology contains the specific phrase "Not to exceed", write the amount that the payment is not to exceed using the template "X% of Y" where X is a numeric value, if the value is a %. If the value is a dollar value, use the format "$X", where X is the numeric value. If "Not to exceed" is not found, write 'N/A'. + +For the format 'X% of Y', if Y is in the below list, then apply the indicated abbreviation. If Y is not in the list, then write the full Y value: +Medicare --> MCR +Medicaid --> MCD +Allowable Charges --> AC +Allowed Amount --> AA +Billed Charges --> BC +Average Sales Price --> ASP +Average Wholesale Price --> AWP +DMAP --> DMAP +Amount Payable by MCR --> MCR +Amount Payable by MCD --> MCD + +Here are some examples of methodology text, and the correct response: +Example 1 - +Methodology: "The maximum compensation for Covered Services rendered to a Covered Person shall be the "Allowed Amount." The Allowed Amount for Covered Services is the lesser of: (i) Allowable Charges; or (ii) as applicable, (a) 100% of the Payor's Medicaid fee schedule in effect on the date of service, or (b) for those Covered Services included in the Service Categories in Table 1 below, the applicable "Contracted Rate" set forth in Table 1 below. Example Service | 80% of the Medicare Rate in effect on the date of service." +Correct Response: +{{'LESSER' : 'Y', 'RATE_STANDARD' : '80% of MCR', 'RATE_SHORT' : '0.8', 'FLAT_FEE_STANDARD' : 'N/A', 'LESSER_RATE' : '100% of AC', 'NOT_TO_EXCEED' : 'N/A'.}} + +Example 2 - +Methodology: "The maximum compensation for Covered Services rendered to a Covered Person shall be the "Allowed Amount." The Allowed Amount for Covered Services is the lesser of: (i) Allowable Charges; (ii) 100% of the Payor's HEDIS target value in effect on the date of service, or (iii) for those Covered Services included in the Service Categories in Table 1 below, the applicable "Contracted Rate" set forth in Table 1 below. Example Service | $900.00 Per Unit" +Correct Response: {{'LESSER' : 'Y', 'RATE_STANDARD' : '100% of Payor's HEDIS target value', 'RATE_SHORT' : '1', 'FLAT_FEE_STANDARD' : '$900.00 Per Unit', 'LESSER_RATE' : '100% of AC', 'NOT_TO_EXCEED' : 'N/A'.}} + +Example 3 - +Methodology: "The maximum compensation for Covered Services rendered to a Covered Person shall be the "Allowed Amount." The Allowed Amount for Covered Services is the lesser of: (i) Allowable Charges or (ii) Medicaid Allowed Amount" +Correct Response: {{'LESSER' : 'Y', 'RATE_STANDARD' : '100% of MCD', 'RATE_SHORT' : '1', 'FLAT_FEE_STANDARD' : 'N/A', 'LESSER_RATE' : '100% of AC', 'NOT_TO_EXCEED' : 'N/A'.}} + +Example 4 - +Methodology: "The maximum compensation for Covered Services rendered to a Covered Person shall be the "Allowed Amount." The Allowed Amount for Covered Services is the lesser of: (i) Provider's Billed Charges or (ii) The amount payable by Medicare" +Correct Response: {{'LESSER' : 'Y', 'RATE_STANDARD' : '100% of MCR', 'RATE_SHORT' : '1', 'FLAT_FEE_STANDARD' : 'N/A', 'LESSER_RATE' : '100% of BC', 'NOT_TO_EXCEED' : 'N/A'.}} + +Example 5 - +Methodology: "Multiple procedures performed during the same day will be reimbursed at 100% for the primary procedure, 50% for the second procedure, and 50% for the third procedure, subsequent procedures shall not be eligible for reimbursement." +Correct Response: {{'LESSER' : 'N', 'RATE_STANDARD' : '100% of AC / 50% of AC / 50% of AC / 0% of AC', 'RATE_SHORT' : '1 / 0.5 / 0.5 / 0', 'FLAT_FEE_STANDARD' : 'N/A', 'LESSER_RATE' : 'N/A', 'NOT_TO_EXCEED' : 'N/A'.}} + +Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting. +""" + + +def BOTTOM_UP_ESCALATOR(d, page): + return f"""### PAGE START ### {page} ### PAGE END + +The above text contains a number of reimbursement terms. For the rest of this request, focus only on the specific term listed below: +Service: {d['FULL_SERVICE']} +Methodology: {d['FULL_METHODOLOGY']} + +Identify if the above reimbursement term is subject to any escalators, or increases in rate over time. Also identify if there are any exclusions. Fill out the following fields: +'RATE_ESCALATOR_IND' : If there is language on the page stating that the given rate will increase over time, return 'Y'. Otherwise return 'N'. +'RATE_ESCALATOR_DT' : If there is a rate escalator, on what date or time period does it kick in? Write the date or time period only. If 'N', return 'N/A' for this field. + +For the escalator fields, ensure that only updates and increases to the RATE itself (the percentage) are counted. Updates to Codes and other features do not apply here. + +Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting. +""" + + +def BOTTOM_UP_100_6(d): + return f"""Analyze the service and full methodology text listed below: +Service: {d['FULL_SERVICE']} +Methodology: {d['FULL_METHODOLOGY']} + +If the context indicates that the Service is a drug paid at 100% ASP + 6%, return 'Y'. Otherwise, return 'N'. + +Return only the answer, without using complete sentences. +""" + + +##################################################################################### +##################################### TOP DOWN ###################################### +##################################################################################### + + +def TOP_DOWN_PRIMARY(page): + return f"""### PAGE START ### {page} ### PAGE END + +The preceding text is one page of a contract between a Payer and a provider in their network. Extract the names of certain entities listed in the contract and return a single dictionary with the values below: + +Here are the attributes to be included in each dictionary, and instructions on how to correctly answer: +'EXHIBIT' : List any exhibit, attachment, or amendment names found on the page. Write the full name of the exhibit, including the exhibit number or letter, as well as any other subtitles describing the contents of the exhibit. Do NOT write the page number. If no Exhibit, Attachment, or Amendment is found, write 'N/A'. +'CONTRACT_LOB' : List the Line of Business mentioned on the page. Choose ONLY from the following: {valid.VALID_LOBS}. If there are multiple on the page, write them in a comma-separated list. If no LOB is found, write 'N/A'. +'CONTRACT_PROGRAM' : List any Programs or Plans mentioned on the page. {valid.VALID_PROGRAMS}. If no program is found, write 'N/A'. +'PROV_TYPE' : Write the Provider Type of the page. It will likely be found in the header. Choose ONLY from the following: {valid.VALID_PROV_TYPES}. Note that Facility may also be referred to as Hospital, Clinic, or similar. Professional may also be referred to as Physician, Physician Services, Practicioner, Provider or similar. Do NOT write 'N/A' for this field. +'DEFAULT_TERM' : If there is a section labelled 'Default' or similar, write that entire section here. It may also be a term that describes the payment rate if there is no established payment amount. If there is no such term, write 'N/A'. +'DEFAULT_RATE' : Write the rate from the Default Term in the form 'X% of Y'. If there is no Default Term, or no rate in the Default Term, write 'N/A'. +'CDM_IND' : Is there language on the page regarding CDM Neutralization? Write 'Y' if yes, 'N' if no. +'CHARGEMASTER' : Is there language on the page related to Chargemaster Protections? If so, return the sentence or sentences. If not, return 'N/A'. +'ADD_ON_REIMBURSEMENT_LANGUAGE' : Extract the Add On Reimbursement language present in the contract. If it is not present, answer N/A. + +If there are multiple answers for a given attribute, return the answers in a comma-separated list. + +For all attributes, only write an answer if it is written on the page. Do not make up new phrases or words or try to deduce answers that are not seen. Simply write 'N/A' if none of the listed options are seen. + +Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting. +""" + + +TD_PRIMARY_FIELDS = [ + "EXHIBIT", + "CONTRACT_LOB", + "CONTRACT_PROGRAM", + "PROV_TYPE", + "DEFAULT_TERM", + "DEFAULT_RATE", + "CDM_IND", + "CHARGEMASTER", + "ADD_ON_REIMBURSEMENT_LANGUAGE", +] + + +def TOP_DOWN_EXHIBIT_CHECK(page_start): + return f"""Text: {page_start} + +The above text is the start of a page. +Return 'Y' ONLY if the text contains the header of an Exhibit, Article, Amendment, Schedule, Attachment, or Addendum. + +'Additional Provisions' is not an Exhibit name. + +Otherwise, return 'N'. +""" + + +def TOP_DOWN_EXCLUSIONS(page): + return f"""### PAGE START ### {page} ### PAGE END + +The above text contains a page with at least one term related to exclusions or inclusions + +If it is indicated that any type of service or other situation should be EXCLUDED (or NOT INCLUDED), write just the exclusion clause here. + +Include language that specifically has words like 'exclusion', 'excluded', 'not to include'. +If there is no exclusion clause, write 'N/A'. +The exclusion clause may be a single sentence, an entire paragraph, or multiple paragraphs. + +Only write the EXACT text from the page, with no other commentary or explanation. Do not write ANY words in your response that are not found on the page. +""" + + +def TOP_DOWN_MEDICALLY_NECESSARY(page): + return f"""### PAGE START ### {page} ### PAGE END + +The above text contains a page with at least one Medical Necessity term. + +Extract the Medical Necessity paragraph from the page. + +Only write the EXACT text from the page, with no other commentary or explanation. Do not write ANY words in your response that are not found on the page. +""" + + +##################################################################################### +#################################### CONDITIONAL #################################### +##################################################################################### + + +def CONDITIONAL_IP_OP_SMALL(s, m): + return f"""Service: {s} +Methodology: {m} + +Is the term above related to Inpatient or Outpatient services? + +Write only 'IP' for Inpatient, 'OP' for Outpatient, or 'N/A' if the answer is unclear. + +Only write IP or OP if 'Inpatient' or 'Outpatient' is clearly and explicitly written in the Service or Methodology. Otherwise, write 'N/A'. Do not attempt to deduce the answer based on your prior knowledge. + +Return only the answer, without using complete sentences. +""" + + +def CONDITIONAL_IP_OP_BIG(d, page): + return f"""### PAGE START ### {page} ### PAGE END + +The above text contains a number of reimbursement terms. For the rest of this request, focus only on the specific term listed below: +Service: {d['FULL_SERVICE']} +Methodology: {d['FULL_METHODOLOGY']} + +Based on the context given, is the term above related to Inpatient or Outpatient services. + +Write only 'IP' for Inpatient, 'OP' for Outpatient. Only write IP or OP if it is EXPLICITLY written on the page and applies to the Service and Methodology. + +If neither, simply write 'N/A'. + +Return only the answer, without using complete sentences. +""" + + +def CONDITIONAL_PROV_2(d, page): + if "PROFESSIONAL" in d["PROV_TYPE"].upper(): + valid_list = valid.VALID_PROF + elif "ANCILLARY" in d["PROV_TYPE"].upper(): + valid_list = valid.VALID_ANC + elif "FACILITY" in d["PROV_TYPE"].upper(): + valid_list = valid.VALID_FAC + else: + valid_list = valid.VALID_PROF + ", " + valid.VALID_ANC + ", " + valid.VALID_FAC + return f"""### PAGE START ### {page} ### PAGE END + +The above text contains a number of reimbursement terms. For the rest of this request, focus only on the specific term listed below: +Service: {d['FULL_SERVICE']} +Methodology: {d['FULL_METHODOLOGY']} + +Based on the context given, what is the Provider Type applicable to the term above? + +Choose ONLY from the following: +{valid_list} +Do not attempt to deduce the Provider Type from context clues. It will be clearly and explicitly written in the text itself. If there is not clear Provider Type from the list, write 'N/A'. + +Return only the answer, without using complete sentences. +""" + + +def CONDITIONAL_DSH_IME_UC(page): + return f"""### PAGE START ### {page} ### PAGE END + +Is there language on the page saying that DSH, GME, IME, or UC is included? Write 'Y' or 'N'. + +Return only the answer, without using complete sentences. +""" + + +def CONDITIONAL_RBRVS(page): + return f"""### PAGE START ### {page} ### PAGE END + +Is RBRVS language found anywhere on the page? Write 'Y' or 'N'. + +Return only the answer 'Y' or 'N', without using complete sentences. +""" + + +def CONDITIONAL_STOP_LOSS(page): + return f"""### PAGE START ### {page} ### PAGE END + +Does the page include a sentence describing stop-loss thresholds? If yes, write the threshold dollar value only. If not, write 'N/A'. + +Return only the answer, without using complete sentences. +""" + + +def CONDITIONAL_LOB_CHECK(d, page): + return f"""### PAGE START ### {page} ### PAGE END + +Above is a page of a contract, with several Services listed. For the rest of this task, focus only on the Service and methodology listed below: +Service: {d['FULL_SERVICE']} +Methodology: {d['FULL_METHODOLOGY']} + +Below are the Line of Business values previously identified as being present on this page. +CONTRACT_LOB : {d['CONTRACT_LOB']} + +First, identify the section of the page that the Service and Methodology belong to. This may be a specific subsection or paragraph. Do not write this answer out, only use it to inform your search for the LOB value. + +Next, identify which single value of CONTRACT_LOB the given Service falls under. It can ONLY be ONE of values in the list. Do NOT return the original list. +The correct answer will likely be found in the same subsection or paragraph that the Service and Methodology are in. It might be listed somewhere close above the Service and Methodology text. + +Again, there is only ONE correct answer in the list. Choose carefully. + +Return only the Line of Business answer, without using complete sentences. +""" + + +##################################################################################### +############################# PRE- AND POST-PROCESSING ############################## +##################################################################################### +def FIX_JSON(d): + return f"""dict: {d} + +The above dictionary was not parsable by the json.loads() function. +This may be due to the keys having commas, single- or double-quotes, or other json characters. +Modify the values and formatting of the string to make it json parsable. Remove punctuation from the values if needed. + +Return only the json object, with no other commentary. +""" + + +def ALIGN_AND_FORMAT_TABLES(page): + return f"""{page} + +----- End Page ----- + +Above is a page with one or more misplaced tables in json format. + +Return the exact same page with the tables properly aligned and formatted, without changing anything else about the page. Here are instructions and examples to help: + +Alignment Instructions: +Align the tables in the text using the Table Header text found after -------Table Start--------. If there is no header, align based on context clues in the text such as "the table below", etc. +If there are multiple tables on the page, the first table should be placed in the first valid spot, the second in the second, etc. +If the Table Header is None, place the updated table at the top of the page. + +Formatting Instructions: +Here is a template format to use: + +Original JSON: "-------Table Start-------- Table Header {{'Column 1' : ['Value 1', 'Value 2', 'Value 3'], 'Column 2' : ['Value 4', 'Value 5', 'Value 6']}}-------Table End--------" + +Output Format: +Table Header +Column 1 : Value 1 | Column 2 : Value 4 +Column 1 : Value 2 | Column 2 : Value 5 +Column 1 : Value 3 | Column 2 : Value 6 + +Ensure that all text other than the tables is exactly the same + +Write only the page text, with no additional commentary. +""" + + +def LOB_SWEEPER(exhibit): + return f"""Exhibit Name: {exhibit} + +Based on the above Exhibit Name, which one or more of the following LOB values does the Exhibit refer to? Only choose LOBs that are explicitely written in the Exhibit. + +Valid LOBs: {valid.VALID_LOBS} + +Return ONLY the LOB, with no additional commentary or explanation. If there are no LOBs written, simply write 'N/A'. +""" + + +##################################################################################### +###################################### AC Fields #################################### +##################################################################################### + + +def AC_SINGLE_FIELD_TEMPLATE(context, question): + return f"""## START CONTRACT TEXT ## +{context} +## END CONTRACT TEXT ## + +Answer the following question, using the above contract text given as context: {question}. + +Do NOT add any additional commentary explaining the answer. Return ONLY the exact text from the contract, or N/A. +""" + + +# TODO: FIX PROMPT BELOW FOR JSON OUTPUT +def AC_MULTI_FIELD_TEMPLATE(context, questions): + return f"""## START CONTRACT TEXT ## +{context} +## END CONTRACT TEXT ## + +Answer the following questions, using the above contract text given as context: +{questions}. + +Do NOT add any additional commentary explaining the answer. Return ONLY the exact text from the contract, or N/A. +""" + + +AC_DICT = { + "CONTRACT_EFFECTIVE_DT": "Extract the contract effective date. This may be mentioned in one of the following locations: the signatory section, the preamble of the agreement, or the start of the amendment. Look for phrases such as /'This amendment is effective/'. Return this date in YYYY-MM-DD format.", + "CONTRACT_TITLE": "What is the title of the contract? It generally contains the word Agreement and is found at the very beginning of the document before a paragraph. This will be filled out if there is an amendment or numbered amendment. It typically contains the words agreement, amendment, or contract. An amendment typically has a number attached to it. The typical structure is the provider name, followed by the phrase with agreement, followed potentially by a number signifying a version of the contract. Only return the title, do not return any context.", + "PAYER_NAME": "What is the name of the payer that is a party to the contract as stated in the Preamble?", + "PROV_GROUP_NAME": "What is the name of the Group provider that is a party to the contract? This is generally found near keyword provider. Only return the name of the Group provider along with dba name, do not provide any context.", + "PROV_GROUP_NPI": "What is the group provider's national provider identifier number mentioned in the contract? This may be found on the signature page or on the roster. It consists of 10 digits. Only return these 10 digits.", + "PROV_GROUP_TIN_SIGNATORY": "What is the Group provider's taxpayer identification number stated near signature in the contract? This is a 9 digit long number typically with a hyphen after the first 2 digits. Return only this 9 digit number with a hyphen after the first 2 digits.", + "PROV_GROUP_TIN": "What is the Group provider's taxpayer identification number stated in the contract? This is a 9 digit long number typically with a hyphen after the first 2 digits in it. Return only this 9 digit number with a hyphen after the first 2 digits. As an example, it has the format of XX-XXXXXXX Do not elaborate or add context. If not found, then only return your answer with N/A. Do not pull any dates, do not pull any numbers that deviate from earlier described pattern. It must always be 9 digits. Either return the number found, or return N/A if not found. Strictly follow this instructions. Do not return any other information.", + "PROV_NPI_OTHER": 'What are all the other provider national provider identifier numbers (NPI) associated with this agreement? They consist of 10 digits and do not contain "-". Provide the list of all the other provider identifier numbers including those that are present in the table or attachment.', + "PROV_TIN_OTHER": "What are the provider taxpayer identification numbers (TIN) for each of the other providers associated with this agreement? They consist of 9 digits and may contain a hyphen after the first 2 digits. This is usually listed in the preample of the contract or on a roster. Provide only the 9 digit TIN.", + "TERMINATION_UPON_NOTICE": 'Extract the number of days can the contract be terminated by either party by giving notice. The answer may be found near the words "written notice of such termination" or "prior to the expiration". Only extract a value if one is present. Only return the number of days and nothing else.', + "TERMINATION_UPON_CAUSE": 'After how many days can the contract be terminated by either party upon breach of material term? The answer is usually found in section 7.2.2 or 10.2 but might be present in other sections as well. The answer is present after the Term language section. The answer may be found near the words "is in breach of any material term or condition". Here is an example of possible input text and the correct response. EXAMPLE INPUT TEXT: "By either party upon ninety (90) days prior written notice if the other party is in material breach of this Agreement, except that such termination shall not take place if the breach is cured within sixty (60) days following the written notice". EXAMPLE RESPONSE: 90. Only return the answer, with no other commentary or explanation.', + "TIME_TO_OBJECT": "How many days does the provider have to object to the amendment? The answer may be present in the Amendment section. Only return the number of days nothing else.", + "ASSIGNMENTS_CLAUSE_IND": 'Is assignment clause present in contract language? The answer is usually found in section 8.3 but might not always be present there. It can be found in the section with title containing "Assignment". Answer with a Yes or No, do not provide any other context.', + "NOTICE_PROVIDER_NAME": 'What is the notice provider name present usually near keywords like "Attn:". Return the full name, do not return any context. If it only says President/CEO and not the name itself then return N/A. Ensure that you do not pull the Payer\'s (EG. Superior Health Plan) info instead. Only pull the provider name. If it is not present, answer with N/A', + "NOTICE_PROVIDER_ADDRESS": "What is the notice provider address mentioned after the name of the provider. Only return the address of the provider, do not return any context. New Info --> Ensure Payer address (Eg. Superior Health Plan) isn't pulled. The payer address usually starts with 2100 South... Ensure you do not pull this. Always pull the provider address only.Ensure that you pull the full address with the city and state and zipcode. MAKE SURE YOU ALWAYS PULL THE CITY AND STATE AND ZIPCODE AT THE END. If it is not present in the text file, then do your best to atleast return the state. If it is not present, answer with N/A.", + "NPI_NAME": "What is the group provider's name whose national provider identifier number is mentioned in the contract? This may be found on the signature page or on the roster. Only return the name, do not provide any context.", + "HEALTH_PLAN_STATE": "Which state is this health plan for?", + "SEQUESTRATION_LANGUAGE": "Is sequestration language present in the contract? It may be after reimbursement or compensation section. It is only present in case of medicare and medicare advantage. If Yes, extract the complete paragraph.", + # "SEQUESTRATION_REDUCTIONS_IND":"Are sequestration and reduction percentage present in the contract? Answer Yes if both are present otherwise answer No.", + "CONTRACT_AUTO_RENEWAL_IND": "Does the contract automatically renew? Answer with a Yes or No, do not provide any other context. If the information is not present, answer N/A.", + "AMEND_CONTRACT_NOTICE_IND": 'Is amendment clause present in contract language? The answer is usually found in section 8.7 but might not always be present there. It may be found in the section with title containing "Amendment". Answer may be found near keywords "Agreement may be amended". Answer with a Yes or No, do not leave the answer as blank. Do not provide any other context. Here is an example of possible input text and the correct response. EXAMPLE INPUT TEXT: "may amend this Agreement by giving Provider written notice". EXAMPLE RESPONSE: Yes. If the agreement does not explicitly indicate that it can be amended by giving written notice, the answer will be No.', + "AFFILIATION_CLAUSE_IND": 'Is affiliation clause present in contract language? Answer may be found near keywords like "affiliate means a person or entity directly". Answer with a Yes or No, do not provide any other context.', + # "TERM_CLAUSE": """Extract the subsection or paragraph(s) labeled Term. It may also be labeled Termination. In instances there may be 2 bodies of text that you will need to pull. For example, first paragraph may be 'MCO must follow the procedures outlines in Section...' and the second one may be 'Not later than thirty (30) days following receipt of the termination notice...'. In this case, ensure that YOU ALWAYS PULL the whole text of the two paragraphs. If there are no such sections, extract the paragraph that most closely resembles this label. If there is still no correct answer, simply return 'N/A'.""", + "TERM_CLAUSE": """Extract the term or termination sections (or subsections or paragraphs), which are always present in the contract. They contain information related to term and termination of the contract. The biggest hint of a term or termination section will be the heading 'Term and Termination'. A secondary hint is that the first paragraph may be 'MCO must follow the procedures outlined in Section...' followed by 'Not later than XX days following receipt of the termination notice...'. In this case, ensure that YOU ALWAYS PULL the whole text of all relevant paragraphs.""", + # "TIMELY_FILING": "Extract all of the clauses of text and section paragraphs along with their section number that are pertaining to the Timely Filing clause. This clause usually details the conditions and days for when a provider must submit a claim by. Look for section heading titled /'Payment of Clean Claims/'. Look for language such as /'all provider claims shall be processed within 30 days from the date of claim reciept by the MCO/'. We want to extract all following langauage such as MCO shall not pay any claim submitted by provider..., MCO must adjudicate all appealed claims, till the whole section that starts with 'MCO may deny a claim for failure to file'. If it is not present, answer N/A.", + "CREDENTIALING_APP_IND": "Is credentialing defined in section 2.4 or any other subsection of the contract? Answer with a Yes or No, do not provide any other context.", + "CONTRACT_TERMINATION_DT": "What is the contract termination date? This might be listed within the Terms or Terms of Agreement section of the contract, or can be calculated by adding contract duration to the contract effective date. Only return the date, do not return any context.", + "TIME_TO_OBJECT": "How many days does the provider have to object to the amendment? The answer may be present in the Amendment section. Only return the number of days nothing else.", + "NON_RENEWAL_LANGUAGE": "Extract the section of the contract that describes non-renewal obligations, especially the number of days notice that must be given. Write the full relevant sentence or paragraph. Ensure that the exact term 'non-renewal' is present in the answer. Do NOT return an answer that does not have 'non-renewal'.", + "NON_RENEWAL_DAYS": "How many days of notice does the section indicate must be given to terminate the agreement? Examples are: 30, 60, 90, 180. Only return the number of days.", + "ACCESS_TO_MEDICAL_RECORDS": "Access clause contains information regarding access to medical records. It may be found in section 4.2 of the contract. Extract the Access clause present in the contract. If it is not present, answer N/A.", + # "ADD_ON_REIMBURSEMENT_IND":"Add On Reimbursement is usually present in Compensation section of the contract. Is Add On Reimbursement present in the Compensation section of the contract? Answer with a Yes or No, do not provide any other context.", + # "ADD_ON_REIMBURSEMENT_LANGUAGE":"Add On Reimbursement language is usually present in Compensation section of the contract. Extract the Add On Reimbursement language present in the contract. If it is not present, answer N/A.", + "CARVEOUT_VENDORS": "Carve-Out Vendors clause may be found in section 2.9 of the contract. Extract the Carve-Out Vendors clause present in the contract. If it is not present, answer N/A.", + "CLAIMS_EDITING_LANGUAGE": "Claims Editing Language is typically found in the claims section in the beginning parts of the contract. Extract Claims Editing Language present in the contract. If it is not present, answer N/A.", + # "CLAIMS_EDITING_LANGUAGE_IND":"Claims Editing Language is typically found in the claims section in the beginning parts of the contract. Is Claims Editing Language present in the contract? Answer with a Yes or No, do not provide any other context.", + "CLEAN_CLAIM": 'Extract the "Clean Claim" subsection present in definition section of the contract. If it is not present, answer N/A.', + # "CONFLICTS_BETWEEN_CERTAIN_DOCUMENTS_IND":"Is Conflicts Between Certain Documents clause present in the contract? Answer with a Yes or No, do not provide any other context.", + "CONFLICTS_BETWEEN_CERTAIN_DOCUMENTS_LANGUAGE": "Extract the Conflicts Between Certain Documents clause present in the contract. If it is not present, answer N/A.", + # "COST_SETTLEMENT_IND":"Is cost settlement language present in the contract? Answer with a Yes or No, do not provide any other context.", + "COST_SETTLEMENT_LANGUAGE": "Cost settlement language may be present in disputes and arbitration subsection of the contract. It usually contains keywords like settlementor cost settlement. Extract the cost settlement language present in the contract. If it is not present, answer N/A.", + "DEEMER_AMENDMENT": "Deemer Amendment clause may be found in section 8.7.2 of the contract. Extract the Deemer Amendment clause present in the contract. If it is not present, answer N/A.", + # "DELEGATED_FUNCTION_IND":"Is Delegated Function clause present in the contract? Answer with a Yes or No, do not provide any other context.", + "DELEGATED_TERMS": "Extract Delegated Terms present in the contract. If it is not present, answer N/A.", + "ECM": 'What is the "ECM" number mentioned in the contract? Only Answer the ECM number, do not provide any context. Do not provide ICM number. If ECM number is not present, answer N/A.', + # "EXCLUSIVITY_REQUIREMENT_IND":"Is Exclusivity requirement clause present in the contract? Answer with a Yes or No, do not provide any other context.", + "EXCLUSIVITY_REQUIREMENT_LANGUAGE": "Extract Exclusivity requirement clause present in the contract. If it is not present, answer N/A.", + # "GUARANTEE_OF_PROVIDER_YIELD_IND":"Is Guarantee of Provider Yield Language present in the contract? Answer with a Yes or No, do not provide any other context.", + "GUARANTEE_OF_PROVIDER_YIELD_LANGUAGE": "Extract Guarantee of Provider Yield Language present in the contract. If it is not present, answer N/A.", + "HCBS_SERVICES": 'Extract HCBS services clause present either in the compensation or exhibit section of the contract. It may contain keywords like "for covered HCBS services, plan shall pay". Do not include definitions of HCBS. If it is not present, answer N/A.', + "INDEMNIFICATION": "Indemnification clause may be found in section 5.2 of the contract. Extract the Indemnification clause present in the contract. If it is not present, answer N/A.", + # "INDEPENDENT_REVIEW_IND":"Is Independent Review language present in the contract? Answer with a Yes or No, do not provide any other context.", + "INDEPENDENT_REVIEW_LANGUAGE": "Extract Independent Review language present in the contract. If it is not present, answer N/A.", + # "INVOICE_PRICING_IND":"Is table with heading \"INVOICED SERVICES\" present in compensation schedule of the contract?", + "INVOICE_PRICING_LANGUAGE": 'Extract table with heading "INVOICED SERVICES" if it is present in compensation schedule of the contract. If it is not present, answer N/A', + # "LATE_PAID_CLAIMS_IND":"Is Late Paid Claims language present in the contract? Answer with a Yes or No, do not provide any other context.", + "LATE_PAID_CLAIMS_LANGUAGE": "Extract Late Paid Claims language present in the contract. This will be a sentence or paragraph describing late paid claims, interest payments or similar. Here is an example of a correct response: 'Any Clean Claim, as defined in 42 C.F.R. § 422.500, shall be paid within thirty (30) days of receipt by Plan at such address as may be designated by Plan, and Plan shall pay interest on any Clean Claim not paid within thirty (30) days of such receipt by Plan at the rate of interest required by law, or as otherwise set forth in the Provider Manual.'. Here is an example of an incorrect response: 'Any Clean Claim, as defined in 42 C.F.R. 422.500, shall be paid within thirty (30) days of receipt by Health Plan, Payor or (if Provider contracts with Downstream Entities) Provider, as applicable, as designated by Provider or such Downstream Entity, as applicable.'. If no valid answer is present, answer N/A.", + # "MEDICAL_NECESSITY_LANGUAGE":"Extract the \"Medically Necessary\" or \"Medical Necessity\" care (or services) verbiage mentioned in the compensation schedule or section of the contract. Do not mention what \"Medically Necessary\" means. If it is not present in compensation schedule or section, answer N/A.", + # "MEDICAL_NECESSITY_LANGUAGE_IND":"Is \"Medically Necessary\" or \"Medical Necessity\" care (or services) verbiage mentioned in the compensation schedule or section of the contract? Answer with a Yes or No, do not provide any other context.", + "MEMBER_CONFINEMENT_DAYS_LANGUAGE": "Extract Member Confinement Days Language present in the contract. If it is not present, answer N/A.", + # "MEMBER_CONFINEMENT_DAYS_LANGUAGE_IND":"Is Member Confinement Days Language present in the contract? Answer with a Yes or No, do not provide any other context.", + "NATIONAL_AGREEMENT_IND": "Are more than one states covered in the contract? Answer with a Yes or No, do not provide any other context.", + # "NETWORK_ACCESS_FEES_IND":"Is \"Network Access Fee\" present in the contract? If yes what is the fee mentioned?", + "NETWORK_ACCESS_FEES_LANGUAGE": 'Extract "Network Access Fee" verbiage present in the contract. If it is not present, answer N/A.', + # "NONSTANDARD_APPEALS_PROCESS_IND":"Is Nonstandard Appeals Process present in the contract? Answer with a Yes or No, do not provide any other context.", + "NONSTANDARD_APPEALS_PROCESS_LANGUAGE": "Extract Nonstandard Appeals Process language present in the contract. If it is not present, answer N/A.", + "PAYMENT_IN_ADVANCE_OF_CLAIMS_SUBMISSION_LANGUAGE": "Extract Payment in Advance of Claims Submission language present in the contract. If it is not present, answer N/A.", + # "PAYMENT_IN_ADVANCE_OF_CLAIMS_SUBMISSION_LANGUAGE_IND":"Is Payment in Advance of Claims Submission language present in the contract? Answer with a Yes or No, do not provide any other context.", + "PAYOR": '"Payor" is usually found in section 1.12 of the contract. Extract the "Payor" subsection present in definition section of the contract. If it is not present, answer N/A.', + "PMPM": "What is the PMPM rate mentioned in the contract? Only return the rate. Do not provide any context. If it is not present, answer N/A.", + "PREAUTHORIZATION": "Preauthorization clause may be found in section 2.7 of the contract. Extract the Preauthorization clause present in the contract. If it is not present, answer N/A.", + "PRODUCT_REMOVAL": "Product removal may be found in the products attachment, stating a product is being removed. Extract product removal language present in the contract. If it is not present, answer N/A.", + # "PROVIDER_BASED_BILLING_EXCLUSION_IND":"Provider-based Billing Exclusion is often listed under the ‘Additional Provisions’ section. Is Provider-based Billing Exclusion mentioned the contract? Answer with a Yes or No, do not provide any other context.", + "PROVIDER_BASED_BILLING_EXCLUSION_LANGUAGE": "Provider-based Billing Exclusion is often listed under the ‘Additional Provisions’ section. Extract the Provider-based Billing Exclusion language mentioned in the contract. If it is not present, answer N/A.", + "RECOVERY_RIGHTS": "Recovery Rights clause may be found in section 3.5 of the contract. Extract the entire Recovery Rights sentence or paragraph present in the contract. If it is not present, answer N/A.", + # "SINGLE_CODE_MULTIPLE_RATES_IND":"Is Single Code Multiple Rates present in the compensation or exhibit section of the contract? Answer with a Yes or No, do not provide any other context.", + # "SINGLE_CODE_MULTIPLE_RATES_LANGUAGE":"Extract Single Code Multiple Rates present in the compensation or exhibit section of the contract. If it is not present, answer N/A.", + "TEMPLATE": "Is this contract in standard Centene template containing definition, terms clauses and other standard sections. Answer with a Yes or No, do not provide any other context.", + "ARBITRATION_AND_DISPUTES": "Dispute Resolution clause is usually found in section 6.1 of the contract. Arbitration clause is usually found in section 6.2 of the contract. Extract both the Dispute Resolution and the arbitration clause present in the contract. In case one of them is present, extract that one. If none of them are present, answer N/A.", + # "DISPARAGEMENT_PROHIBITION_IND":"Is Disparagement Prohibition clause present in the contract? It may be found in section 2.1 of the contract. Answer with a Yes or No, do not provide any other context.", + "DISPARAGEMENT_PROHIBITION_LANGUAGE": "Disparagement Prohibition clause may be found in section 2.1 of the contract. Extract the Disparagement Prohibition clause present in the contract. If it is not present, answer N/A.", + "ELIGIBILITY_VERIFICATION": "Eligibility Verification or Determination clause may be found in section 2.6 of the contract. Extract the Eligibility Verification or Determination clause present in the contract. If it is not present, answer N/A.", + "INSURANCE_REQUIREMENT": "Extract Insurance requirement mentioned in the contract. If it is not present, answer N/A.", + "PARTICIPATION_IN_PRODUCTS": "Participation in Products clause may be found in section 2.2.2 of the contract. Extract the Participation in Products clause present in the contract. If it is not present, answer N/A.", + "POLICIES_AND_PROCEDURES": "Extract the entire Policies and Procedures sentence or paragraph present in the contract. Do not extract language about conflicts and construction. Do not extract language about PP (Preferred Provider) nor PPG (Preferred Provider Group). If it is not present, answer N/A.", + "REGULATORY_REQUIREMENTS": 'Extract the entire Regulatory Requirements paragraph present in the contract. It usually contains the keyword "Regulatory Requirements". If it is not present, answer N/A.', + # "RELATIONSHIP_OF_PARTIES_IND":"Is Relationship of Parties clause present in the contract? Answer with a Yes or No, do not provide any other context.", + "RELATIONSHIP_OF_PARTIES_LANGUAGE": "Extract the entire Relationship of Parties passage present in the contract. If it is not present, answer N/A.", +} + + +def AC_EFFECTIVE_DATE_CLEANUP(contract_text): + return f"""### Contract Start ### {contract_text} ### Contract End ### +Above is a contract. What is the contract effective date mentioned in any of the following locations: the signatory section, the preamble of the agreement, or the start of the amendment? Look for phrases such as /'This amendment is effective/'. +If there is no clear effective date, return the date from the signature page. +Return the date converted to YYYY-MM-DD format, with no other commentary or explanation. If there is no date, simply return 'N/A'.""" diff --git a/fieldExtraction/src/table_analysis.py b/fieldExtraction/src/table_analysis.py new file mode 100644 index 0000000..59bbb1f --- /dev/null +++ b/fieldExtraction/src/table_analysis.py @@ -0,0 +1,38 @@ +import table_funcs +import config +import utils +import preprocessing_funcs + +import concurrent.futures +import pandas as pd +import os + +input_dict = utils.read_input() +table_analysis_dicts = [] + + +def table_analysis(file_object): + filename, contract_text = file_object + text_dict = preprocessing_funcs.split_text(contract_text) + file_dict = {"Filename": filename, "Filename_pdf": filename.replace(".txt", ".pdf")} + if utils.contains_reimbursement(contract_text): + file_dict["Contains Reimbursement"] = True + else: + file_dict["Contains Reimbursement"] = False + stats_dict = table_funcs.get_table_stats(text_dict) + file_dict.update(stats_dict) + table_analysis_dicts.append(file_dict) + + +with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor: + futures = [executor.submit(table_analysis, item) for item in input_dict.items()] + for future in concurrent.futures.as_completed(futures): + try: + future.result() + except Exception as e: + print(f"Error: {e}") + +table_analysis_df = pd.DataFrame(table_analysis_dicts) +table_analysis_df.to_csv( + os.path.join(config.REPORTING_OUTPUT_DIRECTORY, config.TABLE_ANALYSIS_NAME) +) diff --git a/fieldExtraction/src/table_funcs.py b/fieldExtraction/src/table_funcs.py new file mode 100644 index 0000000..20adac5 --- /dev/null +++ b/fieldExtraction/src/table_funcs.py @@ -0,0 +1,227 @@ +import re +import ast + +import prompts +import claude_funcs +import config +import utils + + +def get_table_stats(text_dict): + ( + table_count, + num_table_pages, + rate_count, + ) = ( + 0, + 0, + 0, + ) + table_pages, rate_pages, rates_more_than_10 = [], [], [] + for page_num, page_text in text_dict.items(): + if utils.contains_reimbursement(page_text): + rate_pages.append(page_num) + rates_on_page = sum(1 for char in page_text if char in {"$", "%"}) + rate_count += rates_on_page + if rates_on_page >= 10: + rates_more_than_10.append(page_num) + + table_page_count = page_text.count("Table Start") + if table_page_count > 0: + table_count += table_page_count + num_table_pages += 1 + table_pages.append(page_num) + + return { + "Table Count": table_count, + "Table Page Count": num_table_pages, + "Table Pages": table_pages, + "Rate Count": rate_count, + "Rate Pages": rate_pages, + "Num Rate Pages": len(rate_pages), + ">=10 Rate Pages": rates_more_than_10, + "Num >=10 Rates": len(rates_more_than_10), + } + + +def align_and_format_tables(text_dict, filename): + for page_num, page_text in text_dict.items(): + if "Table Start" in page_text and utils.contains_reimbursement(page_text): + prompt = prompts.ALIGN_AND_FORMAT_TABLES(page_text) + aligned_page = claude_funcs.invoke_claude( + prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 8192 + ) + text_dict[page_num] = aligned_page + return text_dict + + +def clean_tables(text): + def replace_colon(text): + # Define a function to use in re.sub to check the context of the match + def replacer(match): + # Extract the character after ':' to check if it's '[' + following_text = match.group(1) + if following_text.strip().startswith("["): + return match.group( + 0 + ) # Return the original match (':' and whatever follows) + else: + return "" + match.group(1) # Replace ':' with ';' and return the rest + + # Use a regular expression to find ':' and the text that follows + pattern = r":(\s*.)" + replaced_text = re.sub(pattern, replacer, text) + return replaced_text + + def remove_unquoted_brackets(text): + # Define a function to check the content between brackets + def replacer(match): + content = match.group(1) + if "'" not in content and (len(content) > 0): + return content # Return just the content without brackets + else: + return match.group( + 0 + ) # Return the original match if it contains a quote + + # Use a regular expression to find brackets and the text within + pattern = r"\[([^\[\]]*?)\]" + cleaned_text = re.sub(pattern, replacer, text) + return cleaned_text + + # Clean tables + text = text.strip("{}") # Remove brackets + text = replace_colon(text) + text = remove_unquoted_brackets(text) + return text + + +def convert_to_dict(table_text): + """ + Converts a formatted table text into a dictionary where each key maps to a list of values. + + This function processes a string representation of a table, extracting key-value pairs and converting them into a dictionary. + Each key maps to a list of values parsed from the input string. The function also returns the number of elements in each list. + + Parameters: + table_text (str): The input string representing the table, formatted with keys and values. + + Returns: + tuple: A dictionary with keys mapping to lists of values, and an integer representing the number of elements in each list. + """ + table_text = clean_tables(table_text) + # print(table_text, '\n') + table_text_list = table_text.split("]") + table_text_list = [item for item in table_text_list if len(item) > 0] + + final_dict = {} + num_elements = 0 + for key_value_text in table_text_list: + key = key_value_text.split(":")[0].strip(", '") + try: + value = ":".join(key_value_text.split(":")[1:]).strip() + "]" + # print(value, '\n') + if len(value) > 1: + value_list = ast.literal_eval(value) + final_dict[key] = value_list + num_elements = len(value_list) + except: + value = ":".join(key_value_text.split(":")[1:]).strip() + "']" + # print(value, '\n') + if len(value) > 1: + value_list = ast.literal_eval(value) + final_dict[key] = value_list + num_elements = len(value_list) + + return final_dict, num_elements + + +def format_table(table_json, table_size): + """ + Formats a dictionary of table data into a string representation of the table. + + This function takes a dictionary where each key maps to a list of values and converts it into a formatted string. + The keys are used as headers, and each row is constructed by iterating through the lists up to the specified table size. + The resulting string separates headers and values with ' : ' and includes newline characters to delineate rows. + + Parameters: + table_json (dict): The dictionary containing the table data, with keys as column headers and lists as column values. + table_size (int): The number of rows in the table. + + Returns: + str: The formatted string representation of the table. + """ + table_text = "" + keys = [key for key in table_json.keys()] + table_text += " : ".join(keys) + "\n" + for i in range(table_size): + for key in keys: + if table_json[key][i]: + # table_text += key + ': ' + table_json[key][i] + ', ' + table_text += table_json[key][i] + " : " + table_text += r"\n" + return table_text + + +# def align_and_format_tables(text_dict): +# """ +# Aligns and formats tables within the text of each page in a dictionary. + +# This function searches for table text within the input text of each page, marked by 'Table Start' and 'Table End' tags. +# It extracts, converts, and formats these tables, then replaces the original table text with the formatted version while +# maintaining the alignment with surrounding text. The cleaned and formatted text for each page is then stored in a new dictionary. + +# Parameters: +# text_dict (dict): A dictionary where keys are page numbers and values are the text content of those pages, potentially containing tables. + +# Returns: +# dict: A new dictionary with aligned and formatted tables in the text for each page. +# """ +# aligned_text_dict = {} +# for key, text in text_dict.items(): +# aligned_text = text +# if 'Table Start' in text: +# table_texts = re.findall(r'-------Table Start--------(.*?)-------Table End--------', text, re.DOTALL) +# for table_text in table_texts: +# # print(table_text, '\n') + +# try: +# # Extract pretable text +# pretable = table_text.split('{')[0].strip() +# # Extract and format table text +# #table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace("'", '"') + "}" +# table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0] + "}" + +# table, table_size = convert_to_dict(table_only) +# table_formatted = format_table(table, table_size) +# except: +# table_formatted = table_text + +# # Align +# if 'None' in pretable or text.count(pretable) < 2: # If pretable is None or not found (other than the original), align to top of the page +# aligned_text = table_formatted.replace(pretable, "") + aligned_text.replace(table_text, "") +# elif text.count(pretable) == 2: # if only one pretable location, align formatted table there +# aligned_text = aligned_text.replace(table_text, "") # Remove original table +# aligned_text = aligned_text.replace(pretable, pretable + table_formatted) # Align new table +# else: # More than one pretable location +# aligned_text = aligned_text.replace(table_text, "") +# start_index, end_index = find_pretable_index(aligned_text, pretable) +# if start_index: # If a match found, realign +# aligned_text = text[0:start_index] + pretable + ' ' + table_formatted + text[end_index:].replace(table_text, '') +# else: # if not match found, don't realign +# aligned_text = aligned_text.replace(table_text, pretable + ' ' + table_formatted) # replace with formatted without aligning +# aligned_text_dict[key] = aligned_text.replace("-------Table Start--------", "").replace("-------Table End--------", "") +# else: +# aligned_text_dict[key] = text + +# return aligned_text_dict + + +# def find_pretable_index(text, pretable): +# pretable_escaped = re.escape(pretable) +# pattern = rf'(?<=[ \n\r]){pretable_escaped}(?=[ \n\r])' +# matches = [(match.start(), match.end()) for match in re.finditer(pattern, text)] +# if matches: +# return matches[-1] +# else: +# return False, 0 diff --git a/fieldExtraction/src/textract_template.py b/fieldExtraction/src/textract_template.py new file mode 100644 index 0000000..9a4ef6a --- /dev/null +++ b/fieldExtraction/src/textract_template.py @@ -0,0 +1,223 @@ +""" +This script was written to extract .txt files from pdf for adhoc client runs. +Ensure you have permissions for API gateway, S3, Lambda function and SQS to execute this (DEVELOPER & above roles in DEV & UAT for Doczy should suffice) +If you have Analyst / Test roles and need the permissions to be elevated or policies to be updated then please ask Sannan Iqbal to make the changes. + +The flow to execute the pipeline is: +Create batch using api req +Add files to the newly created batch along with the batch id as the tag +Construct payload to trigger pipeline +Post request to trigger pipeline with payload +The output text file can be found in the client_bucket/contract_text_file/batch_123456 +And the final LLM parsed outputs can be found in client_bucket/final_output/batch_123456 +""" + +import os +import boto3 +import requests +import json +import boto3 +from datetime import datetime +import os +from botocore.auth import SigV4Auth +from botocore.awsrequest import AWSRequest +from botocore.credentials import get_credentials +from botocore.session import Session +import time + + +# Function to upload files to S3 +def upload_files_to_s3(directory, bucket, batch_id): + contract_list = [] + for filename in os.listdir(directory): + if filename.endswith(".pdf"): + file_path = os.path.join(directory, filename) + s3_key = f"contracts-landing-zone/{batch_id}/{filename}" + + # Upload file to S3 + s3_client.upload_file(file_path, bucket, s3_key) + print(f"Uploaded {filename} to S3 bucket\n") + + # Add tags to the uploaded file + s3_client.put_object_tagging( + Bucket=bucket, + Key=s3_key, + Tagging={"TagSet": [{"Key": "BatchId", "Value": batch_id}]}, + ) + print(f"Added tags to {filename}\n") + # Add file details to contract list + # For now we can leave it as is since only A, C have been operationalized + contract_list.append( + { + "contract_name": filename, + "groups": [ + "A", # This needs to be dynamic in UI 1 based on what group has been selected + "C", + ], + "contract_source_path": s3_key, + } + ) + return contract_list + + +def create_batch(client_bucket, create_batch_url): + myobj = {"client-bucket-name": client_bucket} + + # Call create batch API endpoint + response = requests.post(create_batch_url, json=myobj) + if response.status_code >= 200 and response.status_code < 300: + try: + new_batch_id = json.loads(json.loads(response.text)["body"])["batch_id"] + landing_zone = json.loads(json.loads(response.text)["body"])["landing_zone"] + except: + print(myobj) + print(response.text) + new_batch_id = "failed_cases" + landing_zone = "contracts_landing_zone" + else: + print(response.text) + new_batch_id = "failed_cases" + landing_zone = "contracts_landing_zone" + return new_batch_id, landing_zone + + +def list_filtered_files(bucket_name, prefix, start_date, end_date, profile_name): + """ + List all files in an S3 bucket filtered by date range. + + Parameters: + - bucket_name: str, the name of the S3 bucket + - prefix: str, the prefix (folder) in the S3 bucket + - start_date: str, the start date in ISO 8601 format + - end_date: str, the end date in ISO 8601 format + - profile_name: str, the AWS profile name + + Returns: + - list of str: the keys of the filtered files + """ + # Parse the dates + start_date = datetime.fromisoformat(start_date.replace("Z", "+00:00")) + end_date = datetime.fromisoformat(end_date.replace("Z", "+00:00")) + + # Initialize a session using the specified profile + session = boto3.Session(profile_name=profile_name) + s3_client = session.client("s3") + + # List objects in the bucket with the specified prefix + paginator = s3_client.get_paginator("list_objects_v2") + page_iterator = paginator.paginate(Bucket=bucket_name, Prefix=prefix) + + # Filter files by date + filtered_files = [] + for page in page_iterator: + if "Contents" in page: + for obj in page["Contents"]: + last_modified = obj["LastModified"] + if start_date <= last_modified <= end_date: + filtered_files.append(obj["Key"]) + + return filtered_files + + +def download_files(bucket_name, file_keys, profile_name, local_directory): + """ + Download files from an S3 bucket. + + Parameters: + - bucket_name: str, the name of the S3 bucket + - file_keys: list of str, the keys of the files to download + - profile_name: str, the AWS profile name + - local_directory: str, the local directory to download files to + """ + # Initialize a session using the specified profile + session = boto3.Session(profile_name=profile_name) + s3_client = session.client("s3") + + # Ensure the local directory exists + if not os.path.exists(local_directory): + os.makedirs(local_directory) + + # Download each file + for key in file_keys: + file_name = os.path.basename(key) # Get only the file name from the key + local_file_path = os.path.join(local_directory, file_name) + print(f"Downloading {key} to {local_file_path}") + s3_client.download_file(bucket_name, key, local_file_path) + + print("Download completed.") + + +if __name__ == "__main__": + # Define your variables + s3_bucket = "doczyai-use2-u-cn1-s3-textract-processing-001" + # batch_id = 'batch_100524101551' + client_name = "Priority Health" + username = "ADHOC USER" + + # These endpoints are in UAT, please change them to DEV if there are access issues with UAT + api_endpoint = ( + "https://29gm8cek03.execute-api.us-east-2.amazonaws.com/dev/trigger-pipeline" + ) + create_batch_url = ( + "https://29gm8cek03.execute-api.us-east-2.amazonaws.com/dev/create-batch" + ) + + session = boto3.Session( + profile_name="temp_cred" + ) # Change the profile name to the one you have in your .aws/credentials file + s3_client = session.client("s3") + + # Use current directory as PDF directory + pdf_directory = "C:\\Doczy\\Priority Health\\For_Textract\\For_Textract\\UAT- test East Paris Surgical Center copy" + + # Create batch + batch_id, landing_zone = create_batch(s3_bucket, create_batch_url) + print(f"Batch ID: {batch_id}") + print(f"Landing Zone: {landing_zone}") + # batch_id = 'batch_110624213433' + # landing_zone = 'contracts_landing_zone/batch_110624213433/' + + if batch_id == "failed_cases": + print("Batch creation failed. Exiting...") + exit() + # Upload files and get contract list + contract_list = upload_files_to_s3(pdf_directory, s3_bucket, batch_id) + + # Create JSON object + data = { + "s3_bucket": s3_bucket, + "batch_id": batch_id, + "client_name": client_name, + "username": username, + "contract_list": contract_list, + } + + # Make POST request to API + response = requests.post(api_endpoint, json=data) + + # Print response + print(response.status_code) + print(response.json()) + + ################## + # Get text files + + prefix = f"contract-text-file/{batch_id}/" # if you have a specific prefix (folder) in your bucket + + # These dates are to filter the contracts in case there are older contracts in the same batch + start_date = "2024-06-04T00:00:00Z" # ISO 8601 format + end_date = "2024-06-14T23:59:59Z" # ISO 8601 format + profile_name = "temp_cred" + local_directory = "C:\\Doczy\\Priority Health\\For_Textract\\For_Textract\\text otuput" # local directory to save files + + # List filtered files + time.sleep( + 30 + ) # Waiting for the text files to be generated, this may take longer and files may not be available after 30 seconds sometimes + filtered_files = list_filtered_files( + s3_bucket, prefix, start_date, end_date, profile_name + ) + print(f"Filtered files: {len(filtered_files)}") + + # Download files + download_files(s3_bucket, filtered_files, profile_name, local_directory) diff --git a/fieldExtraction/src/tin_pull.py b/fieldExtraction/src/tin_pull.py new file mode 100644 index 0000000..ff469cd --- /dev/null +++ b/fieldExtraction/src/tin_pull.py @@ -0,0 +1,179 @@ +import json +import boto3 +import pandas as pd +import numpy as np +from datetime import datetime +import os +import anthropic +import re +from concurrent.futures import ThreadPoolExecutor, as_completed + +import config +import utils + +# Global configuration variables +S3_REGION = "us-east-2" +BEDROCK_REGION = "us-east-1" +LLM_SELECTED = "Claude Instant" +bedrock_runtime = config.BEDROCK_RUNTIME + + +def find_tin_numbers(text): + # Regular expression to find sets of 9 digits with an optional dash after the first two digits + pattern = r"\b\d{2}-?\d{7}\b" + + # Find all matches in the provided text + matches = re.findall(pattern, text) + + return matches + + +def invoke_llm(context, llm_selected=LLM_SELECTED): + print("Invoking LLM.") + prompt_data = f""" + + Human: Use the following pieces of context to provide a concise answer to the questions at the end. You must answer in python list format. + + {context} + + Question: What is the Group provider's taxpayer identification number stated in the contract? This is a 9 digit long number typically with a hyphen after the first 2 digits. Return only this 9 digit number with a hyphen after the first 2 digits. Do not elaborate or add context. + + Assistant: Answer in list format: [""" + + if llm_selected == "Claude 3 - Sonnet": + model_id = "anthropic.claude-3-5-sonnet-20240620-v1:0" + body = json.dumps( + { + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 4096, + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": anthropic.HUMAN_PROMPT + + prompt_data + + anthropic.AI_PROMPT, + } + ], + } + ], + "temperature": 0.0, + } + ) + elif llm_selected in ["Claude Instant"]: + model_id = "anthropic.claude-instant-v1" + body = json.dumps( + { + "prompt": anthropic.HUMAN_PROMPT + prompt_data + anthropic.AI_PROMPT, + "max_tokens_to_sample": 2048, + "temperature": 0.0, + "top_p": 1, + "top_k": 250, + "stop_sequences": [anthropic.HUMAN_PROMPT], + } + ) + else: + raise ValueError("Unsupported LLM selected") + + try: + response = bedrock_runtime.invoke_model( + body=body, + modelId=model_id, + accept="application/json", + contentType="application/json", + ) + response_body = json.loads(response.get("body").read()) + if llm_selected == "Claude 3 - Sonnet": + response_text = response_body["content"][0]["text"] + elif llm_selected in ["Claude Instant"]: + response_text = response_body["completion"] + else: + print("Unsupported LLM selected") + # print(f"LLM response: {response_text}") + except Exception as e: + print(f"Error invoking LLM: {e}") + response_text = "failed" + return response_text.strip() + + +def main(filename, contract_text): + print(f"Processing contract: {filename}") + try: + df = pd.DataFrame(columns=["Contract Name", "Raw value", "Final value"]) + + tin_number = find_tin_numbers(contract_text) + final_tin = list(set(tin_number)) + + if len(final_tin) != 1: + final_tin = find_tin_numbers( + invoke_llm(contract_text, llm_selected=LLM_SELECTED) + ) + + df["Raw value"] = [tin_number] + df["Final value"] = [final_tin] + df["Contract Name"] = [filename] + + df.to_csv(f"tin_output/{filename}.csv", index=False) + except Exception as e: + print(f"Error processing contract: {e}") + + +# bucket = "centene-national-contracting-files" +# contract_path = "no_tins_text_files" + +# s3_client = boto3.Session(aws_access_key_id=config.S3_ACCESS_KEY_ID, +# aws_secret_access_key=config.S3_SECRET_ACCESS_KEY, +# aws_session_token=config.S3_SESSION_TOKEN, region_name="us-east-2").client('s3') + +# objects = s3_client.list_objects_v2(Bucket=bucket, Prefix=contract_path) +file_list = [] + +OUTPUT_DIRECTORY = r"tin_output/no_tins_text_files" +NUM_WORKERS = 10 +BATCH_SIZE = 10 + + +def process_file(item): + filename, contract_text = item + if filename.endswith(".txt") and not os.path.exists( + os.path.join(OUTPUT_DIRECTORY, (filename[:-3] + "csv")) + ): + main(filename, contract_text) + + +def process_directory(input_dict): + # for item in input_dict.items(): + with ThreadPoolExecutor(max_workers=NUM_WORKERS) as executor: + futures = {executor.submit(process_file, item) for item in input_dict.items()} + for future in as_completed(futures): + future.result() # This will raise an exception if the callable raised one + + +input_dict = utils.read_input(path="data/cnc/no_tins") +process_directory(input_dict) + +individual_path = r"tin_output/no_tins_text_files" + + +def individual_to_dict(filename): + df = pd.read_csv(os.path.join(individual_path, filename)) + df = df.dropna(axis=1, how="all") + column_names = ["Filename", "Answer", "Final Answer"] + new_column_names = column_names[: len(df.columns)] + list( + df.columns[len(column_names) :] + ) + df.columns = new_column_names + df_dict = {"Filename": filename} + df_dict["Answer"] = df["Answer"].iloc[0] + df_dict["Final Answer"] = df["Final Answer"].iloc[0] + return df_dict + + +all_dicts = [] +for filename in os.listdir(individual_path): + df_dict = individual_to_dict(filename) + all_dicts.append(df_dict) +final_df = pd.DataFrame(all_dicts) +final_df.to_csv("results.csv", index=False) diff --git a/fieldExtraction/src/top_down_funcs.py b/fieldExtraction/src/top_down_funcs.py new file mode 100644 index 0000000..249519c --- /dev/null +++ b/fieldExtraction/src/top_down_funcs.py @@ -0,0 +1,204 @@ +import prompts +import claude_funcs +import utils +import config +import dict_operations +import valid +import postprocessing_funcs + +import json +import re + + +def run_top_down_primary(text_dict, page_num, filename): + page_text = text_dict[page_num] + prompt = prompts.TOP_DOWN_PRIMARY(page_text) + answer = claude_funcs.invoke_claude( + prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, max_tokens=4096 + ) + answer_dict = dict_operations.secondary_string_to_dict(answer, filename) + for field in prompts.TD_PRIMARY_FIELDS: + if field not in answer_dict: + answer_dict[field] = "N/A" + + # Add indicators + if "ADD_ON_REIMBURSEMENT_LANGUAGE" in answer_dict: + if "N/A" in answer_dict["ADD_ON_REIMBURSEMENT_LANGUAGE"]: + answer_dict["ADD_ON_REIMBURSEMENT_IND"] = "N" + else: + answer_dict["ADD_ON_REIMBURSEMENT_IND"] = "Y" + else: + answer_dict["ADD_ON_REIMBURSEMENT_IND"] = "N" + answer_dict["ADD_ON_REIMBURSEMENT_LANGUAGE"] = "N/A" + + if config.PROV_TYPE: + answer_dict["PROV_TYPE"] = config.PROV_TYPE + return answer_dict + + +def run_top_down(filename, text_dict, bu_results, exhibit_pages): + ## Run TD Primary, Merge with BU + merged_dicts = [] + td_dicts = {} + for bu_dict in bu_results: + page = bu_dict["page_num"] + if page in exhibit_pages: + td_page = page + else: + try: + td_page = str( + max([int(i) for i in exhibit_pages if int(i) < int(page)]) + ) + except: + td_page = page + + if td_page not in td_dicts.keys(): + answer_dict = run_top_down_primary(text_dict, td_page, filename) + td_dicts[td_page] = answer_dict + else: + answer_dict = td_dicts[td_page] + + bu_dict.update(answer_dict) + merged_dicts.append(bu_dict) + + # Run and Merge Exclusions + exclusion_dict = get_td_dict( + text_dict, filename, prompts.TOP_DOWN_EXCLUSIONS, valid.EXCLUSION_TERMS + ) + final_dicts = add_exclusions(merged_dicts, exclusion_dict) + + # Run and Merge Medically Necessary + medically_necessary_dict = get_td_dict( + text_dict, + filename, + prompts.TOP_DOWN_MEDICALLY_NECESSARY, + valid.VALID_MEDICALLY_NECESSARY, + ) + medically_necessary_dict = postprocessing_funcs.filter_dict( + medically_necessary_dict, + pattern=re.compile( + "|".join( + re.escape(keyword) for keyword in valid.INVALID_MEDICALLY_NECESSARY + ), + re.IGNORECASE, + ), + ) + final_dicts = add_medically_necessary(final_dicts, medically_necessary_dict) + + return final_dicts + + +def get_td_dict(text_dict, filename, PROMPT_FUNC, VALID_VALUES): + td_dict = {} + for page_num, page_text in text_dict.items(): + if ( + any(i.upper() in page_text.upper() for i in VALID_VALUES) + and page_num.isdigit() + ): + prompt = PROMPT_FUNC(page_text) + answer = claude_funcs.invoke_claude( + prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 1024 + ) + td_dict[page_num] = answer + return td_dict + + +def add_medically_necessary(merged_dicts, medically_necessary_dict): + if not medically_necessary_dict: + for d in merged_dicts: + d["Medical Necessity Language (Language)"] = "N/A" + d["Medical Necessity Language (Y/N)"] = "N" + return merged_dicts + + sorted_keys = sorted(int(k) for k in medically_necessary_dict.keys()) + + # Find the closest page number that is less than or equal to the given page number + def find_closest_page(page_num): + closest_page = sorted_keys[0] + for key in sorted_keys: + if key <= page_num: + closest_page = key + else: + break + return closest_page + + for d in merged_dicts: + try: + current_page_num = int(d["page_num"]) + # Find the closest preceding medically necessary page + closest_page = find_closest_page(current_page_num) + d["Medical Necessity Language (Language)"] = medically_necessary_dict[ + str(closest_page) + ] + d["Medical Necessity Language (Y/N)"] = "Y" + except: + d["Medical Necessity Language (Language)"] = "N/A" + d["Medical Necessity Language (Y/N)"] = "N" + + return merged_dicts + + +def add_exclusions(merged_dicts, exclusions_dict): + if not exclusions_dict: + for d in merged_dicts: + d["EXCLUSIONS"] = "N/A" + return merged_dicts + + # Prepare by sorting exclusion keys and initializing an empty dictionary to track exclusions by EXHIBIT + sorted_keys = sorted(int(k) for k in exclusions_dict.keys()) + exhibit_exclusions = {} + + # First pass: assign exclusions based on page number + for d in merged_dicts: + page_num = str(d["page_num"]) + if page_num.isdigit(): + for exc_page in sorted_keys: + if exc_page >= int(page_num) and exc_page <= int(page_num) + 2: + new_exclusion = exclusions_dict[str(exc_page)] + # Handle EXCLUSIONS as a list to avoid duplicates + if "EXCLUSIONS" not in d: + d["EXCLUSIONS"] = [] + if new_exclusion not in d["EXCLUSIONS"]: + try: + d["EXCLUSIONS"].append(new_exclusion) + except: + d["EXCLUSIONS"] = [new_exclusion] + + # Map this exclusion to the EXHIBIT value + if "EXHIBIT" in d: + if d["EXHIBIT"] not in exhibit_exclusions: + exhibit_exclusions[d["EXHIBIT"]] = [] + if new_exclusion not in exhibit_exclusions[d["EXHIBIT"]]: + exhibit_exclusions[d["EXHIBIT"]].append(new_exclusion) + break + else: + # If no matching exclusion found, check if 'EXHIBIT' maps to any existing exclusion + if "EXHIBIT" in d and d["EXHIBIT"] in exhibit_exclusions: + for exclusion in exhibit_exclusions[d["EXHIBIT"]]: + if "EXCLUSIONS" not in d: + d["EXCLUSIONS"] = [] + if exclusion not in d["EXCLUSIONS"]: + d["EXCLUSIONS"].append(exclusion) + else: + d["EXCLUSIONS"] = ["N/A"] + else: + d["EXCLUSIONS"] = ["N/A"] + + # Convert lists to semicolon-separated strings + for d in merged_dicts: + if isinstance(d["EXCLUSIONS"], list): + d["EXCLUSIONS"] = "; ".join(d["EXCLUSIONS"]) + + return merged_dicts + + +def get_exhibit_pages(text_dict, filename): + exhibit_pages = [] + for page_num, page in text_dict.items(): + prompt = prompts.TOP_DOWN_EXHIBIT_CHECK(page[0:100]) + answer = claude_funcs.invoke_claude( + prompt, config.MODEL_ID_CLAUDE2, filename, max_tokens=10 + ) + if "Y" in answer: + exhibit_pages.append(page_num) + return exhibit_pages diff --git a/fieldExtraction/src/tracking.py b/fieldExtraction/src/tracking.py new file mode 100644 index 0000000..b8d61f6 --- /dev/null +++ b/fieldExtraction/src/tracking.py @@ -0,0 +1,148 @@ +import config + +import time +import random +import os +import csv + + +def write_stats_to_csv(filename=config.TRACKING_FILE): + max_retries = 5 + for attempt in range(max_retries): + try: + with open(filename, "w", newline="") as csvfile: + writer = csv.writer(csvfile) + writer.writerow( + [ + "Filename", + "Requests", + "Tokens Sent", + "Tokens Received", + "Total Time (s)", + "Avg Time per Request (s)", + "Claude 2 Requests", + "Claude 3 Requests", + "Claude 3.5 Requests", + ] + ) + + for filename, stats in config.MODEL_STATS.items(): + avg_time = ( + round(stats["total_time"] / stats["requests"], 2) + if stats["requests"] > 0 + else 0 + ) + writer.writerow( + [ + filename, + stats["requests"], + stats["tokens_sent"], + stats["tokens_received"], + round(stats["total_time"], 2), + avg_time, + stats["claude2_requests"], + stats["claude3_requests"], + stats["claude35_requests"], + ] + ) + + # Write global stats + writer.writerow([]) + writer.writerow(["GLOBAL STATS", "Value"]) + writer.writerow( + ["Total Requests", config.GLOBAL_STATS["total_requests"]] + ) + writer.writerow( + ["Total Tokens Sent", config.GLOBAL_STATS["total_tokens_sent"]] + ) + writer.writerow( + [ + "Total Tokens Received", + config.GLOBAL_STATS["total_tokens_received"], + ] + ) + writer.writerow( + ["Total Time (s)", round(config.GLOBAL_STATS["total_time"], 2)] + ) + writer.writerow( + [ + "Avg Time per Request (s)", + ( + round( + config.GLOBAL_STATS["total_time"] + / config.GLOBAL_STATS["total_requests"], + 2, + ) + if config.GLOBAL_STATS["total_requests"] > 0 + else 0 + ), + ] + ) + writer.writerow( + [ + "Total Claude 2 Requests", + config.GLOBAL_STATS["total_claude2_requests"], + ] + ) + writer.writerow( + [ + "Total Claude 3 Requests", + config.GLOBAL_STATS["total_claude3_requests"], + ] + ) + writer.writerow( + [ + "Total Claude 3.5 Requests", + config.GLOBAL_STATS["total_claude35_requests"], + ] + ) + writer.writerow(["Total Files Processed", len(config.MODEL_STATS)]) + break + except PermissionError: + if attempt < max_retries - 1: + time.sleep(random.uniform(0.1, 0.5)) + else: + print( + f"Unable to write to {filename} after {max_retries} attempts. It may be open in another program." + ) + + +def update_results_csv(item, results, filename="processing_results.csv"): + max_retries = 5 + for attempt in range(max_retries): + try: + file_exists = os.path.isfile(filename) + with open(filename, "a", newline="") as csvfile: + writer = csv.writer(csvfile) + if not file_exists: + writer.writerow(["Filename"] + list(results.keys())) + writer.writerow([item[0]] + list(results.values())) + break + except PermissionError: + if attempt < max_retries - 1: + time.sleep(random.uniform(0.1, 0.5)) + else: + print( + f"Unable to write to {filename} after {max_retries} attempts. It may be open in another program." + ) + + +def write_batch_results(batch_results, filename="batch_results.csv"): + max_retries = 5 + for attempt in range(max_retries): + try: + file_exists = os.path.isfile(filename) + with open(filename, "a", newline="") as csvfile: + writer = csv.writer(csvfile) + if not file_exists: + writer.writerow(["Filename"] + list(batch_results[0][1].keys())) + for item, results in batch_results: + writer.writerow([item[0]] + list(results.values())) + break + except PermissionError: + if attempt < max_retries - 1: + time.sleep(random.uniform(0.1, 0.5)) + else: + print( + f"Unable to write to {filename} after {max_retries} attempts. It may be open in another program." + ) diff --git a/fieldExtraction/src/utils.py b/fieldExtraction/src/utils.py new file mode 100644 index 0000000..46fa988 --- /dev/null +++ b/fieldExtraction/src/utils.py @@ -0,0 +1,225 @@ +import os +import re +import pandas as pd +import shutil +import time +import sys +import numpy as np + +import config + + +def read_local(file_path): + # Check if the file is a text file + if os.path.isfile(file_path) and file_path.endswith(".txt"): + try: + # First attempt to open the file with UTF-8 encoding + with open(file_path, "r", encoding="utf-8") as file: + file_contents = file.read() + return file_contents + except UnicodeDecodeError: + # If UTF-8 fails, try reading the file with ANSI encoding + try: + with open(file_path, "r", encoding="cp1252") as file: + file_contents = file.read() + return file_contents + except UnicodeDecodeError: + # If ANSI also fails, log an error message or handle it accordingly + print(f"Failed to decode {file_path} with UTF-8 and cp1252 encodings.") + + +def list_s3_files(prefix=config.S3_PREFIX, bucket=config.S3_BUCKET): + """List all files in an S3 folder.""" + response = config.S3_CLIENT.list_objects_v2(Bucket=bucket, Prefix=prefix) + return [item["Key"] for item in response.get("Contents", [])] + + +def read_s3(): + s3_client = config.S3_CLIENT + bucket = config.S3_BUCKET + prefix = config.S3_PREFIX + + file_list = [] + continuation_token = None + + while True: + + if continuation_token: + response = s3_client.list_objects_v2( + Bucket=bucket, Prefix=prefix, ContinuationToken=continuation_token + ) + else: + response = s3_client.list_objects_v2(Bucket=bucket, Prefix=prefix) + + file_list += [ + obj["Key"] + for obj in response.get("Contents", []) + if not obj["Key"].endswith("/") + ] + + if response.get("IsTruncated"): + continuation_token = response.get("NextContinuationToken") + else: + break + + contract_list = sorted(file_list) + files = {} + + for contract in contract_list: + print(f"Reading {contract}") + try: + data = s3_client.get_object(Bucket=bucket, Key=contract) + contents = data["Body"].read() + context = contents.decode("utf-8") + path, filename = os.path.split(contract) + files[filename] = context + except: + pass + + return files + + +def read_input(local_path=config.LOCAL_PATH): + if config.READ_MODE == "local": + files = {} + for file in os.listdir(local_path): + full_path = os.path.join(local_path, file) + file_text = read_local(full_path) + files[file] = file_text + return files + elif config.READ_MODE == "s3": + return read_s3() + + +def consolidate_individual(input_folder="results", output_folder="output"): + dfs = [] + for filename in os.listdir(input_folder): + if filename.endswith(".csv"): + filepath = os.path.join(input_folder, filename) + dfs.append(pd.read_csv(filepath)) + + # Remove temp folder + if input_folder == "temp" and os.path.exists(input_folder): + shutil.rmtree(input_folder) + + consolidated_df = pd.concat(dfs, ignore_index=True) + + # Write output + version = 1 + existing_files = [ + filename + for filename in os.listdir(output_folder) + if filename.startswith(f"consolidated_results_{config.TODAY}") + ] + if existing_files: + versions = [ + int(file.split("_v")[1].split(".")[0]) + for file in existing_files + if "_v" in file + ] + if versions: + version = max(versions) + 1 + filename = f"consolidated_results_{config.TODAY}_v{version}.csv" + consolidated_df.to_csv(os.path.join(output_folder, filename)) + + +def preprocess_text_file(file_path): + # Attempt to open the file with UTF-8 encoding first + try: + with open(file_path, "r", encoding="utf-8") as file: + text = file.read() + except UnicodeDecodeError: + # If UTF-8 fails, try reading the file with ANSI (cp1252) encoding + try: + with open(file_path, "r", encoding="cp1252") as file: + text = file.read() + except UnicodeDecodeError: + # If ANSI also fails, log an error message or handle it accordingly + print(f"Failed to decode {file_path} with UTF-8 and cp1252 encodings.") + return [] + + # Split the text into pages based on a specific marker + pages = re.split(r"Start of Page No\. = \d+", text) + return pages + + +def format_td_check(td_dicts, dont_include_list): + final_str = "" + dict_count = 1 + for td_dict in td_dicts: + final_str += str(dict_count) + ". " + for k in td_dict.keys(): + if k not in dont_include_list: + final_str += k + ": " + td_dict[k] + ", " + final_str += "\n" + dict_count += 1 + return final_str + + +def contains_reimbursement(text, page="1"): + if isinstance(text, dict): + return page.isdigit() and ( + "%" in text[page] + or "$" in text[page] + or "percent " in text[page] + or "compensation schedule" in text[page].lower() + or "reimbursement schedule" in text[page].lower() + ) + elif isinstance(text, str): + return ( + "%" in text + or "$" in text + or "percent " in text + or "compensation schedule" in text.lower() + or "reimbursement schedule" in text.lower() + ) + else: + print("contains_reimbursement - Invalid data type") + + +def contains_keyword(text, keywords): + # FILL OUT HERE + return + + +def filter_already_processed(input_dict, fields=config.FIELDS): + already_processed = [] + for folder_name in os.listdir(config.OUTPUT_DIRECTORY): + # ABC + if ( + fields == "abc" + and config.AC_RESULTS_NAME + in os.listdir(os.path.join(config.OUTPUT_DIRECTORY, folder_name)) + and config.B_RESULTS_NAME + in os.listdir(os.path.join(config.OUTPUT_DIRECTORY, folder_name)) + ): + already_processed.append(folder_name + ".txt") + # Fill AC Here + if ( + config.AC_RESULTS_NAME + in os.listdir(os.path.join(config.OUTPUT_DIRECTORY, folder_name)) + and config.FIELDS == "ac" + ): + already_processed.append(folder_name + ".txt") + # Fill B Here + if ( + config.B_RESULTS_NAME + in os.listdir(os.path.join(config.OUTPUT_DIRECTORY, folder_name)) + and config.FIELDS == "b" + ): + already_processed.append(folder_name + ".txt") + + input_dict = { + key: input_dict[key] + for key in input_dict.keys() + if key not in already_processed + } + return input_dict + + +def is_empty(value): + if pd.isna(value): + return True + else: + empty_values = [None, "", "N/A", "NA", "null", "none", "NaN", np.nan, "nan"] + return value in empty_values diff --git a/fieldExtraction/src/valid.py b/fieldExtraction/src/valid.py new file mode 100644 index 0000000..ce229d4 --- /dev/null +++ b/fieldExtraction/src/valid.py @@ -0,0 +1,646 @@ +import config +import pandas as pd + + +VALID_LOBS = [ + "MEDICARE", + "MEDICARE ADVANTAGE", + "MEDICAID", + "NON-MEDICAID", + "MARKETPLACE", + "COMMERCIAL", + "COMMERCIAL-EXCHANGE", + "GROUP", + "MEDICAID-MEDICARE", +] + + +## PROGRAMS ## +state_list = config.STATE.split("-") +VALID_PROGRAMS = [] +for state in state_list: + if state == "TX": + VALID_PROGRAMS += [ + "CHIP", + "CHIP-P", + "CHIP-PERINATE", + "STAR", + "STAR+PLUS", + "DUAL SPECIAL NEEDS PLAN", + "DSNP", + "D-SNP", + "FOSTER CARE", + "MA Plan", + "MA-PD", + "MMP", + "HMO", + "PPO", + "EPO", + ] + elif state == "IL": + VALID_PROGRAMS += [ + "Meridian Medicaid Plan", + "YouthCare HealthChoice Illinois", + "Wellcare", + "Ambetter", + "Meridican Medicare-Medicaid Plan", + "Meridian Managed Long Term Services and Support Plan", + ] + elif state == "FL": + VALID_PROGRAMS += [ + "Sunshine Health", + "Wellcare", + "Ambetter by Sunshine Health", + "Childrens Medical Services Health Plan", + ] + elif state == "KY": + VALID_PROGRAMS += [ + "Wellcare", + "Wellcare of Kentucky", + "Ambetter for Wellcare of Kentucky", + ] + elif state == "OH": + VALID_PROGRAMS += [ + "Buckeye Health Plan", + "MyCare Ohio", + "Wellcare", + "Ambetter from Buckeye Health Plan", + ] + elif state == "MI": + VALID_PROGRAMS += [ + "Meridian", + "Meridian Complete", + "Wellcare", + "Ambetter from Meridian", + ] + elif state == "LA": + VALID_PROGRAMS += [ + "Louisiana Healthcare Connections", + "Wellcare", + "Ambetter from Louisiana Healthcare Connections", + ] + elif state == "OK": + VALID_PROGRAMS += [ + "Oklahoma Complete Health", + "Wellcare", + "Ambetter of Oklahoma", + ] + elif state == "NY": + VALID_PROGRAMS += ["Fidelis Care", "Wellcare", "Ambetter from Fidelis Care"] +VALID_PROGRAMS = list(set(VALID_PROGRAMS)) +if VALID_PROGRAMS: + VALID_PROGRAMS = f"One or more programs may be valid. ONLY from the following values: {VALID_PROGRAMS}. If there language such as 'MA PLAN/MA-PD PLAN/DSNP PLAN' and also 'Medicare -HMO' present, then ensure you extract HMO as well with the other plans" +else: + VALID_PROGRAMS = f"A program is defined as a plan or product offering from an insurance company. Look for the brand name of a health plan." + + +VALID_PROV_TYPES = ["Facility", "Professional", "Ancillary"] + +# VALID_NETWORKS = ['HMO', 'PPO', 'EPO'] + +VALID_MSR = [ + "Multiple Procedures", + "Multiple Outpatient", + "Multiple Surgical", + "Multiple Surgeries", +] + +VALID_DEFAULT = [ + "codes not found in", + "all other services", + "services not listed", + "except for those listed below", + "codes not covered", + "services not defined", + "no established payment", + "no established fee amount", + "procedures not listed", + "default", +] + +INVALID_DEFAULT = ["Except as otherwise provided"] + +NULL_ANSWER_TERMS = [ + "do not have", + "do not see", + "does not specify", + "does not explicitly", + "don't know", + "Not specified", + "don't see", + "don't have", + "Nothing found", + "does not contain", + "unable to find", +] + +EXCLUSION_TERMS = [ + "Exclusions", + "Excluded", + "Exclude", + "Not including", + "Not to include", +] + +VALID_MEDICALLY_NECESSARY = ["Medically Necessary", "Medical Necessity"] +INVALID_MEDICALLY_NECESSARY = [ + "nan", + "N/A", + "Defined", + "Records", + "Payments", + "Definition", + "mean", + "Those medical services which:", + "twenty-four", + "seven", + "24", + "7", +] + +INVALID_HCBS = [ + "nan", + "N/A", + "Defined", + "Definition", + "mean", +] # TODO add postprocessing for HCBS + +# Provider Type Level 2 +VALID_PROF = [ + "Physician", + "Primary Care Provider", + "Dual Capacity Physician", + "Mid-Level Practicioner", + "Specialist", + "OB/GYN", + "Behavioral Health", + "Therapist", + "Surgery", + "Independent RHC", + "Ambulance Services", + "Home Health", +] +VALID_ANC = [ + "DME", + "Durable Medical Equipment", + "Radiology", + "Lab", + "Home Health", + "Hospice", + "Dialysis", + "PT/OT/ST", + "Urgent Care", + "Home Infusion", + "Ambulatory Surgery Center", + "ASC", + "Ambulance Services", + "Home Health", +] +VALID_FAC = [ + "Skilled Nursing Facility", + "Ambulatory Surgery Center", + "ASC", + "Hospital", + "Clinic", + "Rural Health Clinic", + "FQHC", + "Long Term Care", + "Community Mental Health Center", +] + + +def select_valid_prov_2(provider_type): + if isinstance(provider_type, pd.Series): + # If it's a Series, use str.contains() method + if provider_type.str.contains("PROFESSIONAL", case=False, na=False).any(): + valid_list = VALID_PROF + elif provider_type.str.contains("ANCILLARY", case=False, na=False).any(): + valid_list = VALID_ANC + elif provider_type.str.contains("FACILITY", case=False, na=False).any(): + valid_list = VALID_FAC + else: + valid_list = VALID_PROF + VALID_ANC + VALID_FAC + else: + # If it's a single string, use the original logic + if "PROFESSIONAL" in provider_type.upper(): + valid_list = VALID_PROF + elif "ANCILLARY" in provider_type.upper(): + valid_list = VALID_ANC + elif "FACILITY" in provider_type.upper(): + valid_list = VALID_FAC + else: + valid_list = VALID_PROF + VALID_ANC + VALID_FAC + return valid_list + + +# List of fields that will have derived indicators +DERIVED_INDICATOR_FIELDS = [ + "PROVIDER_BASED_BILLING_EXCLUSION_LANGUAGE", + "MEDICAL_NECESSITY_LANGUAGE", + "INVOICE_PRICING_LANGUAGE", + "GUARANTEE_OF_PROVIDER_YIELD_LANGUAGE", + "CLAIMS_EDITING_LANGUAGE", + "DISPARAGEMENT_PROHIBITION_LANGUAGE", + "NONSTANDARD_APPEALS_PROCESS_LANGUAGE", + "RELATIONSHIP_OF_PARTIES_LANGUAGE", + "CONFLICTS_BETWEEN_CERTAIN_DOCUMENTS_LANGUAGE", + "PAYMENT_IN_ADVANCE_OF_CLAIMS_SUBMISSION_LANGUAGE", + "NETWORK_ACCESS_FEES_LANGUAGE", + "MEMBER_CONFINEMENT_DAYS_LANGUAGE", + "INDEPENDENT_REVIEW_LANGUAGE", + "EXCLUSIVITY_REQUIREMENT_LANGUAGE", + "LATE_PAID_CLAIMS_LANGUAGE", + "COST_SETTLEMENT_LANGUAGE", + "DELEGATED_TERMS", +] + +B_MAPPING = { + "Filename": "Contract Name", + "Parent Agreement Code": "Parent Agreement Code", + "Pages": "Pages", + "page_num": "Page_Num", + "EXHIBIT": "Attachment/Exhibit", + "CONTRACT_LOB": "Line of Business", + "PROV_TYPE": "Provider Type", + "PROV_TYPE_LEVEL_2": "Provider Type - Level 2", + "IP_OP": "IP/OP", + "FULL_SERVICE": "Service Type", + "CONTRACT_PROGRAM": "Plan Type", + "LESSER": "Lesser of Logic language, included (Y/N)", + "LESSER_RATE": "Lesser of Rate", + "FULL_METHODOLOGY": "Reimb. Methodology", + "SHORT_METHODOLOGY": "Reimb. Methodology_Short", + "RATE_STANDARD": "If rate is % of Payor or MCR [STANDARD]", + "RATE_SHORT": "If rate is % of Payor or MCR [STANDARD]_Short", + "FLAT_FEE_STANDARD": "FLAT FEE", + "DEFAULT_TERM": "Default Term", + "DEFAULT_RATE": "Default Rate", + "Medical Necessity Language (Language)": "Medical Necessity Language (Language)", + "Medical Necessity Language (Y/N)": "Medical Necessity Language (Y/N)", + 'Inclusion of essential RBRVS "Fee Source" Language (Y/N)': 'Inclusion of essential RBRVS "Fee Source" Language (Y/N)', + "CDM_IND": "CDM Neutralization Language, included (Y/N)", + "CHARGEMASTER": "CONTRACT_CHARGEMASTER_PROTECTION_LANGUAGE", + "ADD_ON_REIMBURSEMENT_LANGUAGE": "Add On Reimbursement (Language)", + "ADD_ON_REIMBURSEMENT_IND": "Add On Reimbursement (Y/N)", + "SINGLE_CODE_MULTIPLE_RATES_LANGUAGE": "Single Code Multiple Rates (Language)", + "SINGLE_CODE_MULTIPLE_RATES_IND": "Single Code Multiple Rates (Y/N)", + "IP - DSH/IME/UC, included (Y/N)": "IP - DSH/IME/UC, included (Y/N)", + "IP - Stoploss Catastrophic Threshold": "IP - Stoploss Catastrophic Threshold", + "EXCLUSIONS": "Exclusions", + "NOT_TO_EXCEED": "Not to Exceed", + "RATE_ESCALATOR_IND": "Escalator or COLA (Y/N)", + "RATE_ESCALATOR_DT": "Escalator I, Eff. Date", +} + +AC_MAPPING = { + "Filename": "Contract Name", + "CONTRACT_TITLE": "Agreement_Name (Contract Title)", + "PAYER_NAME": "PAYER NAME", + "AFFILIATION_CLAUSE_IND": "Affiliate (Y/N)", + "TERM_CLAUSE": "Term Clause", + "CONTRACT_AUTO_RENEWAL_IND": "Contract Auto-Renewal Indicator", + "CONTRACT_TERMINATION_DT": "Termination Date", + "TERMINATION_UPON_NOTICE": "Termination Upon Notice - Days", + "TERMINATION_UPON_CAUSE": "Termination With Cause - Days", + "AMEND_CONTRACT_NOTICE_IND": "Amend Contract Upon notice Flag (Y/N)", + "TIME_TO_OBJECT": "Timeframe to Object - Days", + "ASSIGNMENTS_CLAUSE_IND": "Assignments Clause (Y/N)", + "CONTRACT_EFFECTIVE_DT": "Contract Effective Date", + "PROV_GROUP_TIN": "IRS #", + "PROV_GROUP_NAME": "IRS_Name", + "PROV_GROUP_NPI": "NPI (10-digits)", + "PROV_GROUP_TIN_SIGNATORY": "PROV_GROUP_TIN_SIGNATORY", + "PROV_TIN_OTHER": "PROV_TIN_OTHER", + "PROV_NPI_OTHER": "PROV_NPI_OTHER", + "NOTICE_PROVIDER_NAME": "Notice to Provider Name", + "NOTICE_PROVIDER_ADDRESS": "Notice to Provider Address", + "HEALTH_PLAN_STATE": "Health Plan State", + "CREDENTIALING_APP_IND": "Credentialing Application Indicator", + "SEQUESTRATION_LANGUAGE": "Sequestration Language", + "SEQUESTRATION_REDUCTIONS_IND": "Sequestration Reductions (Y/N)", + "NPI_NAME": "NPI Name", + "DELEGATED_TERMS_IND": "Delegated Function Indicator", + "DELEGATED_TERMS": "Delegated Terms", + "ECM": "ECM", + "NATIONAL_AGREEMENT_IND": "National Agreement Indicator", + "COST_SETTLEMENT_LANGUAGE_IND": "Cost Settlement (Y/N)", + "COST_SETTLEMENT_LANGUAGE": "Cost Settlement (Language)", + "LATE_PAID_CLAIMS_LANGUAGE_IND": "Late Paid Claims (Y/N)", + "LATE_PAID_CLAIMS_LANGUAGE": "Late Paid Claims (Language)", + "DEEMER_AMENDMENT": "Deemer Amendment", + "REGULATORY_REQUIREMENTS": "Regulatory Requirements", + "RECOVERY_RIGHTS": "Recovery Rights", + "ARBITRATION_AND_DISPUTES": "Arbitration and Disputes", + "EXCLUSIVITY_REQUIREMENT_LANGUAGE_IND": "Exclusivity Requirement (Y/N)", + "EXCLUSIVITY_REQUIREMENT_LANGUAGE": "Exclusivity Requirement (Language)", + "PAYOR": "Payor", + "PARTICIPATION_IN_PRODUCTS": "Participation in Products", + "CLEAN_CLAIM": "Clean Claim", + "INDEPENDENT_REVIEW_LANGUAGE_IND": "Independent Review (Y/N)", + "INDEPENDENT_REVIEW_LANGUAGE": "Independent Review (Language)", + "INDEMNIFICATION": "Indemnification", + "ACCESS_TO_MEDICAL_RECORDS": "Access to Medical Records", + "MEMBER_CONFINEMENT_DAYS_LANGUAGE_IND": "Member Confinement Days Language (Y/N)", + "MEMBER_CONFINEMENT_DAYS_LANGUAGE": "Member Confinement Days Language (Language)", + "NETWORK_ACCESS_FEES_LANGUAGE_IND": "Network Access Fees (Y/N)", + "NETWORK_ACCESS_FEES_LANGUAGE": "Network Access Fees (Language)", + "PAYMENT_IN_ADVANCE_OF_CLAIMS_SUBMISSION_LANGUAGE_IND": "Payment in Advance of Claims Submission Language (Y/N)", + "PAYMENT_IN_ADVANCE_OF_CLAIMS_SUBMISSION_LANGUAGE": "Payment in Advance of Claims Submission Language", + "ELIGIBILITY_VERIFICATION": "Eligibility Verification", + "PREAUTHORIZATION": "Preauthorization", + "POLICIES_AND_PROCEDURES": "Policies and Procedures", + "INSURANCE_REQUIREMENT": "Insurance Requirement", + "CARVEOUT_VENDORS": "Carve-Out Vendors", + "CONFLICTS_BETWEEN_CERTAIN_DOCUMENTS_LANGUAGE_IND": "Conflicts Between Certain Documents (Y/N)", + "CONFLICTS_BETWEEN_CERTAIN_DOCUMENTS_LANGUAGE": "Conflicts Between Certain Documents (Language)", + "RELATIONSHIP_OF_PARTIES_LANGUAGE_IND": "Relationship of Parties (Y/N)", + "RELATIONSHIP_OF_PARTIES_LANGUAGE": "Relationship of Parties (Language)", + "NONSTANDARD_APPEALS_PROCESS_LANGUAGE_IND": "Nonstandard Appeals Process (Y/N)", + "NONSTANDARD_APPEALS_PROCESS_LANGUAGE": "Nonstandard Appeals Process (Language)", + "PRODUCT_REMOVAL": "Product Removal", + "DISPARAGEMENT_PROHIBITION_LANGUAGE_IND": "Disparagement Prohibition (Y/N)", + "DISPARAGEMENT_PROHIBITION_LANGUAGE": "Disparagement Prohibition (Language)", + "CLAIMS_EDITING_LANGUAGE_IND": "Claims Editing Language (Y/N)", + "CLAIMS_EDITING_LANGUAGE": "Claims Editing Language (Language)", + "GUARANTEE_OF_PROVIDER_YIELD_LANGUAGE_IND": "Guarantee of Provider Yield (Y/N)", + "GUARANTEE_OF_PROVIDER_YIELD_LANGUAGE": "Guarantee of Provider Yield (Language)", + "HCBS_SERVICES": "HCBS Services", + "ADD_ON_REIMBURSEMENT_IND": "Add On Reimbursement (Y/N)", + "ADD_ON_REIMBURSEMENT_LANGUAGE": "Add On Reimbursement (Language)", + "PMPM": "PMPM", + "SINGLE_CODE_MULTIPLE_RATES_IND": "Single Code Multiple Rates (Y/N)", + "SINGLE_CODE_MULTIPLE_RATES_LANGUAGE": "Single Code Multiple Rates (Language)", + "INVOICE_PRICING_LANGUAGE_IND": "Invoice Pricing (Y/N)", + "INVOICE_PRICING_LANGUAGE": "Invoice Pricing (Language)", + "MEDICAL_NECESSITY_LANGUAGE_IND": "Medical Necessity Language (Y/N)", + "MEDICAL_NECESSITY_LANGUAGE": "Medical Necessity Language (Language)", + "TEMPLATE": "Template", + "PROVIDER_BASED_BILLING_EXCLUSION_LANGUAGE_IND": "Provider-based Billing Exclusion (Y/N)", + "PROVIDER_BASED_BILLING_EXCLUSION_LANGUAGE": "Provider-based Billing Exclusion (Language)", + "NON_RENEWAL_LANGUAGE": "Non-Renewal Language", + "NON_RENEWAL_DAYS": "Non-Renewal - Days", + "TIMELY_FILING": "Timely Filing", +} + + +ABC_COLUMNS = [ + "Contract Name", + "Agreement_Name (Contract Title)", + "PAYER NAME", + "Health Plan State", + "Affiliate (Y/N)", + "Credentialing Application Indicator", + "Term Clause", + "Contract Auto-Renewal Indicator", + "Termination Date", + "Termination Upon Notice - Days", + "Termination With Cause - Days", + "Amend Contract Upon notice Flag (Y/N)", + "Timeframe to Object - Days", + "Assignments Clause (Y/N)", + "Contract Effective Date", + "IRS #", + "IRS_Name", + "NPI (10-digits)", + "NPI Name", + "PROV_GROUP_TIN_SIGNATORY", + "PROV_TIN_OTHER", + "PROV_NPI_OTHER", + "Notice to Provider Name", + "Notice to Provider Address", + "Sequestration Language", + "Sequestration Reductions (Y/N)", + "Parent Agreement Code", + "Pages", + "Page_Num", + "Attachment/Exhibit", + "Line of Business", + "Provider Type", + "Provider Type - Level 2", + "IP/OP", + "Service Type", + "Plan Type", + "Lesser of Logic language, included (Y/N)", + "Lesser of Rate", + "Reimb. Methodology", + "Reimb. Methodology_Short", + "If rate is % of Payor or MCR [STANDARD]", + "If rate is % of Payor or MCR [STANDARD]_Short", + "FLAT FEE", + "Default Term", + "Default Rate", + 'Inclusion of essential RBRVS "Fee Source" Language (Y/N)', + "CDM Neutralization Language, included (Y/N)", + "CONTRACT_CHARGEMASTER_PROTECTION_LANGUAGE", + "IP - DSH/IME/UC, included (Y/N)", + "IP - Stoploss Catastrophic Threshold", + "Exclusions", + "Not to Exceed", + "Escalator or COLA (Y/N)", + "Escalator I, Eff. Date", + "Delegated Function Indicator", + "Delegated Terms", + "ECM", + "National Agreement Indicator", + "Cost Settlement (Y/N)", + "Cost Settlement (Language)", + "Late Paid Claims (Y/N)", + "Late Paid Claims (Language)", + "Deemer Amendment", + "Regulatory Requirements", + "Recovery Rights", + "Arbitration and Disputes", + "Exclusivity Requirement (Y/N)", + "Exclusivity Requirement (Language)", + "Payor", + "Participation in Products", + "Clean Claim", + "Independent Review (Y/N)", + "Independent Review (Language)", + "Indemnification", + "Access to Medical Records", + "Member Confinement Days Language (Y/N)", + "Member Confinement Days Language (Language)", + "Network Access Fees (Y/N)", + "Network Access Fees (Language)", + "Payment in Advance of Claims Submission Language (Y/N)", + "Payment in Advance of Claims Submission Language", + "Eligibility Verification", + "Preauthorization", + "Policies and Procedures", + "Insurance Requirement", + "Carve-Out Vendors", + "Conflicts Between Certain Documents (Y/N)", + "Conflicts Between Certain Documents (Language)", + "Relationship of Parties (Y/N)", + "Relationship of Parties (Language)", + "Nonstandard Appeals Process (Y/N)", + "Nonstandard Appeals Process (Language)", + "Product Removal", + "Disparagement Prohibition (Y/N)", + "Disparagement Prohibition (Language)", + "Claims Editing Language (Y/N)", + "Claims Editing Language (Language)", + "Guarantee of Provider Yield (Y/N)", + "Guarantee of Provider Yield (Language)", + "HCBS Services", + "Add On Reimbursement (Y/N)", + "Add On Reimbursement (Language)", + "PMPM", + "Single Code Multiple Rates (Y/N)", + "Single Code Multiple Rates (Language)", + "Invoice Pricing (Y/N)", + "Invoice Pricing (Language)", + "Medical Necessity Language (Y/N)", + "Medical Necessity Language (Language)", + "Template", + "Provider-based Billing Exclusion (Y/N)", + "Provider-based Billing Exclusion (Language)", + "Non-Renewal Language", + "Timely Filing", + "Non-Renewal - Days", +] + + +BASE_COLUMNS = [ + "Lesser of Logic Language, included (Y/N)", + "Lesser of Rate", + "Reimb. methodology", + "Reimb. methodology_short", + "If rate is % of Payer or MCR [STANDARD]", + "If rate is % of Payer or MCR [STANDARD]_Short", + "If rate is Flat Fee [STANDARD]", + "Default Term", + "Default Rate", + "Exclusions", + "Not to Exceed", + "Escalator or COLA (Y/N)", + "Escalator I, Eff. Date", +] + +SERVICE_FILTER = [ + "LIABILITY", + "RISK", + "LOBBYING", + "DAMAGES", + "CONFIDENTIALITY", + "ARBITRATION", + "FALSE CLAIMS ACT", + "UNSPECIFIED", + "AUDIT", + "INTEREST", + "N/A", + "BUSINESS", + "COMPLIANCE", + "Medical Assistance Program", + "MATERIAL SUBCONTRACT", + "GIFTS", + "GRATUITIES", + "Insurance", + "Clean Claim", + "Employment verification", + "Suspension", + "thing of value", + "laws", + "safe drinking water act", + "immigration reform", + "social security", + "Lease", + "related entity", + "Negative balance", + "medicaid payments", + "federal water pollution control act", + "microbacteriology", + "epsdt", + "appeal", + "prompt pay", + "capitation", + "harmless", + "30 days", + "fee source", + "Rent-to-Purchase", + "chargemaster", + "stop-loss", + "mobile crisis", + "quality incentive program", +] + +METHODOLOGY_FILTER = [ + "interest", + "clean claim", + "timely", + "adjudicate", + "appeal", + "30 days", + "https", + "prior authorization", + "compliance", + "provider-based billing", + "not less than the level and amount of payment", + "Fee Change Effective Date", + "management fees", + "PMPM", + "quality incentive program", + "if there is no established payment", +] + +STATE_MAP = { + "AL": "Alabama", + "AK": "Alaska", + "AZ": "Arizona", + "AR": "Arkansas", + "AS": "American Samoa", + "CA": "California", + "CO": "Colorado", + "CT": "Connecticut", + "DE": "Delaware", + "DC": "District of Columbia", + "FL": "Florida", + "GA": "Georgia", + "GU": "Guam", + "HI": "Hawaii", + "ID": "Idaho", + "IL": "Illinois", + "IN": "Indiana", + "IA": "Iowa", + "KS": "Kansas", + "KY": "Kentucky", + "LA": "Louisiana", + "ME": "Maine", + "MD": "Maryland", + "MA": "Massachusetts", + "MI": "Michigan", + "MN": "Minnesota", + "MS": "Mississippi", + "MO": "Missouri", + "MT": "Montana", + "NE": "Nebraska", + "NV": "Nevada", + "NH": "New Hampshire", + "NJ": "New Jersey", + "NM": "New Mexico", + "NY": "New York", + "NC": "North Carolina", + "ND": "North Dakota", + "MP": "Northern Mariana Islands", + "OH": "Ohio", + "OK": "Oklahoma", + "OR": "Oregon", + "PA": "Pennsylvania", + "PR": "Puerto Rico", + "RI": "Rhode Island", + "SC": "South Carolina", + "SD": "South Dakota", + "TN": "Tennessee", + "TX": "Texas", + "TT": "Trust Territories", + "UT": "Utah", + "VT": "Vermont", + "VA": "Virginia", + "VI": "Virgin Islands", + "WA": "Washington", + "WV": "West Virginia", + "WI": "Wisconsin", + "WY": "Wyoming", +}