From d6abe44a697abe2af56bc8379239a4597338651b Mon Sep 17 00:00:00 2001 From: Rahul Ailaboina Date: Tue, 8 Apr 2025 20:14:49 +0000 Subject: [PATCH] Merged in feature/postprocess-Cleanup (pull request #464) Feature/postprocess Cleanup * Added aarete derived * removed the dev file * reverted * Merge branch 'main' into feature/postprocess-Cleanup * Update poetry lock * Update poetry and pyproject * Update poetry * Update field path * Update test_date_postprocess * Update tests * Merged main into feature/postprocess-Cleanup * Update test * Update test * addedfield_json_path * minor changes * removed * Updated * minor change * added llm invoke * removed unnecessary imports * add * self added * removed comments * Merged main into feature/postprocess-Cleanup * Merged main into feature/postprocess-Cleanup Approved-by: Katon Minhas --- fieldExtraction/poetry.lock | 164 ++++++------ fieldExtraction/pyproject.toml | 2 + fieldExtraction/src/investment/postprocess.py | 244 ++++++++++++++++-- .../tests/investment_postprocess_test.py | 104 ++++++++ 4 files changed, 417 insertions(+), 97 deletions(-) create mode 100644 fieldExtraction/tests/investment_postprocess_test.py diff --git a/fieldExtraction/poetry.lock b/fieldExtraction/poetry.lock index 5470100..44bd4b0 100644 --- a/fieldExtraction/poetry.lock +++ b/fieldExtraction/poetry.lock @@ -3806,91 +3806,80 @@ files = [ [[package]] name = "orjson" -version = "3.10.15" +version = "3.10.16" description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main"] files = [ - {file = "orjson-3.10.15-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:552c883d03ad185f720d0c09583ebde257e41b9521b74ff40e08b7dec4559c04"}, - {file = "orjson-3.10.15-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:616e3e8d438d02e4854f70bfdc03a6bcdb697358dbaa6bcd19cbe24d24ece1f8"}, - {file = "orjson-3.10.15-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7c2c79fa308e6edb0ffab0a31fd75a7841bf2a79a20ef08a3c6e3b26814c8ca8"}, - {file = "orjson-3.10.15-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:73cb85490aa6bf98abd20607ab5c8324c0acb48d6da7863a51be48505646c814"}, - {file = "orjson-3.10.15-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:763dadac05e4e9d2bc14938a45a2d0560549561287d41c465d3c58aec818b164"}, - {file = "orjson-3.10.15-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a330b9b4734f09a623f74a7490db713695e13b67c959713b78369f26b3dee6bf"}, - {file = "orjson-3.10.15-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a61a4622b7ff861f019974f73d8165be1bd9a0855e1cad18ee167acacabeb061"}, - {file = "orjson-3.10.15-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acd271247691574416b3228db667b84775c497b245fa275c6ab90dc1ffbbd2b3"}, - {file = "orjson-3.10.15-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:e4759b109c37f635aa5c5cc93a1b26927bfde24b254bcc0e1149a9fada253d2d"}, - {file = "orjson-3.10.15-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:9e992fd5cfb8b9f00bfad2fd7a05a4299db2bbe92e6440d9dd2fab27655b3182"}, - {file = "orjson-3.10.15-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f95fb363d79366af56c3f26b71df40b9a583b07bbaaf5b317407c4d58497852e"}, - {file = "orjson-3.10.15-cp310-cp310-win32.whl", hash = "sha256:f9875f5fea7492da8ec2444839dcc439b0ef298978f311103d0b7dfd775898ab"}, - {file = "orjson-3.10.15-cp310-cp310-win_amd64.whl", hash = "sha256:17085a6aa91e1cd70ca8533989a18b5433e15d29c574582f76f821737c8d5806"}, - {file = "orjson-3.10.15-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:c4cc83960ab79a4031f3119cc4b1a1c627a3dc09df125b27c4201dff2af7eaa6"}, - {file = "orjson-3.10.15-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ddbeef2481d895ab8be5185f2432c334d6dec1f5d1933a9c83014d188e102cef"}, - {file = "orjson-3.10.15-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9e590a0477b23ecd5b0ac865b1b907b01b3c5535f5e8a8f6ab0e503efb896334"}, - {file = "orjson-3.10.15-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a6be38bd103d2fd9bdfa31c2720b23b5d47c6796bcb1d1b598e3924441b4298d"}, - {file = "orjson-3.10.15-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ff4f6edb1578960ed628a3b998fa54d78d9bb3e2eb2cfc5c2a09732431c678d0"}, - {file = "orjson-3.10.15-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b0482b21d0462eddd67e7fce10b89e0b6ac56570424662b685a0d6fccf581e13"}, - {file = "orjson-3.10.15-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bb5cc3527036ae3d98b65e37b7986a918955f85332c1ee07f9d3f82f3a6899b5"}, - {file = "orjson-3.10.15-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d569c1c462912acdd119ccbf719cf7102ea2c67dd03b99edcb1a3048651ac96b"}, - {file = "orjson-3.10.15-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:1e6d33efab6b71d67f22bf2962895d3dc6f82a6273a965fab762e64fa90dc399"}, - {file = "orjson-3.10.15-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:c33be3795e299f565681d69852ac8c1bc5c84863c0b0030b2b3468843be90388"}, - {file = "orjson-3.10.15-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:eea80037b9fae5339b214f59308ef0589fc06dc870578b7cce6d71eb2096764c"}, - {file = "orjson-3.10.15-cp311-cp311-win32.whl", hash = "sha256:d5ac11b659fd798228a7adba3e37c010e0152b78b1982897020a8e019a94882e"}, - {file = "orjson-3.10.15-cp311-cp311-win_amd64.whl", hash = "sha256:cf45e0214c593660339ef63e875f32ddd5aa3b4adc15e662cdb80dc49e194f8e"}, - {file = "orjson-3.10.15-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9d11c0714fc85bfcf36ada1179400862da3288fc785c30e8297844c867d7505a"}, - {file = "orjson-3.10.15-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dba5a1e85d554e3897fa9fe6fbcff2ed32d55008973ec9a2b992bd9a65d2352d"}, - {file = "orjson-3.10.15-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7723ad949a0ea502df656948ddd8b392780a5beaa4c3b5f97e525191b102fff0"}, - {file = "orjson-3.10.15-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6fd9bc64421e9fe9bd88039e7ce8e58d4fead67ca88e3a4014b143cec7684fd4"}, - {file = "orjson-3.10.15-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dadba0e7b6594216c214ef7894c4bd5f08d7c0135f4dd0145600be4fbcc16767"}, - {file = "orjson-3.10.15-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b48f59114fe318f33bbaee8ebeda696d8ccc94c9e90bc27dbe72153094e26f41"}, - {file = "orjson-3.10.15-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:035fb83585e0f15e076759b6fedaf0abb460d1765b6a36f48018a52858443514"}, - {file = "orjson-3.10.15-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d13b7fe322d75bf84464b075eafd8e7dd9eae05649aa2a5354cfa32f43c59f17"}, - {file = "orjson-3.10.15-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7066b74f9f259849629e0d04db6609db4cf5b973248f455ba5d3bd58a4daaa5b"}, - {file = "orjson-3.10.15-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:88dc3f65a026bd3175eb157fea994fca6ac7c4c8579fc5a86fc2114ad05705b7"}, - {file = "orjson-3.10.15-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b342567e5465bd99faa559507fe45e33fc76b9fb868a63f1642c6bc0735ad02a"}, - {file = "orjson-3.10.15-cp312-cp312-win32.whl", hash = "sha256:0a4f27ea5617828e6b58922fdbec67b0aa4bb844e2d363b9244c47fa2180e665"}, - {file = "orjson-3.10.15-cp312-cp312-win_amd64.whl", hash = "sha256:ef5b87e7aa9545ddadd2309efe6824bd3dd64ac101c15dae0f2f597911d46eaa"}, - {file = "orjson-3.10.15-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:bae0e6ec2b7ba6895198cd981b7cca95d1487d0147c8ed751e5632ad16f031a6"}, - {file = "orjson-3.10.15-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f93ce145b2db1252dd86af37d4165b6faa83072b46e3995ecc95d4b2301b725a"}, - {file = "orjson-3.10.15-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7c203f6f969210128af3acae0ef9ea6aab9782939f45f6fe02d05958fe761ef9"}, - {file = "orjson-3.10.15-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8918719572d662e18b8af66aef699d8c21072e54b6c82a3f8f6404c1f5ccd5e0"}, - {file = "orjson-3.10.15-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f71eae9651465dff70aa80db92586ad5b92df46a9373ee55252109bb6b703307"}, - {file = "orjson-3.10.15-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e117eb299a35f2634e25ed120c37c641398826c2f5a3d3cc39f5993b96171b9e"}, - {file = "orjson-3.10.15-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:13242f12d295e83c2955756a574ddd6741c81e5b99f2bef8ed8d53e47a01e4b7"}, - {file = "orjson-3.10.15-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7946922ada8f3e0b7b958cc3eb22cfcf6c0df83d1fe5521b4a100103e3fa84c8"}, - {file = "orjson-3.10.15-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:b7155eb1623347f0f22c38c9abdd738b287e39b9982e1da227503387b81b34ca"}, - {file = "orjson-3.10.15-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:208beedfa807c922da4e81061dafa9c8489c6328934ca2a562efa707e049e561"}, - {file = "orjson-3.10.15-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eca81f83b1b8c07449e1d6ff7074e82e3fd6777e588f1a6632127f286a968825"}, - {file = "orjson-3.10.15-cp313-cp313-win32.whl", hash = "sha256:c03cd6eea1bd3b949d0d007c8d57049aa2b39bd49f58b4b2af571a5d3833d890"}, - {file = "orjson-3.10.15-cp313-cp313-win_amd64.whl", hash = "sha256:fd56a26a04f6ba5fb2045b0acc487a63162a958ed837648c5781e1fe3316cfbf"}, - {file = "orjson-3.10.15-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:5e8afd6200e12771467a1a44e5ad780614b86abb4b11862ec54861a82d677746"}, - {file = "orjson-3.10.15-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da9a18c500f19273e9e104cca8c1f0b40a6470bcccfc33afcc088045d0bf5ea6"}, - {file = "orjson-3.10.15-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bb00b7bfbdf5d34a13180e4805d76b4567025da19a197645ca746fc2fb536586"}, - {file = "orjson-3.10.15-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:33aedc3d903378e257047fee506f11e0833146ca3e57a1a1fb0ddb789876c1e1"}, - {file = "orjson-3.10.15-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dd0099ae6aed5eb1fc84c9eb72b95505a3df4267e6962eb93cdd5af03be71c98"}, - {file = "orjson-3.10.15-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7c864a80a2d467d7786274fce0e4f93ef2a7ca4ff31f7fc5634225aaa4e9e98c"}, - {file = "orjson-3.10.15-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c25774c9e88a3e0013d7d1a6c8056926b607a61edd423b50eb5c88fd7f2823ae"}, - {file = "orjson-3.10.15-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:e78c211d0074e783d824ce7bb85bf459f93a233eb67a5b5003498232ddfb0e8a"}, - {file = "orjson-3.10.15-cp38-cp38-musllinux_1_2_armv7l.whl", hash = "sha256:43e17289ffdbbac8f39243916c893d2ae41a2ea1a9cbb060a56a4d75286351ae"}, - {file = "orjson-3.10.15-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:781d54657063f361e89714293c095f506c533582ee40a426cb6489c48a637b81"}, - {file = "orjson-3.10.15-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:6875210307d36c94873f553786a808af2788e362bd0cf4c8e66d976791e7b528"}, - {file = "orjson-3.10.15-cp38-cp38-win32.whl", hash = "sha256:305b38b2b8f8083cc3d618927d7f424349afce5975b316d33075ef0f73576b60"}, - {file = "orjson-3.10.15-cp38-cp38-win_amd64.whl", hash = "sha256:5dd9ef1639878cc3efffed349543cbf9372bdbd79f478615a1c633fe4e4180d1"}, - {file = "orjson-3.10.15-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:ffe19f3e8d68111e8644d4f4e267a069ca427926855582ff01fc012496d19969"}, - {file = "orjson-3.10.15-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d433bf32a363823863a96561a555227c18a522a8217a6f9400f00ddc70139ae2"}, - {file = "orjson-3.10.15-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:da03392674f59a95d03fa5fb9fe3a160b0511ad84b7a3914699ea5a1b3a38da2"}, - {file = "orjson-3.10.15-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3a63bb41559b05360ded9132032239e47983a39b151af1201f07ec9370715c82"}, - {file = "orjson-3.10.15-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3766ac4702f8f795ff3fa067968e806b4344af257011858cc3d6d8721588b53f"}, - {file = "orjson-3.10.15-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a1c73dcc8fadbd7c55802d9aa093b36878d34a3b3222c41052ce6b0fc65f8e8"}, - {file = "orjson-3.10.15-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b299383825eafe642cbab34be762ccff9fd3408d72726a6b2a4506d410a71ab3"}, - {file = "orjson-3.10.15-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:abc7abecdbf67a173ef1316036ebbf54ce400ef2300b4e26a7b843bd446c2480"}, - {file = "orjson-3.10.15-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:3614ea508d522a621384c1d6639016a5a2e4f027f3e4a1c93a51867615d28829"}, - {file = "orjson-3.10.15-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:295c70f9dc154307777ba30fe29ff15c1bcc9dfc5c48632f37d20a607e9ba85a"}, - {file = "orjson-3.10.15-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:63309e3ff924c62404923c80b9e2048c1f74ba4b615e7584584389ada50ed428"}, - {file = "orjson-3.10.15-cp39-cp39-win32.whl", hash = "sha256:a2f708c62d026fb5340788ba94a55c23df4e1869fec74be455e0b2f5363b8507"}, - {file = "orjson-3.10.15-cp39-cp39-win_amd64.whl", hash = "sha256:efcf6c735c3d22ef60c4aa27a5238f1a477df85e9b15f2142f9d669beb2d13fd"}, - {file = "orjson-3.10.15.tar.gz", hash = "sha256:05ca7fe452a2e9d8d9d706a2984c95b9c2ebc5db417ce0b7a49b91d50642a23e"}, + {file = "orjson-3.10.16-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:4cb473b8e79154fa778fb56d2d73763d977be3dcc140587e07dbc545bbfc38f8"}, + {file = "orjson-3.10.16-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:622a8e85eeec1948690409a19ca1c7d9fd8ff116f4861d261e6ae2094fe59a00"}, + {file = "orjson-3.10.16-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c682d852d0ce77613993dc967e90e151899fe2d8e71c20e9be164080f468e370"}, + {file = "orjson-3.10.16-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c520ae736acd2e32df193bcff73491e64c936f3e44a2916b548da048a48b46b"}, + {file = "orjson-3.10.16-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:134f87c76bfae00f2094d85cfab261b289b76d78c6da8a7a3b3c09d362fd1e06"}, + {file = "orjson-3.10.16-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b59afde79563e2cf37cfe62ee3b71c063fd5546c8e662d7fcfc2a3d5031a5c4c"}, + {file = "orjson-3.10.16-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:113602f8241daaff05d6fad25bd481d54c42d8d72ef4c831bb3ab682a54d9e15"}, + {file = "orjson-3.10.16-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4fc0077d101f8fab4031e6554fc17b4c2ad8fdbc56ee64a727f3c95b379e31da"}, + {file = "orjson-3.10.16-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:9c6bf6ff180cd69e93f3f50380224218cfab79953a868ea3908430bcfaf9cb5e"}, + {file = "orjson-3.10.16-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:5673eadfa952f95a7cd76418ff189df11b0a9c34b1995dff43a6fdbce5d63bf4"}, + {file = "orjson-3.10.16-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5fe638a423d852b0ae1e1a79895851696cb0d9fa0946fdbfd5da5072d9bb9551"}, + {file = "orjson-3.10.16-cp310-cp310-win32.whl", hash = "sha256:33af58f479b3c6435ab8f8b57999874b4b40c804c7a36b5cc6b54d8f28e1d3dd"}, + {file = "orjson-3.10.16-cp310-cp310-win_amd64.whl", hash = "sha256:0338356b3f56d71293c583350af26f053017071836b07e064e92819ecf1aa055"}, + {file = "orjson-3.10.16-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:44fcbe1a1884f8bc9e2e863168b0f84230c3d634afe41c678637d2728ea8e739"}, + {file = "orjson-3.10.16-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78177bf0a9d0192e0b34c3d78bcff7fe21d1b5d84aeb5ebdfe0dbe637b885225"}, + {file = "orjson-3.10.16-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:12824073a010a754bb27330cad21d6e9b98374f497f391b8707752b96f72e741"}, + {file = "orjson-3.10.16-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ddd41007e56284e9867864aa2f29f3136bb1dd19a49ca43c0b4eda22a579cf53"}, + {file = "orjson-3.10.16-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0877c4d35de639645de83666458ca1f12560d9fa7aa9b25d8bb8f52f61627d14"}, + {file = "orjson-3.10.16-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9a09a539e9cc3beead3e7107093b4ac176d015bec64f811afb5965fce077a03c"}, + {file = "orjson-3.10.16-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31b98bc9b40610fec971d9a4d67bb2ed02eec0a8ae35f8ccd2086320c28526ca"}, + {file = "orjson-3.10.16-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0ce243f5a8739f3a18830bc62dc2e05b69a7545bafd3e3249f86668b2bcd8e50"}, + {file = "orjson-3.10.16-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:64792c0025bae049b3074c6abe0cf06f23c8e9f5a445f4bab31dc5ca23dbf9e1"}, + {file = "orjson-3.10.16-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:ea53f7e68eec718b8e17e942f7ca56c6bd43562eb19db3f22d90d75e13f0431d"}, + {file = "orjson-3.10.16-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a741ba1a9488c92227711bde8c8c2b63d7d3816883268c808fbeada00400c164"}, + {file = "orjson-3.10.16-cp311-cp311-win32.whl", hash = "sha256:c7ed2c61bb8226384c3fdf1fb01c51b47b03e3f4536c985078cccc2fd19f1619"}, + {file = "orjson-3.10.16-cp311-cp311-win_amd64.whl", hash = "sha256:cd67d8b3e0e56222a2e7b7f7da9031e30ecd1fe251c023340b9f12caca85ab60"}, + {file = "orjson-3.10.16-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:6d3444abbfa71ba21bb042caa4b062535b122248259fdb9deea567969140abca"}, + {file = "orjson-3.10.16-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:30245c08d818fdcaa48b7d5b81499b8cae09acabb216fe61ca619876b128e184"}, + {file = "orjson-3.10.16-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0ba1d0baa71bf7579a4ccdcf503e6f3098ef9542106a0eca82395898c8a500a"}, + {file = "orjson-3.10.16-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eb0beefa5ef3af8845f3a69ff2a4aa62529b5acec1cfe5f8a6b4141033fd46ef"}, + {file = "orjson-3.10.16-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6daa0e1c9bf2e030e93c98394de94506f2a4d12e1e9dadd7c53d5e44d0f9628e"}, + {file = "orjson-3.10.16-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9da9019afb21e02410ef600e56666652b73eb3e4d213a0ec919ff391a7dd52aa"}, + {file = "orjson-3.10.16-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:daeb3a1ee17b69981d3aae30c3b4e786b0f8c9e6c71f2b48f1aef934f63f38f4"}, + {file = "orjson-3.10.16-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80fed80eaf0e20a31942ae5d0728849862446512769692474be5e6b73123a23b"}, + {file = "orjson-3.10.16-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73390ed838f03764540a7bdc4071fe0123914c2cc02fb6abf35182d5fd1b7a42"}, + {file = "orjson-3.10.16-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a22bba012a0c94ec02a7768953020ab0d3e2b884760f859176343a36c01adf87"}, + {file = "orjson-3.10.16-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5385bbfdbc90ff5b2635b7e6bebf259652db00a92b5e3c45b616df75b9058e88"}, + {file = "orjson-3.10.16-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:02c6279016346e774dd92625d46c6c40db687b8a0d685aadb91e26e46cc33e1e"}, + {file = "orjson-3.10.16-cp312-cp312-win32.whl", hash = "sha256:7ca55097a11426db80f79378e873a8c51f4dde9ffc22de44850f9696b7eb0e8c"}, + {file = "orjson-3.10.16-cp312-cp312-win_amd64.whl", hash = "sha256:86d127efdd3f9bf5f04809b70faca1e6836556ea3cc46e662b44dab3fe71f3d6"}, + {file = "orjson-3.10.16-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:148a97f7de811ba14bc6dbc4a433e0341ffd2cc285065199fb5f6a98013744bd"}, + {file = "orjson-3.10.16-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:1d960c1bf0e734ea36d0adc880076de3846aaec45ffad29b78c7f1b7962516b8"}, + {file = "orjson-3.10.16-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a318cd184d1269f68634464b12871386808dc8b7c27de8565234d25975a7a137"}, + {file = "orjson-3.10.16-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:df23f8df3ef9223d1d6748bea63fca55aae7da30a875700809c500a05975522b"}, + {file = "orjson-3.10.16-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b94dda8dd6d1378f1037d7f3f6b21db769ef911c4567cbaa962bb6dc5021cf90"}, + {file = "orjson-3.10.16-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f12970a26666a8775346003fd94347d03ccb98ab8aa063036818381acf5f523e"}, + {file = "orjson-3.10.16-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:15a1431a245d856bd56e4d29ea0023eb4d2c8f71efe914beb3dee8ab3f0cd7fb"}, + {file = "orjson-3.10.16-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c83655cfc247f399a222567d146524674a7b217af7ef8289c0ff53cfe8db09f0"}, + {file = "orjson-3.10.16-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:fa59ae64cb6ddde8f09bdbf7baf933c4cd05734ad84dcf4e43b887eb24e37652"}, + {file = "orjson-3.10.16-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:ca5426e5aacc2e9507d341bc169d8af9c3cbe88f4cd4c1cf2f87e8564730eb56"}, + {file = "orjson-3.10.16-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:6fd5da4edf98a400946cd3a195680de56f1e7575109b9acb9493331047157430"}, + {file = "orjson-3.10.16-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:980ecc7a53e567169282a5e0ff078393bac78320d44238da4e246d71a4e0e8f5"}, + {file = "orjson-3.10.16-cp313-cp313-win32.whl", hash = "sha256:28f79944dd006ac540a6465ebd5f8f45dfdf0948ff998eac7a908275b4c1add6"}, + {file = "orjson-3.10.16-cp313-cp313-win_amd64.whl", hash = "sha256:fe0a145e96d51971407cb8ba947e63ead2aa915db59d6631a355f5f2150b56b7"}, + {file = "orjson-3.10.16-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:c35b5c1fb5a5d6d2fea825dec5d3d16bea3c06ac744708a8e1ff41d4ba10cdf1"}, + {file = "orjson-3.10.16-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9aac7ecc86218b4b3048c768f227a9452287001d7548500150bb75ee21bf55d"}, + {file = "orjson-3.10.16-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6e19f5102fff36f923b6dfdb3236ec710b649da975ed57c29833cb910c5a73ab"}, + {file = "orjson-3.10.16-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:17210490408eb62755a334a6f20ed17c39f27b4f45d89a38cd144cd458eba80b"}, + {file = "orjson-3.10.16-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fbbe04451db85916e52a9f720bd89bf41f803cf63b038595674691680cbebd1b"}, + {file = "orjson-3.10.16-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6a966eba501a3a1f309f5a6af32ed9eb8f316fa19d9947bac3e6350dc63a6f0a"}, + {file = "orjson-3.10.16-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01e0d22f06c81e6c435723343e1eefc710e0510a35d897856766d475f2a15687"}, + {file = "orjson-3.10.16-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:7c1e602d028ee285dbd300fb9820b342b937df64d5a3336e1618b354e95a2569"}, + {file = "orjson-3.10.16-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:d230e5020666a6725629df81e210dc11c3eae7d52fe909a7157b3875238484f3"}, + {file = "orjson-3.10.16-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:0f8baac07d4555f57d44746a7d80fbe6b2c4fe2ed68136b4abb51cfec512a5e9"}, + {file = "orjson-3.10.16-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:524e48420b90fc66953e91b660b3d05faaf921277d6707e328fde1c218b31250"}, + {file = "orjson-3.10.16-cp39-cp39-win32.whl", hash = "sha256:a9f614e31423d7292dbca966a53b2d775c64528c7d91424ab2747d8ab8ce5c72"}, + {file = "orjson-3.10.16-cp39-cp39-win_amd64.whl", hash = "sha256:c338dc2296d1ed0d5c5c27dfb22d00b330555cb706c2e0be1e1c3940a0895905"}, + {file = "orjson-3.10.16.tar.gz", hash = "sha256:d2aaa5c495e11d17b9b93205f5fa196737ee3202f000aaebf028dc9a73750f10"}, ] [[package]] @@ -6658,6 +6647,17 @@ files = [ {file = "widgetsnbextension-4.0.13.tar.gz", hash = "sha256:ffcb67bc9febd10234a362795f643927f4e0c05d9342c727b65d2384f8feacb6"}, ] +[[package]] +name = "word2number" +version = "1.1" +description = "Convert number words eg. three hundred and forty two to numbers (342)." +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "word2number-1.1.zip", hash = "sha256:70e27a5d387f67b04c71fbb7621c05930b19bfd26efd6851e6e0f9969dcde7d0"}, +] + [[package]] name = "wrapt" version = "1.17.2" @@ -6980,4 +6980,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.1" python-versions = "^3.12" -content-hash = "03435060e2b2a4ae17acdcf99612942ba403b87d70ed70f98fd25ce7bbd33ae1" +content-hash = "15c3df839b1215946e8c25d276c6bfdb7c49f9e50bd1bd8707679c5a95aa2ac9" diff --git a/fieldExtraction/pyproject.toml b/fieldExtraction/pyproject.toml index bd25725..144e303 100644 --- a/fieldExtraction/pyproject.toml +++ b/fieldExtraction/pyproject.toml @@ -27,6 +27,8 @@ nltk = "^3.9.1" rank-bm25 = "^0.2.2" sentence-transformers = "^3.4.1" faiss-cpu = "^1.10.0" +word2number = "^1.1" +orjson = "^3.10.16" [tool.poetry.group.dev.dependencies] black = "^24.10.0" diff --git a/fieldExtraction/src/investment/postprocess.py b/fieldExtraction/src/investment/postprocess.py index c8cd4f9..c365c2e 100644 --- a/fieldExtraction/src/investment/postprocess.py +++ b/fieldExtraction/src/investment/postprocess.py @@ -1,36 +1,250 @@ +import json +import re from src.constants.investment_columns import COLUMN_ORDER import src.config as config from src import postprocessing_funcs from crosswalk.crosswalk_utils import CrosswalkBuilder, apply_crosswalk -from pathlib import Path -import pandas as pd import os +import ast +from word2number import w2n +from datetime import datetime +import pandas as pd from src.prompts.investment_prompts import FieldSet, invoke_derived_term_date + # Determine the base directory for the project BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # Path to the mappings folder MAPPINGS_DIR = os.path.join(BASE_DIR, 'crosswalk', 'mappings') + + +def normalize_indicator_field(value: str) -> str: + """ + Normalize the indicator field value to 'Y' or 'N'. + + Args: + value (str): The input value from an _IND field. + Returns: + str: 'Y' if the value is 'Y', otherwise 'N'. + """ + if isinstance(value, str) and value.strip().upper() == "Y": + return "Y" + return 'N' + + +def format_rate_fields_with_commas(value: str) -> str: + """ + Format the rate with commas and two decimal places. + + Args: + value (str): A string representing a numeric value. + Returns: + str: The formatted rate string. + """ + if value is None: + return "0.00" + + try: + numeric_value = float(value) + return f"{numeric_value:,.2f}" + except ValueError: + return "0.00" + + +def remove_hyphens(value: str) -> str: + """ + Remove hyphens from the input string. + + Args: + value (str): The input string. + Returns: + str: The cleaned string with hyphens removed. + """ + if not isinstance(value, str): + return "" + return value.replace("-", "") + + +def flatten_singleton_string_list(list_str: str) -> str: + """Take in a list within a string, and if it has a single element, return that element. + If it has multiple elements, return the list as a string. + + Args: + list_str (str): A string representation of a list. + + Returns: + str: The single element if the list has one element, otherwise the list as a string. + """ + try: + # Convert the string to a list using ast.literal_eval + list_obj = ast.literal_eval(list_str) + if len(list_obj) == 1: + return list_obj[0] + return list_obj + except (ValueError, SyntaxError): + return list_str + +def is_yyyymmdd_format(date_str: str) -> bool: + """ + Check if a string is in YYYYMMDD format. + """ + try: + datetime.strptime(date_str, "%Y%m%d") + return True + except (ValueError, TypeError): + return False + +def reformat_to_yyyymmdd(date_str: str) -> str: + """ + Reformat a date string to YYYYMMDD. + Logs before and after if formatting is successful. + """ + if not isinstance(date_str, str): + return date_str + + known_formats = [ + "%Y-%m-%d", "%m/%d/%Y", "%Y/%m/%d", "%d-%b-%Y", "%d/%m/%Y" + ] + + for fmt in known_formats: + try: + reformatted = datetime.strptime(date_str, fmt).strftime("%Y%m%d") + return reformatted + except ValueError: + continue + return date_str + +def date_postprocess(df, field_json_path): + """ + Postprocess the date fields in the DataFrame. + """ + # Date field postprocessing using FieldSet definition + date_fields = FieldSet(file_path=field_json_path).filter(format="date") + for field in date_fields.fields: + if field.field_name in df.columns: + df[field.field_name] = list(map(postprocessing_funcs.date_postprocessing, df[field.field_name])) + + # Derived termination date + df['AARETE_DERIVED_TERMINATION_DT'] = list(map(invoke_derived_term_date, df['AARETE_DERIVED_EFFECTIVE_DT'], df['TERMINATION_DT'])) + + return df + +def normalize_auto_renewal_term(text: str) -> str: + """ + Normalize the auto-renewal term to a standard format. + + Args: + text (str): The input auto-renewal term. + + Returns: + str: The normalized auto-renewal term. + """ + if not text or not isinstance(text, str): + return "" + + # Convert to lowercase and strip whitespace + text = text.lower().strip() + + # Remove numbers enclosed in parentheses (e.g., "(1) ", "(12) ") + text = re.sub(r"\(\d+\) ", "", text) + + # Define replacements + replacements = { + r"\bone\b": "1", + r"\btwo\b": "2", + r"\bthree\b": "3", + r"\btwelve\b": "12", + r"year to year": "1 year", + r"12 months": "1 year", + r"\btwelve months\b": "1 year", + r"\bone year\b": "1 year", + r"\bone-year\b": "1 year", + r"\b1 year\b": "1 year", + r"\bmonth to month\b": "1 month", + } + + # Apply replacements + for pattern, replacement in replacements.items(): + text = re.sub(pattern, replacement, text) + + # Remove any remaining parentheses or extra spaces + text = text.strip().replace("(", "").replace(")", "") + + text = text.replace("-", " ") + if text == "year": + text = "1 year" + + return text + +def normalize_cpt_fields(value): + """ + Normalize code values to a list format. + + Args: + value (Any): Input value which could be a list, string, or other types. + + Returns: + list: A normalized list of string codes. + """ + if pd.isna(value): # Handle NaN values + result = [] + elif isinstance(value, (list, tuple)): # If already a list or tuple + result = [str(v).strip() for v in value] + elif isinstance(value, str): # If it's a string + value = value.strip() + if value.startswith("[") and value.endswith("]"): # String representation of a list + try: + result = [str(v).strip() for v in ast.literal_eval(value)] + except (ValueError, SyntaxError): + result = [value] # If parsing fails, treat it as a single value + elif "-" in value: # If it's a range + result = [value] + else: # Single value + result = [value] + elif isinstance(value, (float, int)): # If it's a number + result = [str(int(value)).strip()] + else: + result = [str(value).strip()] # Default case for other types + + return result # Return the list directly + def postprocess(df): if df.shape[0] > 0: - # Add postprocessing here df['CLIENT_NAME'] = config.CLIENT_NAME # TIN and NPI postprocessing df = postprocessing_funcs.clean_tin_npi_other(df) + + df = date_postprocess(df, config.FIELD_JSON_PATH) - # date postprocessing - date_fields = FieldSet(file_path=config.FIELD_JSON_PATH).filter(format="date") - for field in date_fields.fields: - df[field.field_name] = list(map(postprocessing_funcs.date_postprocessing, df[field.field_name])) # we use `map` to allow memoization to work + # Format rate fields + df['REIMB_FEE_RATE'] = df['REIMB_FEE_RATE'].apply(format_rate_fields_with_commas) + df['REIMB_PCT_RATE'] = df['REIMB_PCT_RATE'].apply(format_rate_fields_with_commas) + + + for col in df.columns: + if "_IND" in col: + df[col] = df[col].apply(normalize_indicator_field) + # Normalize _IND fields and clean up TIN/NPI fields + if "TIN" in col or "NPI" in col: + df[col] = df[col].apply(remove_hyphens) + # Apply the flatten_singleton_string_list function to the 'CPT' column + if "_CD" in col and "CPT" not in col: + df[col] = df[col].apply(flatten_singleton_string_list) + # Convert DATE to YYYYMMDD format + if "_DT" in col or "DATE" in col: + df['DATE_VALID'] = df[col].apply(is_yyyymmdd_format) + df.loc[~df['DATE_VALID'], col] = df.loc[~df['DATE_VALID'], col].apply(reformat_to_yyyymmdd) + if "CPT" in col: + df[col] = df[col].apply(normalize_cpt_fields) + df[col] = df[col].apply(lambda x: str(x) if isinstance(x, list) else x) - # Derived termination date - df['AARETE_DERIVED_TERMINATION_DT'] = list(map(invoke_derived_term_date, df['AARETE_DERIVED_EFFECTIVE_DT'], df['TERMINATION_DT'])) - - # Special handling for AARETE_DERIVED_TERMINATION_DT - # Overwrite AARETE_DERIVED_TERMINATION_DT "9999/01/01" if AUTO_RENEWAL_IND is "Y" - if "AARETE_DERIVED_TERMINATION_DT" in df.columns and "AUTO_RENEWAL_IND" in df.columns: + # Normalize the 'AUTO_RENEWAL_TERM' column + df["AUTO_RENEWAL_TERM"] = df["AUTO_RENEWAL_TERM"].apply(normalize_auto_renewal_term) + + # Special handling for derived termination date if auto-renewal is present + if "AARETE_DERIVED_TERMINATION_DT" in df.columns and "CONTRACT_AUTO_RENEWAL_IND" in df.columns: df.loc[ df["AUTO_RENEWAL_IND"] == "Y", "AARETE_DERIVED_TERMINATION_DT" @@ -61,7 +275,7 @@ def postprocess(df): # Standardize output column order - this should ALWAYS be the final postprocessing step df = postprocessing_funcs.reorder_columns(df, COLUMN_ORDER) - + return df else: - return df + return df \ No newline at end of file diff --git a/fieldExtraction/tests/investment_postprocess_test.py b/fieldExtraction/tests/investment_postprocess_test.py new file mode 100644 index 0000000..5a20dfd --- /dev/null +++ b/fieldExtraction/tests/investment_postprocess_test.py @@ -0,0 +1,104 @@ +from unittest.mock import patch, MagicMock +import unittest +from unittest.mock import patch +import pandas as pd +import src.config as config +from src.investment.postprocess import ( + normalize_indicator_field, + format_rate_fields_with_commas, + remove_hyphens, + flatten_singleton_string_list, + is_yyyymmdd_format, + reformat_to_yyyymmdd, + date_postprocess, + normalize_auto_renewal_term, + normalize_cpt_fields +) + + +class TestPostprocessFunctions(unittest.TestCase): + + def test_normalize_indicator_field(self): + self.assertEqual(normalize_indicator_field("Y"), "Y") + self.assertEqual(normalize_indicator_field("y"), "Y") + self.assertEqual(normalize_indicator_field("N"), "N") + self.assertEqual(normalize_indicator_field(""), "N") + self.assertEqual(normalize_indicator_field(None), "N") + + def test_format_rate_fields_with_commas(self): + self.assertEqual(format_rate_fields_with_commas("1234.567"), "1,234.57") + self.assertEqual(format_rate_fields_with_commas("1000"), "1,000.00") + self.assertEqual(format_rate_fields_with_commas("abc"), "0.00") + self.assertEqual(format_rate_fields_with_commas(None), "0.00") + + def test_remove_hyphens(self): + self.assertEqual(remove_hyphens("123-45-6789"), "123456789") + self.assertEqual(remove_hyphens("123-456"), "123456") + self.assertEqual(remove_hyphens(None), "") + self.assertEqual(remove_hyphens(""), "") + + def test_flatten_singleton_string_list(self): + self.assertEqual(flatten_singleton_string_list("['123']"), "123") + self.assertEqual(flatten_singleton_string_list("['123', '456']"), ['123', '456']) + self.assertEqual(flatten_singleton_string_list("invalid"), "invalid") + self.assertEqual(flatten_singleton_string_list(None), None) + + def test_is_yyyymmdd_format(self): + self.assertTrue(is_yyyymmdd_format("20230101")) + self.assertFalse(is_yyyymmdd_format("01/01/2023")) + self.assertFalse(is_yyyymmdd_format("2023-01-01")) + self.assertFalse(is_yyyymmdd_format(None)) + + def test_reformat_to_yyyymmdd(self): + self.assertEqual(reformat_to_yyyymmdd("2023-01-01"), "20230101") + self.assertEqual(reformat_to_yyyymmdd("01/01/2023"), "20230101") + self.assertEqual(reformat_to_yyyymmdd("01-Jan-2023"), "20230101") + self.assertEqual(reformat_to_yyyymmdd("invalid"), "invalid") + self.assertEqual(reformat_to_yyyymmdd(None), None) + + @patch("src.utils.llm_utils.invoke_claude") # ✅ correct target + @patch("src.investment.postprocess.invoke_derived_term_date") + @patch("src.investment.postprocess.FieldSet") + def test_date_postprocess(self, mock_fieldset, mock_derived_term_date, mock_invoke_claude): + + # ✅ Only mock fields that exist in the test DataFrame + mock_fieldset.return_value.filter.return_value.fields = [ + MagicMock(field_name="CONTRACT_EFFECTIVE_DT"), + MagicMock(field_name="CONTRACT_TERMINATION_DT"), + MagicMock(field_name="TERMINATION_DT") + ] + + # ✅ Simulate LLM response that date_postprocessing would extract from + mock_invoke_claude.return_value = "|20230101|" + mock_derived_term_date.return_value = "20231231" + + df = pd.DataFrame({ + "CONTRACT_EFFECTIVE_DT": ["2023-01-01", "01/01/2023", "invalid"], + "CONTRACT_TERMINATION_DT": ["2023-12-31", "12/31/2023", "invalid"], + "TERMINATION_DT": ["2023-12-31", "12/31/2023", "invalid"], + "AARETE_DERIVED_EFFECTIVE_DT": ["2023-01-01", "2023-01-01", "2023-01-01"] + }) + + result_df = date_postprocess(df, "mock_path") + + assert result_df["CONTRACT_EFFECTIVE_DT"].tolist() == ["20230101", "20230101", "20230101"] + assert result_df["AARETE_DERIVED_TERMINATION_DT"].tolist() == ["20231231", "20231231", "20231231"] + + + def test_normalize_auto_renewal_term(self): + self.assertEqual(normalize_auto_renewal_term("12 months"), "1 year") + self.assertEqual(normalize_auto_renewal_term("one year"), "1 year") + self.assertEqual(normalize_auto_renewal_term("month to month"), "1 month") + self.assertEqual(normalize_auto_renewal_term("(12) 12 months"), "1 year") + self.assertEqual(normalize_auto_renewal_term(None), "") + + def test_normalize_cpt_fields(self): + self.assertEqual(normalize_cpt_fields("[123, 456]"), ["123", "456"]) + self.assertEqual(normalize_cpt_fields("123-456"), ["123-456"]) + self.assertEqual(normalize_cpt_fields("123"), ["123"]) + self.assertEqual(normalize_cpt_fields(None), []) + self.assertEqual(normalize_cpt_fields(123), ["123"]) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file