Merged in feature/doczy-codes (pull request #619)

Feature/doczy codes * Ignore service and procedure generic terms * UPdate PT/OT/ST codes * Standardize stop-loss * UPdate for multiple identical values * Modify code funcs * Run highest match Implicit Level first * Fix implicit code bug * Add PT individually to special case * change revenue code conditions * Merge branch 'main' into feature/doczy-codes * Merged main into feature/doczy-codes * Update main * Relocate create_code_embeddings back to scripts * Remove investment code_funcs (use codes.code_funcs now) * Add docstrings * stop-loss * Update unit tests * Remove test * Remove reimb_dates * fix dtypes * Remove prints * update prompt * Update code breakout unit test * Merge branch 'main' into feature/doczy-codes * Merge branch 'main' into feature/doczy-codes Approved-by: Alex Galarce
2025-08-01 22:36:01 +00:00
parent 84a09002fa
commit c2d5db19d4
47 changed files with 1843 additions and 1362 deletions
@@ -0,0 +1,30 @@
+
+import src.utils.embedding_utils as embedding_utils
+from crosswalk.crosswalk_utils import CrosswalkBuilder
+import src.codes.code_funcs as code_funcs
+
+import os
+import pandas as pd
+
+from sentence_transformers import SentenceTransformer
+
+# Load model
+roberta_model = SentenceTransformer("all-roberta-large-v1")
+
+mapping_dir = "crosswalk/mapping_csvs/proc_cd"
+pkl_dir = "embeddings"
+for filename in os.listdir(mapping_dir):
+    if filename.endswith(".csv") and "level" in filename:
+        print(f"Creating embeddings for {filename}")
+        stripped_filename = filename.replace(".csv", "")
+        mapping_df = pd.read_csv(os.path.join(mapping_dir, filename))
+        proc_crosswalk = CrosswalkBuilder().from_df(mapping_df, from_col="Code", to_col="Description")
+        proc_choices = [x for x in proc_crosswalk.mapping.values()]
+        os.makedirs(os.path.join(pkl_dir, stripped_filename), exist_ok=True)
+        proc_index = embedding_utils.create_faiss_index(choices=proc_choices, 
+                                                        model=roberta_model, 
+                                                        save_path=os.path.join(pkl_dir, stripped_filename, "faiss_index.bin"),
+                                                        embedding_path=os.path.join(pkl_dir, stripped_filename, "embeddings.npy"),
+                                                        choices_path=os.path.join(pkl_dir, stripped_filename, "choices.pkl")
+                                                        )        
+
@@ -1,54 +0,0 @@
-
-import src.utils.embedding_utils as embedding_utils
-from src.investment.code_funcs import get_proc_crosswalk, get_clean_value
-from crosswalk.crosswalk_utils import CrosswalkBuilder
-
-import faiss
-import pickle
-
-import os
-import pandas as pd
-import numpy as np
-
-from sentence_transformers import SentenceTransformer
-
-# Load model
-roberta_model = SentenceTransformer("all-roberta-large-v1")
-
-# If running proc_code for the first time
-"""
-mapping_dir = "crosswalk/mapping_csvs/proc_cd"
-pkl_dir = "embeddings"
-for filename in os.listdir(mapping_dir):
-    if filename.endswith(".csv"):
-        print(f"Creating embeddings for {filename}")
-        stripped_filename = filename.strip(".csv")
-        mapping_df = pd.read_csv(os.path.join(mapping_dir, filename))
-        proc_crosswalk = CrosswalkBuilder().from_df(mapping_df, from_col="Code", to_col="Description")
-        proc_choices = [get_clean_value(x) for x in proc_crosswalk.mapping.values()]
-        os.makedirs(os.path.join(pkl_dir, stripped_filename), exist_ok=True)
-        proc_index = embedding_utils.create_faiss_index(choices=proc_choices, 
-                                                        model=roberta_model, 
-                                                        save_path=os.path.join(pkl_dir, stripped_filename, "faiss_index.bin"),
-                                                        embedding_path=os.path.join(pkl_dir, stripped_filename, "embeddings.npy"),
-                                                        choices_path=os.path.join(pkl_dir, stripped_filename, "choices.pkl")
-                                                        )        
-"""
-# If running diag_code for the first time
-mapping_dir = "crosswalk/mapping_csvs/diag_cd"
-pkl_dir = "embeddings"
-for filename in os.listdir(mapping_dir):
-    if filename.endswith(".csv"):
-        print(f"Creating embeddings for {filename}")
-        stripped_filename = filename.strip(".csv")
-        mapping_df = pd.read_csv(os.path.join(mapping_dir, filename))
-        mapping_df = mapping_df.astype(str)
-        diag_crosswalk = CrosswalkBuilder().from_df(mapping_df, from_col="Code", to_col="Description")
-        diag_choices = [get_clean_value(x) for x in diag_crosswalk.mapping.values()]
-        os.makedirs(os.path.join(pkl_dir, stripped_filename), exist_ok=True)
-        proc_index = embedding_utils.create_faiss_index(choices=diag_choices,
-                                                        model=roberta_model,
-                                                        save_path=os.path.join(pkl_dir, stripped_filename, "faiss_index.bin"),
-                                                        embedding_path=os.path.join(pkl_dir, stripped_filename, "embeddings.npy"),
-                                                        choices_path=os.path.join(pkl_dir, stripped_filename, "choices.pkl")
-                                                        )