Merged in feature/multithreading (pull request #828)

Feature/multithreading

* fixes

* Merge main into feature/multithreading

Resolved conflicts:
- Kept timing instrumentation in file_processing.py
- Kept new 3-step Exhibit-based approach for one-to-n processing
- Maintained parallelization improvements (20 workers)

Changes include:
- Timing utils integration for performance monitoring
- Increased max_workers from 5 to 20 across all components
- Parallelized code_breakout and grouper_breakout
- Fixed tin_npi_funcs function call parameters

* Fix max_workers error for empty documents

- Add check to skip parallel processing when no pages exist
- Use min(len(all_page_tasks), 20) to prevent max_workers=0
- Handles edge case of documents with no exhibits or pages

* Fix max_workers=0 errors in one_to_n_funcs

- Add checks before all ThreadPoolExecutor creations
- Prevents errors when processing empty lists:
  - carveout_and_special_case
  - breakout
  - special_case_breakout
  - filter_services_without_reimbursements
  - run_lob_relationship
- Ensures executor only created when there are items to process

* Reduce code processing parallelism to prevent API throttling

Lower max_workers from 20 to 10 for code_breakout and grouper_breakout
to prevent overwhelming Bedrock API with concurrent requests

* fixed

* Merged main into feature/multithreading

* Move documentation into folder

* Fix logging statements

* Merge branch 'main' into feature/multithreading

* Refactor for clarity

* Update previous exhibit passing logic

* properly simplify exhibits

* Merged main into feature/multithreading

* update conditional for None

* exhibit multithreading changed

* exhibit multithreading changed

* Merge remote-tracking branch 'origin/main' into feature/multithreading

* synced with main

* Parallelize dynamic assignment, refactor HSC field worker, add timing/exhibit unit tests, tidy imports/ignore helpers

* analyze_regression.py edited online with Bitbucket
* count_pages.py edited online with Bitbucket
* compare_regressed_with_baseline.py edited online with Bitbucket
* simple_testbed_compare.py edited online with Bitbucket
* run_testbed_metrics_regressed.py edited online with Bitbucket

Approved-by: Katon Minhas

fieldExtraction/src/codes/code_funcs.py

@@ -1,6 +1,7 @@
 import ast
 import json
 import logging
+import concurrent.futures
 import os
 import re
 
@@ -720,15 +721,20 @@ def code_breakout(merged_results: pd.DataFrame, constants: Constants):
     Returns:
         pd.DataFrame: DataFrame with updated code fields.
     """
+    import concurrent.futures
 
     # Fill empty AARETE_DERIVED_CLAIM_TYPE_CD with mode
     answer_dicts = fill_claim_type(merged_results.to_dict(orient="records"))
 
-    answer_dicts_with_code = []
-    for answer_dict in answer_dicts:
+    # Parallelize code extraction
+    def process_single_code(answer_dict):
         code_answer_dict = extract_codes_from_service(answer_dict, constants)
         answer_dict.update(code_answer_dict)
-        answer_dicts_with_code.append(answer_dict)
+        return answer_dict
+
+    max_workers = min(len(answer_dicts), 10)
+    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+        answer_dicts_with_code = list(executor.map(process_single_code, answer_dicts))
 
     # Fill in GROUPER_CD_DESC based on GROUPER_TYPE and GROUPER_CD
     answer_dicts_with_code = [
@@ -754,32 +760,52 @@ def grouper_breakout(results_with_code: pd.DataFrame):
     Returns:
         pd.DataFrame: DataFrame with updated grouper fields.
     """
-
     # try grouper fields again if they are empty and grouper_cd is not empty
     answer_dicts_with_code = results_with_code.to_dict(orient="records")
-    final_answer_dicts = []
     GROUPER_QUESTIONS = FieldSet(
         file_path=config.FIELD_JSON_PATH, field_type="grouper_breakout"
     ).print_prompt_dict()
+
+    # Separate dicts that need grouper breakout from those that don't
+    needs_breakout = []
+    no_breakout_needed = []
+
     for answer_dict in answer_dicts_with_code:
         if string_utils.is_empty(
             answer_dict.get("GROUPER_TYPE")
         ) and not string_utils.is_empty(answer_dict.get("GROUPER_CD")):
-            # run groper breakout prompt
-            grouper_breakout_prompt = prompt_templates.GROUPER_BREAKOUT(
-                answer_dict.get("SERVICE_TERM", ""),
-                answer_dict.get("REIMB_TERM", ""),
-                GROUPER_QUESTIONS,
-            )
-            claude_answer_raw = llm_utils.invoke_claude(
-                grouper_breakout_prompt, "sonnet_latest", ""
-            )
-            try:
-                grouper_answer = string_utils.universal_json_load(claude_answer_raw)
-            except Exception as e:
-                grouper_answer = {"GROUPER_TYPE": e}
+            needs_breakout.append(answer_dict)
+        else:
+            no_breakout_needed.append(answer_dict)
 
-            answer_dict.update(grouper_answer)
-        final_answer_dicts.append(answer_dict)
+    logging.debug(f"Grouper breakout: {len(needs_breakout)} rows need LLM processing, {len(no_breakout_needed)} rows skip")
 
-    return pd.DataFrame(final_answer_dicts)
+    # Parallelize LLM calls for rows that need breakout
+    def process_grouper_breakout(answer_dict):
+        grouper_breakout_prompt = prompt_templates.GROUPER_BREAKOUT(
+            answer_dict.get("SERVICE_TERM", ""),
+            answer_dict.get("REIMB_TERM", ""),
+            GROUPER_QUESTIONS,
+        )
+        claude_answer_raw = llm_utils.invoke_claude(
+            grouper_breakout_prompt, "sonnet_latest", ""
+        )
+        try:
+            grouper_answer = string_utils.universal_json_load(claude_answer_raw)
+        except Exception as e:
+            grouper_answer = {"GROUPER_TYPE": e}
+
+        answer_dict.update(grouper_answer)
+        return answer_dict
+
+    if needs_breakout:
+        max_workers = min(len(needs_breakout), 10)
+        with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+            processed_breakout = list(executor.map(process_grouper_breakout, needs_breakout))
+    else:
+        processed_breakout = []
+
+    # Combine all results
+    final_answer_dicts = no_breakout_needed + processed_breakout
+
+    return pd.DataFrame(final_answer_dicts)