Merged in feature/self-repair-main (pull request #313)

Feature/self repair main

* fixes

* fixes

* fixes

* fixes

* fixes

* fixes

* fixes

* fixes

* new testing

* new testing

* added unit tests under fieldExtraction/tests/main-unit-tests

* unit test fixes

* unit test fixes

* merge into main

* modified consolidate_output and slight changes to qa_qc_helpers to accomodate qaqc functionality w faizan and michael's changes

* merging self repair into main

* synced with main now

* fixes

* provider billing removed

* provider billing removed

* provider billing fixed

* commented out load dotenv in config

* Merged main into feature/self-repair-main

* modified check_s3_bucket_exists function by returning False if credentials not found for pipeline

* Merge branch 'feature/self-repair-main' of https://bitbucket.org/aarete/doczy.ai into feature/self-repair-main


Approved-by: Michael McGuinness

fieldExtraction/src/utils.py

@@ -2,10 +2,8 @@ import os
 import re
 import pandas as pd
 import shutil
-import time
-import sys
 import numpy as np
-from collections import defaultdict
+import tracking 
 import config
 from botocore.exceptions import ClientError
 from io import StringIO
@@ -188,6 +186,11 @@ def preprocess_text_file(file_path):
     pages = re.split(r"Start of Page No\. = \d+", text)
     return pages
 
+def calculate_progress_stats(completed_ac, total_ac, completed_b, total_b):
+    """Calculate and return progress statistics"""
+    ac_progress = min((completed_ac / total_ac * 100), 100.0) if total_ac > 0 else 0
+    b_progress = min((completed_b / total_b * 100), 100.0) if total_b > 0 else 0
+    return ac_progress, b_progress
 
 def format_td_check(td_dicts, dont_include_list):
     final_str = ""
@@ -275,32 +278,56 @@ def contains_lol(text, page='1'):
 
 
 def filter_already_processed(input_dict, input_dict_b):
-    already_processed_ac, already_processed_b = [], []
-    for folder_name in os.listdir(config.OUTPUT_DIRECTORY):
-        # AC
-        if config.AC_RESULTS_NAME in os.listdir(
-            os.path.join(config.OUTPUT_DIRECTORY, folder_name)
-        ):
-            already_processed_ac.append(folder_name + ".txt")
-        # B
-        if config.B_RESULTS_NAME in os.listdir(
-            os.path.join(config.OUTPUT_DIRECTORY, folder_name)
-        ):
-            already_processed_b.append(folder_name + ".txt")
+    """Filter out already processed files based on either local files or S3 master tracking"""
+    if config.WRITE_TO_S3:
+        # Get processed files from S3 master tracking for this specific batch
+        already_processed_ac, already_processed_b = tracking.get_already_processed_files(config.BATCH_ID)
+        
+        # Keep files that still need either AC or B processing
+        input_dict_ac = {
+            k: v for k, v in input_dict.items() 
+            if k not in already_processed_ac
+        }
+        
+        input_dict_b = {
+            k: v for k, v in input_dict_b.items() 
+            if k not in already_processed_b
+        }
+        
+        print(f"Filtering based on S3 master tracking for batch {config.BATCH_ID}:")
+        print(f"AC: Found {len(already_processed_ac)} already processed, {len(input_dict_ac)} remaining")
+        print(f"B: Found {len(already_processed_b)} already processed, {len(input_dict_b)} remaining")
+        
+    else:
+        # Original local directory checking logic
+        already_processed_ac, already_processed_b = [], []
+        for folder_name in os.listdir(config.OUTPUT_DIRECTORY):
+            folder_path = os.path.join(config.OUTPUT_DIRECTORY, folder_name)
+            if os.path.isdir(folder_path):
+                if config.AC_RESULTS_NAME in os.listdir(folder_path):
+                    already_processed_ac.append(folder_name + ".txt")
+                if config.B_RESULTS_NAME in os.listdir(folder_path):
+                    already_processed_b.append(folder_name + ".txt")
+        
+        input_dict_ac = {
+            k: v for k, v in input_dict.items()
+            if k not in already_processed_ac
+        }
+        
+        input_dict_b = {
+            k: v for k, v in input_dict_b.items()
+            if k not in already_processed_b
+        }
+        
+        print(f"Filtering based on local output directory:")
+        print(f"AC: Found {len(already_processed_ac)} already processed, {len(input_dict_ac)} remaining")
+        print(f"B: Found {len(already_processed_b)} already processed, {len(input_dict_b)} remaining")
 
-    input_dict_ac = {
-        key: input_dict[key]
-        for key in input_dict.keys()
-        if key not in already_processed_ac
-    }
-    input_dict_b = {
-        key: input_dict_b[key]
-        for key in input_dict_b.keys()
-        if key not in already_processed_b
-    }
     return input_dict_ac, input_dict_b
 
 
+
+
 def is_empty(value):
     if pd.isna(value):
         return True