Merged in hotfix/postprocess_csv (pull request #990)

Hotfix/postprocess csv

* date issue_fix

* black_format

* Merged dev into hotfix/postprocess_csv


Approved-by: Katon Minhas

src/scripts/postprocess_existing_output.py

@@ -1,28 +1,38 @@
+import os
+import sys
 import json
 import zipfile
 from pathlib import Path
 
 import pandas as pd
+
 from src.constants.constants import Constants
 from src.pipelines.shared.postprocessing.postprocess import postprocess
+from src.pipelines.shared.postprocessing.postprocessing_funcs import (
+    validate_and_reformat_date,
+)
 
 # File paths
-input_excel_path = "input.csv"
-output_csv_path = "output.csv"
+input_path = sys.argv[1]  # pass xlsx/csv path as first arg
+output_csv_path = str(Path(input_path).with_suffix(".csv"))
 
-# Read Excel file into DataFrame
-print(f"Reading Excel file: {input_excel_path}")
-df = pd.read_csv(input_excel_path, dtype=str)
-print(f"Loaded {len(df)} rows from Excel")
+# Read input file (xlsx or csv) into DataFrame
+ext = os.path.splitext(input_path)[1].lower()
+print(f"Reading input file: {input_path}")
+if ext in [".xlsx", ".xls", ".xlsm"]:
+    df = pd.read_excel(input_path, dtype=str, engine="openpyxl").fillna("")
+elif ext == ".xlsb":
+    df = pd.read_excel(input_path, dtype=str, engine="pyxlsb").fillna("")
+else:
+    df = pd.read_csv(input_path, dtype=str, keep_default_na=False)
+print(f"Loaded {len(df)} rows, {len(df.columns)} columns")
 
-constants = Constants()
-
-# Process the DataFrame
-print("Processing data through postprocess()...")
-cc_df, dashboard_df = postprocess(df, constants)
-print(f"Processing complete. Output has {len(cc_df)} rows")
+# Format date columns to YYYY/MM/DD
+date_cols = [c for c in df.columns if "_DT" in c or "DATE" in c]
+print(f"Formatting {len(date_cols)} date columns: {date_cols}")
+for col in date_cols:
+    df[col] = df[col].apply(validate_and_reformat_date)
 
 # Write to CSV
 print(f"Writing CSV file: {output_csv_path}")
-cc_df.to_csv(output_csv_path, index=False)
-print("Done!")
+df.to_csv(output_csv_path, index=False)