jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in hotfix/postprocess_csv (pull request #990)

Browse Source 
Hotfix/postprocess csv

* date issue_fix

* black_format

* Merged dev into hotfix/postprocess_csv


Approved-by: Katon Minhas
This commit is contained in:
Rahul Ailaboina
2026-05-01 13:54:22 +00:00
committed by Katon Minhas
parent b70759fda0
commit ef44beffeb
1 changed files with 24 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/scripts/postprocess_existing_output.py
+24 -14
View File
@@ -1,28 +1,38 @@
import os
import sys
import json import json
import zipfile import zipfile
from pathlib import Path from pathlib import Path
import pandas as pd import pandas as pd
from src.constants.constants import Constants from src.constants.constants import Constants
from src.pipelines.shared.postprocessing.postprocess import postprocess from src.pipelines.shared.postprocessing.postprocess import postprocess
from src.pipelines.shared.postprocessing.postprocessing_funcs import (
validate_and_reformat_date,
)
# File paths # File paths
input_excel_path = "input.csv" input_path = sys.argv[1] # pass xlsx/csv path as first arg
output_csv_path = "output.csv" output_csv_path = str(Path(input_path).with_suffix(".csv"))
# Read Excel file into DataFrame # Read input file (xlsx or csv) into DataFrame
print(f"Reading Excel file: {input_excel_path}") ext = os.path.splitext(input_path)[1].lower()
df = pd.read_csv(input_excel_path, dtype=str) print(f"Reading input file: {input_path}")
print(f"Loaded {len(df)} rows from Excel") if ext in [".xlsx", ".xls", ".xlsm"]:
df = pd.read_excel(input_path, dtype=str, engine="openpyxl").fillna("")
elif ext == ".xlsb":
df = pd.read_excel(input_path, dtype=str, engine="pyxlsb").fillna("")
else:
df = pd.read_csv(input_path, dtype=str, keep_default_na=False)
print(f"Loaded {len(df)} rows, {len(df.columns)} columns")
constants = Constants() # Format date columns to YYYY/MM/DD
date_cols = [c for c in df.columns if "_DT" in c or "DATE" in c]
# Process the DataFrame print(f"Formatting {len(date_cols)} date columns: {date_cols}")
print("Processing data through postprocess()...") for col in date_cols:
cc_df, dashboard_df = postprocess(df, constants) df[col] = df[col].apply(validate_and_reformat_date)
print(f"Processing complete. Output has {len(cc_df)} rows")
# Write to CSV # Write to CSV
print(f"Writing CSV file: {output_csv_path}") print(f"Writing CSV file: {output_csv_path}")
cc_df.to_csv(output_csv_path, index=False) df.to_csv(output_csv_path, index=False)
print("Done!")