Merged in feature/standardize-column-order (pull request #420)

Feature/standardize column order

* add column reordering

* Standardize column order

* Merged main into feature/standardize-column-order


Approved-by: Alex Galarce

fieldExtraction/src/constants/investment_columns.py

@@ -0,0 +1,103 @@
+
+# This is the ONE AND ONLY source of investment column names and order. Do not reference anything else
+COLUMN_ORDER = [
+    "CONTRACT_SID",
+    "CONTRACT_FILE_NAME",
+    "CONTRACT_CLASSIFICATION",
+    "AARETE_DERIVED_CONTRACT_CLASSIFICATION",
+    "CONTRACT_TITLE",
+    "CONTRACT_AMENDMENT_NUM",
+    "CLIENT_NAME",
+    "PAYER_NAME",
+    "PROV_GROUP_TIN",
+    "PROV_GROUP_NPI",
+    "PROV_GROUP_NAME_FULL",
+    "PROV_TIN_OTHER",
+    "PROV_NPI_OTHER",
+    "PROV_REIMBURSEMENT_TIN",
+    "PROV_REIMBURSEMENT_NPI",
+    "PROV_FULL_NAME",
+    "CONTRACT_EFFECTIVE_DT",
+    "CONTRACT_TERMINATION_DT",
+    "AARETE_DERIVED_TERMINATION_DATE",
+    "CONTRACT_AUTO_RENEWAL_IND",
+    "CONTRACT_AUTO_RENEWAL_TERM",
+    "REIMBURSEMENT_EFFECTIVE_DT",
+    "REIMBURSEMENT_TERMINATION_DT",
+    "CONTRACT_SIGNATORY_COMPLETE_IND",
+    "CONTRACT_CLAIM_TYPE_CD",
+    "AARETE_DERIVED_CLAIM_TYPE_CD",
+    "EXHIBIT_NAME",
+    "EXHIBIT_PAGE",
+    "CONTRACT_PRODUCT",
+    "REIMBURSEMENT_PROV_NAME",
+    "AARETE_DERIVED_PRODUCT",
+    "CONTRACT_LINE_OF_BUSINESS",
+    "AARETE_DERIVED_LINE_OF_BUSINESS",
+    "CONTRACT_PROGRAM",
+    "AARETE_DERIVED_PROGRAM",
+    "CONTRACT_NETWORK",
+    "AARETE_DERIVED_NETWORK",
+    "CONTRACT_SERVICE_AREA",
+    "AARETE_DERIVED_SERVICE_AREA",
+    "CONTRACT_PROV_TYPE",
+    "AARETE_DERIVED_PROV_TYPE",
+    "REIMBURSEMENT_PROV_NAME",
+    "CONTRACT_PROV_SPECIALTY",
+    "AARETE_DERIVED_PROV_SPECIALTY",
+    "CONTRACT_BILL_TYPE_CD",
+    "AARETE_DERIVED_BILL_TYPE_CD",
+    "PROV_PATIENT_AGE_MIN",
+    "PROV_PATIENT_AGE_MAX",
+    "CONTRACT_SERVICE_CD_OR_DESC",
+    "CONTRACT_REIMBURSEMENT_METHOD", 
+    "AARETE_DERIVED_REIMBURSEMENT_METHOD",
+    "CONTRACT_REIMBURSEMENT_FEE_RATE",
+    "CONTRACT_REIMBURSEMENT_PERCENT_RATE",
+    "CONTRACT_FEE_SCHEDULE_DESC",
+    "AARETE_DERIVED_FEE_SCHEDULE",
+    "CONTRACT_FEE_SCHEDULE_VERSION",
+    "AARETE_DERIVED_FEE_SCHEDULE_VERSION",
+    "CONTRACT_LESSER_OF_IND",
+    "CONTRACT_GREATER_OF_IND",
+    "CONTRACT_CARVEOUT_IND",
+    "CONTRACT_CARVEOUT_CD",
+    "CONTRACT_DEFAULT_IND",
+    "CPT4_PROC_CD" ,
+    "CPT4_PROC_DESC",
+    "CPT4_PROC_MOD",
+    "CPT4_PROC_MOD_DESC",
+    "REVENUE_CD",
+    "REVENUE_CD_DESC",
+    "DIAG_CD",
+    "DIAG_CD_DESC",
+    "GROUPER_TYPE",
+    "FACILITY_GROUPER_CD",
+    "FACILITY_GROUPER_DESC",
+    "LINE_NDC_NUM" ,
+    "LINE_NDC_DESC",
+    "CLAIM_ADMIT_TYPE_CD",
+    "AUTH_ADMIT_TYPE_DESC",
+    "CLAIM_STATUS_CD",
+    "CLAIM_STATUS_DESC",
+    "CONTRACT_FACILITY_OUTLIER_TERMS",
+    "FACILITY_OUTLIER_FIXED_LOSS_THRESHOLD",
+    "FACILITY_OUTLIER_PERCENT_RATE_ON_EXCESS_CHARGES",
+    "FACILITY_OUTLIER_MAXIMUM",
+    "FACILITY_OUTLIER_EXCLUSION_CD",
+    "FACILITY_OUTLIER_EXCLUSION_DESC",
+    "CONTRACT_FACILITY_STOP_LOSS_TERMS",
+    "FACILITY_STOP_LOSS_FIXED_LOSS_THRESHOLD",
+    "FACILITY_STOP_LOSS_PERCENT_RATE_ON_EXCESS_CHARGES",
+    "FACILITY_STOP_LOSS_DAILY_MAXIMUM",
+    "FACILITY_STOP_LOSS_EXCLUSION_DESC",
+    "CONTRACT_SEQUESTRATION_PERCENT_RATE",
+    "CONTRACT_SEQUESTRATION_START_DATE",
+    "CONTRACT_SEQUESTRATION_END_DATE",
+    "CONTRACT_DISCOUNT_PERCENT_RATE",
+    "CONTRACT_DISCOUNT_START_DATE",
+    "CONTRACT_DISCOUNT_END_DATE",
+    "CONTRACT_PREMIUM_PERCENT_RATE",
+    "CONTRACT_PREMIUM_START_DATE",
+    "CONTRACT_PREMIUM_END_DATE"
+]

fieldExtraction/src/investment/postprocess.py

@@ -1,4 +1,4 @@
-import src.constants.valid as valid
+from src.constants.investment_columns import COLUMN_ORDER
 import src.config as config
 from src import postprocessing_funcs
 from crosswalk.crosswalk_utils import CrosswalkBuilder, apply_crosswalk
@@ -28,12 +28,10 @@ def postprocess(df):
         # Derived termination date
         df['AARETE_DERIVED_TERMINATION_DATE'] = list(map(invoke_derived_term_date, df['CONTRACT_EFFECTIVE_DT'], df['CONTRACT_TERMINATION_DT']))
 
-        # # applying crosswalk for AARETE_DERIVED_CLAIM_TYPE_CD
-        # claim_type_mapping_path = os.path.join(MAPPINGS_DIR,"crosswalk_claim_type.json")
-        # claim_type_crosswalk = CrosswalkBuilder().from_json(str(claim_type_mapping_path))
-        # print(claim_type_crosswalk.mapping)
-        # df['AARETE_DERIVED_CLAIM_TYPE_CD'] = df['AARETE_DERIVED_CLAIM_TYPE_CD'].apply(lambda x: apply_crosswalk(str(x), claim_type_crosswalk.mapping) if pd.notna(x) else x) 
+        
 
+        # Standardize output column order - this should ALWAYS be the final postprocessing step
+        df = postprocessing_funcs.reorder_columns(df, COLUMN_ORDER)
 
         return df
     else:

fieldExtraction/src/postprocessing_funcs.py

@@ -1134,6 +1134,12 @@ def convert_to_us_date_format(date_str: str) -> str:  # postprocessing_funcs.py
 def remove_unnamed_columns(df):  # postprocessing_funcs.py
     return df[[col for col in df.columns if "Unnamed" not in col]]
 
+
+def reoder_columns(df: pd.DataFrame, column_order: list[str]):
+    # Return each valid column first, in order, then add invalid columns to the end
+    return df[[col for col in column_order if col in df.columns] + [col for col in df.columns if col not in column_order]]
+
+
 @cache # memoize repeated calls to this function
 def date_postprocessing(date: str) -> str:
     """Processes dates using `investment_prompts.date_fix_prompt`.  Following the call to the LLM, the answer is extracted from the response and returned.