diff --git a/fieldExtraction/src/constants/investment_columns.py b/fieldExtraction/src/constants/investment_columns.py new file mode 100644 index 0000000..45e7d17 --- /dev/null +++ b/fieldExtraction/src/constants/investment_columns.py @@ -0,0 +1,103 @@ + +# This is the ONE AND ONLY source of investment column names and order. Do not reference anything else +COLUMN_ORDER = [ + "CONTRACT_SID", + "CONTRACT_FILE_NAME", + "CONTRACT_CLASSIFICATION", + "AARETE_DERIVED_CONTRACT_CLASSIFICATION", + "CONTRACT_TITLE", + "CONTRACT_AMENDMENT_NUM", + "CLIENT_NAME", + "PAYER_NAME", + "PROV_GROUP_TIN", + "PROV_GROUP_NPI", + "PROV_GROUP_NAME_FULL", + "PROV_TIN_OTHER", + "PROV_NPI_OTHER", + "PROV_REIMBURSEMENT_TIN", + "PROV_REIMBURSEMENT_NPI", + "PROV_FULL_NAME", + "CONTRACT_EFFECTIVE_DT", + "CONTRACT_TERMINATION_DT", + "AARETE_DERIVED_TERMINATION_DATE", + "CONTRACT_AUTO_RENEWAL_IND", + "CONTRACT_AUTO_RENEWAL_TERM", + "REIMBURSEMENT_EFFECTIVE_DT", + "REIMBURSEMENT_TERMINATION_DT", + "CONTRACT_SIGNATORY_COMPLETE_IND", + "CONTRACT_CLAIM_TYPE_CD", + "AARETE_DERIVED_CLAIM_TYPE_CD", + "EXHIBIT_NAME", + "EXHIBIT_PAGE", + "CONTRACT_PRODUCT", + "REIMBURSEMENT_PROV_NAME", + "AARETE_DERIVED_PRODUCT", + "CONTRACT_LINE_OF_BUSINESS", + "AARETE_DERIVED_LINE_OF_BUSINESS", + "CONTRACT_PROGRAM", + "AARETE_DERIVED_PROGRAM", + "CONTRACT_NETWORK", + "AARETE_DERIVED_NETWORK", + "CONTRACT_SERVICE_AREA", + "AARETE_DERIVED_SERVICE_AREA", + "CONTRACT_PROV_TYPE", + "AARETE_DERIVED_PROV_TYPE", + "REIMBURSEMENT_PROV_NAME", + "CONTRACT_PROV_SPECIALTY", + "AARETE_DERIVED_PROV_SPECIALTY", + "CONTRACT_BILL_TYPE_CD", + "AARETE_DERIVED_BILL_TYPE_CD", + "PROV_PATIENT_AGE_MIN", + "PROV_PATIENT_AGE_MAX", + "CONTRACT_SERVICE_CD_OR_DESC", + "CONTRACT_REIMBURSEMENT_METHOD", + "AARETE_DERIVED_REIMBURSEMENT_METHOD", + "CONTRACT_REIMBURSEMENT_FEE_RATE", + "CONTRACT_REIMBURSEMENT_PERCENT_RATE", + "CONTRACT_FEE_SCHEDULE_DESC", + "AARETE_DERIVED_FEE_SCHEDULE", + "CONTRACT_FEE_SCHEDULE_VERSION", + "AARETE_DERIVED_FEE_SCHEDULE_VERSION", + "CONTRACT_LESSER_OF_IND", + "CONTRACT_GREATER_OF_IND", + "CONTRACT_CARVEOUT_IND", + "CONTRACT_CARVEOUT_CD", + "CONTRACT_DEFAULT_IND", + "CPT4_PROC_CD" , + "CPT4_PROC_DESC", + "CPT4_PROC_MOD", + "CPT4_PROC_MOD_DESC", + "REVENUE_CD", + "REVENUE_CD_DESC", + "DIAG_CD", + "DIAG_CD_DESC", + "GROUPER_TYPE", + "FACILITY_GROUPER_CD", + "FACILITY_GROUPER_DESC", + "LINE_NDC_NUM" , + "LINE_NDC_DESC", + "CLAIM_ADMIT_TYPE_CD", + "AUTH_ADMIT_TYPE_DESC", + "CLAIM_STATUS_CD", + "CLAIM_STATUS_DESC", + "CONTRACT_FACILITY_OUTLIER_TERMS", + "FACILITY_OUTLIER_FIXED_LOSS_THRESHOLD", + "FACILITY_OUTLIER_PERCENT_RATE_ON_EXCESS_CHARGES", + "FACILITY_OUTLIER_MAXIMUM", + "FACILITY_OUTLIER_EXCLUSION_CD", + "FACILITY_OUTLIER_EXCLUSION_DESC", + "CONTRACT_FACILITY_STOP_LOSS_TERMS", + "FACILITY_STOP_LOSS_FIXED_LOSS_THRESHOLD", + "FACILITY_STOP_LOSS_PERCENT_RATE_ON_EXCESS_CHARGES", + "FACILITY_STOP_LOSS_DAILY_MAXIMUM", + "FACILITY_STOP_LOSS_EXCLUSION_DESC", + "CONTRACT_SEQUESTRATION_PERCENT_RATE", + "CONTRACT_SEQUESTRATION_START_DATE", + "CONTRACT_SEQUESTRATION_END_DATE", + "CONTRACT_DISCOUNT_PERCENT_RATE", + "CONTRACT_DISCOUNT_START_DATE", + "CONTRACT_DISCOUNT_END_DATE", + "CONTRACT_PREMIUM_PERCENT_RATE", + "CONTRACT_PREMIUM_START_DATE", + "CONTRACT_PREMIUM_END_DATE" +] \ No newline at end of file diff --git a/fieldExtraction/src/investment/postprocess.py b/fieldExtraction/src/investment/postprocess.py index b3f5b6d..2ecb0bc 100644 --- a/fieldExtraction/src/investment/postprocess.py +++ b/fieldExtraction/src/investment/postprocess.py @@ -1,4 +1,4 @@ -import src.constants.valid as valid +from src.constants.investment_columns import COLUMN_ORDER import src.config as config from src import postprocessing_funcs from crosswalk.crosswalk_utils import CrosswalkBuilder, apply_crosswalk @@ -28,12 +28,10 @@ def postprocess(df): # Derived termination date df['AARETE_DERIVED_TERMINATION_DATE'] = list(map(invoke_derived_term_date, df['CONTRACT_EFFECTIVE_DT'], df['CONTRACT_TERMINATION_DT'])) - # # applying crosswalk for AARETE_DERIVED_CLAIM_TYPE_CD - # claim_type_mapping_path = os.path.join(MAPPINGS_DIR,"crosswalk_claim_type.json") - # claim_type_crosswalk = CrosswalkBuilder().from_json(str(claim_type_mapping_path)) - # print(claim_type_crosswalk.mapping) - # df['AARETE_DERIVED_CLAIM_TYPE_CD'] = df['AARETE_DERIVED_CLAIM_TYPE_CD'].apply(lambda x: apply_crosswalk(str(x), claim_type_crosswalk.mapping) if pd.notna(x) else x) + + # Standardize output column order - this should ALWAYS be the final postprocessing step + df = postprocessing_funcs.reorder_columns(df, COLUMN_ORDER) return df else: diff --git a/fieldExtraction/src/postprocessing_funcs.py b/fieldExtraction/src/postprocessing_funcs.py index b57a564..da4f311 100644 --- a/fieldExtraction/src/postprocessing_funcs.py +++ b/fieldExtraction/src/postprocessing_funcs.py @@ -1134,6 +1134,12 @@ def convert_to_us_date_format(date_str: str) -> str: # postprocessing_funcs.py def remove_unnamed_columns(df): # postprocessing_funcs.py return df[[col for col in df.columns if "Unnamed" not in col]] + +def reoder_columns(df: pd.DataFrame, column_order: list[str]): + # Return each valid column first, in order, then add invalid columns to the end + return df[[col for col in column_order if col in df.columns] + [col for col in df.columns if col not in column_order]] + + @cache # memoize repeated calls to this function def date_postprocessing(date: str) -> str: """Processes dates using `investment_prompts.date_fix_prompt`. Following the call to the LLM, the answer is extracted from the response and returned.