From 018b060c9bd1c544bb283e6759fd02e3748ea4c1 Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Mon, 1 Jul 2024 10:47:54 -0500 Subject: [PATCH] Update UAT tables for B fields --- .../R__1008_PIPELINE_OUTPUT_B_FIELDS.sql | 61 ++++++++++ ...1009_LOAD_B_FIELDS_PIPELINE_RAW_OUTPUT.sql | 108 ++++++++++++++++++ 2 files changed, 169 insertions(+) create mode 100644 snowflake/UAT/config_interface/R__1008_PIPELINE_OUTPUT_B_FIELDS.sql create mode 100644 snowflake/UAT/config_interface/R__1009_LOAD_B_FIELDS_PIPELINE_RAW_OUTPUT.sql diff --git a/snowflake/UAT/config_interface/R__1008_PIPELINE_OUTPUT_B_FIELDS.sql b/snowflake/UAT/config_interface/R__1008_PIPELINE_OUTPUT_B_FIELDS.sql new file mode 100644 index 0000000..d02885f --- /dev/null +++ b/snowflake/UAT/config_interface/R__1008_PIPELINE_OUTPUT_B_FIELDS.sql @@ -0,0 +1,61 @@ +CREATE TABLE IF NOT EXISTS STG.DOCZY_PIPELINE_RAW_OUTPUT_B ( + SERVICE TEXT, + REIMBURSEMENT_FLAT_FEE TEXT, + REIMBURSEMENT_RATE TEXT, + FULL_METHODOLOGY TEXT, + page_num TEXT, + contract_name TEXT, + LESSER_OF_LANGUAGE_IND TEXT, + GREATER_OF_LANGUAGE_IND TEXT, + REIMBURSEMENT_METHODOLOGY TEXT, + REIMBURSEMENT_FEE_SCHEDULE TEXT, + REIMBURSEMENT_FEE_SCHEDULE_VERSION TEXT, + REIMBURSEMENT_EXCEPTION_IND TEXT, + REIMBURSEMENT_DESCRIBE_EXCEPTION TEXT, + REIMBURSEMENT_ADMITTYPE_CODES TEXT, + REIMBURSEMENT_DIAG_CODES TEXT, + REIMBURSEMENT_GROUPER_CODES TEXT, + REIMBURSEMENT_GROUPER TEXT, + REIMBURSEMENT_PLACEOFSERVICE_CODES TEXT, + REIMBURSEMENT_PROC_CODES TEXT, + REIMBURSEMENT_REVENUE_CODES TEXT, + REIMBURSEMENT_STATUS_INDICATOR_CODES TEXT, + SERVICE_PG TEXT, + REIMBURSEMENT_FLAT_FEE_PG TEXT, + REIMBURSEMENT_RATE_PG TEXT, + FULL_METHODOLOGY_PG TEXT, + Filename_PG TEXT, + LESSER_OF_LANGUAGE_IND_PG TEXT, + GREATER_OF_LANGUAGE_IND_PG TEXT, + REIMBURSEMENT_METHODOLOGY_PG TEXT, + REIMBURSEMENT_FEE_SCHEDULE_PG TEXT, + REIMBURSEMENT_FEE_SCHEDULE_VERSION_PG TEXT, + REIMBURSEMENT_EXCEPTION_IND_PG TEXT, + REIMBURSEMENT_DESCRIBE_EXCEPTION_PG TEXT, + REIMBURSEMENT_ADMITTYPE_CODES_PG TEXT, + REIMBURSEMENT_DIAG_CODES_PG TEXT, + REIMBURSEMENT_GROUPER_CODES_PG TEXT, + REIMBURSEMENT_GROUPER_PG TEXT, + REIMBURSEMENT_PLACEOFSERVICE_CODES_PG TEXT, + REIMBURSEMENT_PROC_CODES_PG TEXT, + REIMBURSEMENT_REVENUE_CODES_PG TEXT, + REIMBURSEMENT_STATUS_INDICATOR_CODES_PG TEXT, + CONTRACT_LOB TEXT, + CONTRACT_LOB_PG TEXT, + CONTRACT_PROGRAM TEXT, + CONTRACT_PROGRAM_PG TEXT, + CONTRACT_NETWORK TEXT, + CONTRACT_NETWORK_PG TEXT, + PRODUCT TEXT, + PRODUCT_PG TEXT, + CONTRACT_MARKETPLACE_METAL_LEVEL TEXT, + CONTRACT_MARKETPLACE_METAL_LEVEL_PG TEXT, + LOB_PRICING_TERMS_EFFECTIVE_DATE TEXT, + LOB_PRICING_TERMS_EFFECTIVE_DATE_PG TEXT, + LOB_PRICING_TERMS_TERMINATION_DATE TEXT, + LOB_PRICING_TERMS_TERMINATION_DATE_PG TEXT, + Corrected_LOB TEXT, + Corrected_PROGRAM TEXT, + Corrected_NETWORK TEXT, + batch_id TEXT +); diff --git a/snowflake/UAT/config_interface/R__1009_LOAD_B_FIELDS_PIPELINE_RAW_OUTPUT.sql b/snowflake/UAT/config_interface/R__1009_LOAD_B_FIELDS_PIPELINE_RAW_OUTPUT.sql new file mode 100644 index 0000000..010e232 --- /dev/null +++ b/snowflake/UAT/config_interface/R__1009_LOAD_B_FIELDS_PIPELINE_RAW_OUTPUT.sql @@ -0,0 +1,108 @@ +CREATE OR REPLACE PROCEDURE STG.LOAD_DOCZY_PIPELINE_RAW_OUTPUT_B_FIELDS(file_name VARCHAR) +RETURNS STRING +LANGUAGE SQL +EXECUTE AS CALLER +AS +$$ +DECLARE + procedure_name VARCHAR; + unique_stage_name VARCHAR; +BEGIN + procedure_name := 'LOAD_DOCZY_PIPELINE_RAW_OUTPUT_LARGE'; + unique_stage_name := 'DOCZY_PIPELINE_RAW_OUTPUT_STAGE_' || REPLACE(UUID_STRING(), '-', '_'); + + CALL stg.log_audit(:procedure_name, 'Section 1', 99, 'START'); + + -- Create a unique stage with dynamic file name + EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.' || unique_stage_name || ' + STORAGE_INTEGRATION = UAT_BUCKET_INTEGRATION + URL = ''s3://doczyai-use2-u-infra-s3-raw-data-ingestion/doczy_pipeline_output/' || :file_name || ''' + FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');'; + + CALL stg.log_audit(:procedure_name, 'Section 1', 99, 'END'); + + CALL stg.log_audit(:procedure_name, 'Section 2', 99, 'START'); + + EXECUTE IMMEDIATE 'COPY INTO DOCZY_PIPELINE_RAW_OUTPUT_B FROM ( + SELECT + NULLIF(TRIM($1), '''') AS SERVICE, + NULLIF(TRIM($2), '''') AS REIMBURSEMENT_FLAT_FEE, + NULLIF(TRIM($3), '''') AS REIMBURSEMENT_RATE, + NULLIF(TRIM($4), '''') AS FULL_METHODOLOGY, + NULLIF(TRIM($5), '''') AS page_num, + NULLIF(TRIM($6), '''') AS contract_name, + NULLIF(TRIM($7), '''') AS LESSER_OF_LANGUAGE_IND, + NULLIF(TRIM($8), '''') AS GREATER_OF_LANGUAGE_IND, + NULLIF(TRIM($9), '''') AS REIMBURSEMENT_METHODOLOGY, + NULLIF(TRIM($10), '''') AS REIMBURSEMENT_FEE_SCHEDULE, + NULLIF(TRIM($11), '''') AS REIMBURSEMENT_FEE_SCHEDULE_VERSION, + NULLIF(TRIM($12), '''') AS REIMBURSEMENT_EXCEPTION_IND, + NULLIF(TRIM($13), '''') AS REIMBURSEMENT_DESCRIBE_EXCEPTION, + NULLIF(TRIM($14), '''') AS REIMBURSEMENT_ADMITTYPE_CODES, + NULLIF(TRIM($15), '''') AS REIMBURSEMENT_DIAG_CODES, + NULLIF(TRIM($16), '''') AS REIMBURSEMENT_GROUPER_CODES, + NULLIF(TRIM($17), '''') AS REIMBURSEMENT_GROUPER, + NULLIF(TRIM($18), '''') AS REIMBURSEMENT_PLACEOFSERVICE_CODES, + NULLIF(TRIM($19), '''') AS REIMBURSEMENT_PROC_CODES, + NULLIF(TRIM($20), '''') AS REIMBURSEMENT_REVENUE_CODES, + NULLIF(TRIM($21), '''') AS REIMBURSEMENT_STATUS_INDICATOR_CODES, + NULLIF(TRIM($22), '''') AS SERVICE_PG, + NULLIF(TRIM($23), '''') AS REIMBURSEMENT_FLAT_FEE_PG, + NULLIF(TRIM($24), '''') AS REIMBURSEMENT_RATE_PG, + NULLIF(TRIM($25), '''') AS FULL_METHODOLOGY_PG, + NULLIF(TRIM($26), '''') AS contract_name_PG, + NULLIF(TRIM($27), '''') AS LESSER_OF_LANGUAGE_IND_PG, + NULLIF(TRIM($28), '''') AS GREATER_OF_LANGUAGE_IND_PG, + NULLIF(TRIM($29), '''') AS REIMBURSEMENT_METHODOLOGY_PG, + NULLIF(TRIM($30), '''') AS REIMBURSEMENT_FEE_SCHEDULE_PG, + NULLIF(TRIM($31), '''') AS REIMBURSEMENT_FEE_SCHEDULE_VERSION_PG, + NULLIF(TRIM($32), '''') AS REIMBURSEMENT_EXCEPTION_IND_PG, + NULLIF(TRIM($33), '''') AS REIMBURSEMENT_DESCRIBE_EXCEPTION_PG, + NULLIF(TRIM($34), '''') AS REIMBURSEMENT_ADMITTYPE_CODES_PG, + NULLIF(TRIM($35), '''') AS REIMBURSEMENT_DIAG_CODES_PG, + NULLIF(TRIM($36), '''') AS REIMBURSEMENT_GROUPER_CODES_PG, + NULLIF(TRIM($37), '''') AS REIMBURSEMENT_GROUPER_PG, + NULLIF(TRIM($38), '''') AS REIMBURSEMENT_PLACEOFSERVICE_CODES_PG, + NULLIF(TRIM($39), '''') AS REIMBURSEMENT_PROC_CODES_PG, + NULLIF(TRIM($40), '''') AS REIMBURSEMENT_REVENUE_CODES_PG, + NULLIF(TRIM($41), '''') AS REIMBURSEMENT_STATUS_INDICATOR_CODES_PG, + NULLIF(TRIM($42), '''') AS CONTRACT_LOB, + NULLIF(TRIM($43), '''') AS CONTRACT_LOB_PG, + NULLIF(TRIM($44), '''') AS CONTRACT_PROGRAM, + NULLIF(TRIM($45), '''') AS CONTRACT_PROGRAM_PG, + NULLIF(TRIM($46), '''') AS CONTRACT_NETWORK, + NULLIF(TRIM($47), '''') AS CONTRACT_NETWORK_PG, + NULLIF(TRIM($48), '''') AS PRODUCT, + NULLIF(TRIM($49), '''') AS PRODUCT_PG, + NULLIF(TRIM($50), '''') AS CONTRACT_MARKETPLACE_METAL_LEVEL, + NULLIF(TRIM($51), '''') AS CONTRACT_MARKETPLACE_METAL_LEVEL_PG, + NULLIF(TRIM($52), '''') AS LOB_PRICING_TERMS_EFFECTIVE_DATE, + NULLIF(TRIM($53), '''') AS LOB_PRICING_TERMS_EFFECTIVE_DATE_PG, + NULLIF(TRIM($54), '''') AS LOB_PRICING_TERMS_TERMINATION_DATE, + NULLIF(TRIM($55), '''') AS LOB_PRICING_TERMS_TERMINATION_DATE_PG, + NULLIF(TRIM($56), '''') AS Corrected_LOB, + NULLIF(TRIM($57), '''') AS Corrected_PROGRAM, + NULLIF(TRIM($58), '''') AS Corrected_NETWORK, + NULLIF(TRIM($59), '''') AS batch_id + FROM @STG.' || unique_stage_name || ' + ) + FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'') + ON_ERROR = ABORT_STATEMENT;'; + + CALL stg.log_audit(:procedure_name, 'Section 2', 99, 'END'); + + CALL stg.log_audit(:procedure_name, 'Section 3', 99, 'START'); + + EXECUTE IMMEDIATE 'INSERT INTO STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) + SELECT STG.AUDIT_SID.NEXTVAL, ''DOCZY_PIPELINE_RAW_OUTPUT_B'', METADATA$FILENAME, CURRENT_TIMESTAMP(), MAX(METADATA$FILE_ROW_NUMBER) + FROM @STG.' || unique_stage_name || ' + GROUP BY 1, 2,3,4;'; + + -- Clean up the stage after use + EXECUTE IMMEDIATE 'DROP STAGE IF EXISTS STG.' || unique_stage_name; + + CALL stg.log_audit(:procedure_name, 'Section 3', 99, 'END'); + + RETURN 'Setup, Load, and Audit Complete'; +END; +$$;