jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/prod_sf_tables (pull request #182)

Browse Source 
Feature/prod sf tables
This commit is contained in:
Umang Mistry
2024-07-01 16:46:07 +00:00
parent 56e3c52805 018b060c9b
commit bf8297613c
3 changed files with 170 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
snowflake/UAT/config_interface/R__1008_PIPELINE_OUTPUT_B_FIELDS.sql
+61
View File
@@ -0,0 +1,61 @@
CREATE TABLE IF NOT EXISTS STG.DOCZY_PIPELINE_RAW_OUTPUT_B (
SERVICE TEXT,
REIMBURSEMENT_FLAT_FEE TEXT,
REIMBURSEMENT_RATE TEXT,
FULL_METHODOLOGY TEXT,
page_num TEXT,
contract_name TEXT,
LESSER_OF_LANGUAGE_IND TEXT,
GREATER_OF_LANGUAGE_IND TEXT,
REIMBURSEMENT_METHODOLOGY TEXT,
REIMBURSEMENT_FEE_SCHEDULE TEXT,
REIMBURSEMENT_FEE_SCHEDULE_VERSION TEXT,
REIMBURSEMENT_EXCEPTION_IND TEXT,
REIMBURSEMENT_DESCRIBE_EXCEPTION TEXT,
REIMBURSEMENT_ADMITTYPE_CODES TEXT,
REIMBURSEMENT_DIAG_CODES TEXT,
REIMBURSEMENT_GROUPER_CODES TEXT,
REIMBURSEMENT_GROUPER TEXT,
REIMBURSEMENT_PLACEOFSERVICE_CODES TEXT,
REIMBURSEMENT_PROC_CODES TEXT,
REIMBURSEMENT_REVENUE_CODES TEXT,
REIMBURSEMENT_STATUS_INDICATOR_CODES TEXT,
SERVICE_PG TEXT,
REIMBURSEMENT_FLAT_FEE_PG TEXT,
REIMBURSEMENT_RATE_PG TEXT,
FULL_METHODOLOGY_PG TEXT,
Filename_PG TEXT,
LESSER_OF_LANGUAGE_IND_PG TEXT,
GREATER_OF_LANGUAGE_IND_PG TEXT,
REIMBURSEMENT_METHODOLOGY_PG TEXT,
REIMBURSEMENT_FEE_SCHEDULE_PG TEXT,
REIMBURSEMENT_FEE_SCHEDULE_VERSION_PG TEXT,
REIMBURSEMENT_EXCEPTION_IND_PG TEXT,
REIMBURSEMENT_DESCRIBE_EXCEPTION_PG TEXT,
REIMBURSEMENT_ADMITTYPE_CODES_PG TEXT,
REIMBURSEMENT_DIAG_CODES_PG TEXT,
REIMBURSEMENT_GROUPER_CODES_PG TEXT,
REIMBURSEMENT_GROUPER_PG TEXT,
REIMBURSEMENT_PLACEOFSERVICE_CODES_PG TEXT,
REIMBURSEMENT_PROC_CODES_PG TEXT,
REIMBURSEMENT_REVENUE_CODES_PG TEXT,
REIMBURSEMENT_STATUS_INDICATOR_CODES_PG TEXT,
CONTRACT_LOB TEXT,
CONTRACT_LOB_PG TEXT,
CONTRACT_PROGRAM TEXT,
CONTRACT_PROGRAM_PG TEXT,
CONTRACT_NETWORK TEXT,
CONTRACT_NETWORK_PG TEXT,
PRODUCT TEXT,
PRODUCT_PG TEXT,
CONTRACT_MARKETPLACE_METAL_LEVEL TEXT,
CONTRACT_MARKETPLACE_METAL_LEVEL_PG TEXT,
LOB_PRICING_TERMS_EFFECTIVE_DATE TEXT,
LOB_PRICING_TERMS_EFFECTIVE_DATE_PG TEXT,
LOB_PRICING_TERMS_TERMINATION_DATE TEXT,
LOB_PRICING_TERMS_TERMINATION_DATE_PG TEXT,
Corrected_LOB TEXT,
Corrected_PROGRAM TEXT,
Corrected_NETWORK TEXT,
batch_id TEXT
);
snowflake/UAT/config_interface/R__1009_LOAD_B_FIELDS_PIPELINE_RAW_OUTPUT.sql
+108
View File
@@ -0,0 +1,108 @@
CREATE OR REPLACE PROCEDURE STG.LOAD_DOCZY_PIPELINE_RAW_OUTPUT_B_FIELDS(file_name VARCHAR)
RETURNS STRING
LANGUAGE SQL
EXECUTE AS CALLER
AS
$$
DECLARE
procedure_name VARCHAR;
unique_stage_name VARCHAR;
BEGIN
procedure_name := 'LOAD_DOCZY_PIPELINE_RAW_OUTPUT_LARGE';
unique_stage_name := 'DOCZY_PIPELINE_RAW_OUTPUT_STAGE_' || REPLACE(UUID_STRING(), '-', '_');
CALL stg.log_audit(:procedure_name, 'Section 1', 99, 'START');
-- Create a unique stage with dynamic file name
EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.' || unique_stage_name || '
STORAGE_INTEGRATION = UAT_BUCKET_INTEGRATION
URL = ''s3://doczyai-use2-u-infra-s3-raw-data-ingestion/doczy_pipeline_output/' || :file_name || '''
FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');';
CALL stg.log_audit(:procedure_name, 'Section 1', 99, 'END');
CALL stg.log_audit(:procedure_name, 'Section 2', 99, 'START');
EXECUTE IMMEDIATE 'COPY INTO DOCZY_PIPELINE_RAW_OUTPUT_B FROM (
SELECT
NULLIF(TRIM($1), '''') AS SERVICE,
NULLIF(TRIM($2), '''') AS REIMBURSEMENT_FLAT_FEE,
NULLIF(TRIM($3), '''') AS REIMBURSEMENT_RATE,
NULLIF(TRIM($4), '''') AS FULL_METHODOLOGY,
NULLIF(TRIM($5), '''') AS page_num,
NULLIF(TRIM($6), '''') AS contract_name,
NULLIF(TRIM($7), '''') AS LESSER_OF_LANGUAGE_IND,
NULLIF(TRIM($8), '''') AS GREATER_OF_LANGUAGE_IND,
NULLIF(TRIM($9), '''') AS REIMBURSEMENT_METHODOLOGY,
NULLIF(TRIM($10), '''') AS REIMBURSEMENT_FEE_SCHEDULE,
NULLIF(TRIM($11), '''') AS REIMBURSEMENT_FEE_SCHEDULE_VERSION,
NULLIF(TRIM($12), '''') AS REIMBURSEMENT_EXCEPTION_IND,
NULLIF(TRIM($13), '''') AS REIMBURSEMENT_DESCRIBE_EXCEPTION,
NULLIF(TRIM($14), '''') AS REIMBURSEMENT_ADMITTYPE_CODES,
NULLIF(TRIM($15), '''') AS REIMBURSEMENT_DIAG_CODES,
NULLIF(TRIM($16), '''') AS REIMBURSEMENT_GROUPER_CODES,
NULLIF(TRIM($17), '''') AS REIMBURSEMENT_GROUPER,
NULLIF(TRIM($18), '''') AS REIMBURSEMENT_PLACEOFSERVICE_CODES,
NULLIF(TRIM($19), '''') AS REIMBURSEMENT_PROC_CODES,
NULLIF(TRIM($20), '''') AS REIMBURSEMENT_REVENUE_CODES,
NULLIF(TRIM($21), '''') AS REIMBURSEMENT_STATUS_INDICATOR_CODES,
NULLIF(TRIM($22), '''') AS SERVICE_PG,
NULLIF(TRIM($23), '''') AS REIMBURSEMENT_FLAT_FEE_PG,
NULLIF(TRIM($24), '''') AS REIMBURSEMENT_RATE_PG,
NULLIF(TRIM($25), '''') AS FULL_METHODOLOGY_PG,
NULLIF(TRIM($26), '''') AS contract_name_PG,
NULLIF(TRIM($27), '''') AS LESSER_OF_LANGUAGE_IND_PG,
NULLIF(TRIM($28), '''') AS GREATER_OF_LANGUAGE_IND_PG,
NULLIF(TRIM($29), '''') AS REIMBURSEMENT_METHODOLOGY_PG,
NULLIF(TRIM($30), '''') AS REIMBURSEMENT_FEE_SCHEDULE_PG,
NULLIF(TRIM($31), '''') AS REIMBURSEMENT_FEE_SCHEDULE_VERSION_PG,
NULLIF(TRIM($32), '''') AS REIMBURSEMENT_EXCEPTION_IND_PG,
NULLIF(TRIM($33), '''') AS REIMBURSEMENT_DESCRIBE_EXCEPTION_PG,
NULLIF(TRIM($34), '''') AS REIMBURSEMENT_ADMITTYPE_CODES_PG,
NULLIF(TRIM($35), '''') AS REIMBURSEMENT_DIAG_CODES_PG,
NULLIF(TRIM($36), '''') AS REIMBURSEMENT_GROUPER_CODES_PG,
NULLIF(TRIM($37), '''') AS REIMBURSEMENT_GROUPER_PG,
NULLIF(TRIM($38), '''') AS REIMBURSEMENT_PLACEOFSERVICE_CODES_PG,
NULLIF(TRIM($39), '''') AS REIMBURSEMENT_PROC_CODES_PG,
NULLIF(TRIM($40), '''') AS REIMBURSEMENT_REVENUE_CODES_PG,
NULLIF(TRIM($41), '''') AS REIMBURSEMENT_STATUS_INDICATOR_CODES_PG,
NULLIF(TRIM($42), '''') AS CONTRACT_LOB,
NULLIF(TRIM($43), '''') AS CONTRACT_LOB_PG,
NULLIF(TRIM($44), '''') AS CONTRACT_PROGRAM,
NULLIF(TRIM($45), '''') AS CONTRACT_PROGRAM_PG,
NULLIF(TRIM($46), '''') AS CONTRACT_NETWORK,
NULLIF(TRIM($47), '''') AS CONTRACT_NETWORK_PG,
NULLIF(TRIM($48), '''') AS PRODUCT,
NULLIF(TRIM($49), '''') AS PRODUCT_PG,
NULLIF(TRIM($50), '''') AS CONTRACT_MARKETPLACE_METAL_LEVEL,
NULLIF(TRIM($51), '''') AS CONTRACT_MARKETPLACE_METAL_LEVEL_PG,
NULLIF(TRIM($52), '''') AS LOB_PRICING_TERMS_EFFECTIVE_DATE,
NULLIF(TRIM($53), '''') AS LOB_PRICING_TERMS_EFFECTIVE_DATE_PG,
NULLIF(TRIM($54), '''') AS LOB_PRICING_TERMS_TERMINATION_DATE,
NULLIF(TRIM($55), '''') AS LOB_PRICING_TERMS_TERMINATION_DATE_PG,
NULLIF(TRIM($56), '''') AS Corrected_LOB,
NULLIF(TRIM($57), '''') AS Corrected_PROGRAM,
NULLIF(TRIM($58), '''') AS Corrected_NETWORK,
NULLIF(TRIM($59), '''') AS batch_id
FROM @STG.' || unique_stage_name || '
)
FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'')
ON_ERROR = ABORT_STATEMENT;';
CALL stg.log_audit(:procedure_name, 'Section 2', 99, 'END');
CALL stg.log_audit(:procedure_name, 'Section 3', 99, 'START');
EXECUTE IMMEDIATE 'INSERT INTO STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT)
SELECT STG.AUDIT_SID.NEXTVAL, ''DOCZY_PIPELINE_RAW_OUTPUT_B'', METADATA$FILENAME, CURRENT_TIMESTAMP(), MAX(METADATA$FILE_ROW_NUMBER)
FROM @STG.' || unique_stage_name || '
GROUP BY 1, 2,3,4;';
-- Clean up the stage after use
EXECUTE IMMEDIATE 'DROP STAGE IF EXISTS STG.' || unique_stage_name;
CALL stg.log_audit(:procedure_name, 'Section 3', 99, 'END');
RETURN 'Setup, Load, and Audit Complete';
END;
$$;
textract-pipeline/terraform/main.tf
+1 -1
View File
@@ -42,7 +42,7 @@ locals {
prod = {
vpc_id = "vpc-00cfffdf5b3ff8aec"
secret_manager_name = "doczy_dev_db_creds"
secret_manager_name = "doczy-dev-db-svc-acc"
devops_s3_bucket_name = "doczyai-use2-p-infra-s3-devops-resources"
api_access_required_roles_arns = ["arn:aws:iam::211125720533:role/doczyai-use2-p-rol-streamlit-server"]
}