diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 2c2427c..2b2d76b 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -1,4 +1,21 @@ definitions: + scripts: + - script: &aws-context-dev + export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO:role/$OIDC_ROLE; + export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token; + echo $BITBUCKET_STEP_OIDC_TOKEN > $(pwd)/web-identity-token; + export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId'); + export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey'); + export AWS_SECURITY_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken'); + export AWS_REGION='us-east-2'; + - script: &aws-context-uat + export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO_UAT:role/$OIDC_ROLE_UAT; + export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token;s + echo $BITBUCKET_STEP_OIDC_TOKEN > $(pwd)/web-identity-token; + export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId'); + export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey'); + export AWS_SECURITY_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken'); + export AWS_REGION='us-east-2'; steps: - step: &snowflake_deploy name: "Deploy Snowflake Changes" @@ -79,12 +96,6 @@ definitions: pipelines: branches: DEV: - # - step: - # <<: *terraform_plan_apply - # condition: - # changesets: - # includePaths: - # - terraform/environments/dev/**/* - step: <<: *snowflake_deploy condition: @@ -105,27 +116,130 @@ pipelines: includePaths: - airflow/dags/* - step: - <<: *terraform_plan_apply - name: Plan and Apply Textract Pipeline + name: "Plan & Apply Textract Pipeline" + image: hugree/bitbucket-aws-python38-tf154:latest + oidc: true + trigger : manual + script: + - *aws-context-dev + - cd textract-pipeline/terraform/ + - terraform validate + # Ensure the backend configuration for s3 obj key is corresponding to the stream name + - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=textract-pipeline/terraform.tfstate" + - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=dev" condition: changesets: includePaths: - - textract-pipeline/terraform/**/* - - textract-pipeline/terraform/* - + - textract-pipeline/terraform/* + - textract-pipeline/terraform/**/* + - step: + name: "Plan & Apply DevOps Pipeline" + image: hugree/bitbucket-aws-python38-tf154:latest + oidc: true + trigger : manual # Keeping it manual so that it doesnt conflict with existing resources + script: + - *aws-context-dev + - cd devops-pipeline/terraform-backend-resources/ + - terraform validate + # Ensure the backend configuration for s3 obj key is correct + - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=devops-pipeline/terraform.tfstate" + - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=dev" + condition: + changesets: + includePaths: + - devops-pipeline/terraform-backend-resources/* + - devops-pipeline/terraform-backend-resources/**/* + - step: + name: "Plan & Apply Streamlit Pipeline" + image: hugree/bitbucket-aws-python38-tf154:latest + oidc: true + trigger : manual # Keeping it manual so that it doesnt conflict with existing resources + script: + - *aws-context-dev + - cd streamlit-server/ + - terraform validate + # Ensure the backend configuration for s3 obj key is correct + - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=streamlit-pipeline/terraform.tfstate" + - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=dev" + condition: + changesets: + includePaths: + - streamlit-server/* + - streamlit-server/**/* + # UAT and PROD pipelines will be updated with Streamlit steps once it is tested and ready UAT: - - step: - <<: *terraform_plan_apply - condition: - paths: - - terraform/uat/**/* - step: <<: *snowflake_deploy condition: - paths: - - snowflake/**/*.sql + changesets: + includePaths: + - snowflake/**/* + - step: + <<: *streamlit_deploy + condition: + changesets: + includePaths: + - "streamlit/*" + - "streamlit/**/*" + - step: + <<: *airflow_dags_deploy + condition: + changesets: + includePaths: + - airflow/dags/* + - step: + name: "Plan & Apply Textract Pipeline" + image: hugree/bitbucket-aws-python38-tf154:latest + oidc: true + trigger : automatic + script: + - *aws-context-uat + - cd textract-pipeline/terraform/ + - terraform validate + # Ensure the backend configuration for s3 obj key is corresponding to the stream name + - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=textract-pipeline/terraform.tfstate" + - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" + condition: + changesets: + includePaths: + - textract-pipeline/terraform/* + - textract-pipeline/terraform/**/* + - step: + name: "Plan & Apply DevOps Pipeline" + image: hugree/bitbucket-aws-python38-tf154:latest + oidc: true + trigger : automatic # Keeping it manual so that it doesnt conflict with existing resources + script: + - *aws-context-uat + - cd devops-pipeline/terraform-backend-resources/ + - terraform validate + # Ensure the backend configuration for s3 obj key is correct + - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=textract-pipeline/terraform.tfstate" + - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" + condition: + changesets: + includePaths: + - devops-pipeline/terraform-backend-resources/* + - devops-pipeline/terraform-backend-resources/**/* + - step: + name: "Plan & Apply Streamlit Pipeline" + image: hugree/bitbucket-aws-python38-tf154:latest + oidc: true + trigger : automatic + script: + - *aws-context-uat + - cd streamlit-server/ + - terraform validate + # Ensure the backend configuration for s3 obj key is correct + - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=textract-pipeline/terraform.tfstate" + - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" + condition: + changesets: + includePaths: + - streamlit-server/* + - streamlit-server/**/* PROD: - step: <<: *terraform_plan_apply diff --git a/snowflake/scripts/config_interface/R__006_PIPELINE_OUTPUT.sql b/snowflake/scripts/config_interface/R__006_PIPELINE_OUTPUT.sql index 43de460..b08d1d8 100644 --- a/snowflake/scripts/config_interface/R__006_PIPELINE_OUTPUT.sql +++ b/snowflake/scripts/config_interface/R__006_PIPELINE_OUTPUT.sql @@ -1,13 +1,15 @@ -- CREATING THE CONFIG TABLE CREATE TABLE IF NOT EXISTS STG.DOCZY_PIPELINE_RAW_OUTPUT ( - CONTRACT_NAME VARCHAR(255), - SF_DB_COL_NAME VARCHAR(255), - ACTUAL_VALUE_STORED VARCHAR(255), - RAW_VALUE VARCHAR(255), - NEW_EXTRACTED_VALUE VARCHAR(255), - SNIPPET VARCHAR(255), - ORIGINAL_PAGE_NUMBER VARCHAR(255), - NEW_PAGE_NUMBER VARCHAR(255), - RESULT VARCHAR(255), - BATCH_ID VARCHAR(255) + DOCUMENT_ID VARCHAR(16777216) NOT NULL, + SF_DB_COL_NAME VARCHAR(255), + ACTUAL_VALUE_STORED VARCHAR(16777216), + RAW_VALUE VARCHAR(16777216), + NEW_EXTRACTED_VALUE VARCHAR(16777216), + SNIPPET VARCHAR(16777216), + ORIGINAL_PAGE_NUMBER VARCHAR(255), + NEW_PAGE_NUMBER VARCHAR(255), + RESULT VARCHAR(16777216), + BATCH_ID VARCHAR(255), + CREATED_TIME TIMESTAMP_NTZ(9), + MODIFIED_TIME TIMESTAMP_NTZ(9) ); \ No newline at end of file