From 88b2a3835f8306dee842378a94d98dbbdfaaede6 Mon Sep 17 00:00:00 2001
From: Umang Mistry <umistry@aarete.com>
Date: Tue, 5 Mar 2024 15:42:32 -0600
Subject: [PATCH] Updated pipelines yaml with EC2 deployment + OIDC connections

---
 airflow/Qa_Dag.py       | 13 +++++++++++++
 bitbucket-pipelines.yml | 30 +++++++++++++++++++++++++++---
 2 files changed, 40 insertions(+), 3 deletions(-)

diff --git a/airflow/Qa_Dag.py b/airflow/Qa_Dag.py
index bceed96..2896f1b 100644
--- a/airflow/Qa_Dag.py
+++ b/airflow/Qa_Dag.py
@@ -67,6 +67,19 @@ def getData():
             print(f"Successful S3 put_object response. Status - {status}")
         else:
             raise AirflowFailException(f"Unsuccessful S3 put_object response. Status - {status}")
+    
+    oldData = df
+    newData = df
+
+    with io.BytesIO() as output:
+        with pd.ExcelWriter(output, engine='xlsxwriter') as writer:
+            oldData.to_excel(writer, sheet_name="Old")
+            newData.to_excel(writer, sheet_name="new")
+            dat = oldData.compare(newData, align_axis=0, keep_shape=True)
+            dat.to_excel(writer, sheet_name="qc")
+        response = s3.put_object(
+                Bucket=bucket, Key=object_key+'QC_Report.xlsx' , Body=output.getvalue()
+        )
 
     print("Dataframe is written to S3 successfully.")
 
diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml
index 16e644c..e8132ab 100644
--- a/bitbucket-pipelines.yml
+++ b/bitbucket-pipelines.yml
@@ -21,10 +21,27 @@ definitions:
           - if [ "$BITBUCKET_BRANCH" == "UAT" ]; then export SNOWFLAKE_DATABASE=$UAT_SNOWFLAKE_DATABASE; export SNOWFLAKE_ROLE=$UAT_SNOWFLAKE_ROLE; fi
           - if [ "$BITBUCKET_BRANCH" == "PROD" ]; then export SNOWFLAKE_DATABASE=$PROD_SNOWFLAKE_DATABASE; export SNOWFLAKE_ROLE=$PROD_SNOWFLAKE_ROLE; fi
           - schemachange -a $SNOWFLAKE_ACCOUNT -u $SNOWFLAKE_USER -r $SNOWFLAKE_ROLE -w $SNOWFLAKE_WAREHOUSE -d $SNOWFLAKE_DATABASE -c $SNOWFLAKE_DATABASE.SCHEMACHANGE.CHANGE_HISTORY --create-change-history-table -v
-        trigger: automatic # uncomment after testing
+        trigger: automatic
         # trigger: manual
         caches:
           - pip
+    - step: &streamlit_deploy
+        name: "Deploy streamlit to EC2"
+        image: python:3.8
+        script:
+          -  python -m pip install --upgrade pip 
+          - apt-get update && apt-get install -y jq git
+          - pip install awscli
+          - export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO:role/$OIDC_ROLE
+          - export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token 
+          - echo $BITBUCKET_STEP_OIDC_TOKEN > $(pwd)/web-identity-token
+          - export STS_OUTPUT=$(aws sts assume-role-with-web-identity --role-arn $AWS_ROLE_ARN --role-session-name BitbucketPipeline --web-identity-token "$BITBUCKET_STEP_OIDC_TOKEN"  --duration-seconds 3600)
+          - export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId')
+          - export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey')
+          - export AWS_SESSION_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken')
+          - aws sts get-caller-identity
+          - aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $DEV_INSTANCE_ID --region $AWS_DEFAULT_REGION --parameters commands='["cd /home/ubuntu/streamlit","(if [ -d doczy.ai ]; then cd doczy.ai && git fetch && git pull; else git clone git@bitbucket.org:aarete/doczy.ai.git; fi)"]'
+        trigger: manual # uncomment after testing
 
 pipelines:
   branches:
@@ -41,7 +58,14 @@ pipelines:
             changesets:
               includePaths:
               - snowflake/**/*
-    uat:
+      - step:
+          <<: *streamlit_deploy
+          condition:
+            changesets:
+              includePaths:
+              - streamlit/**/*
+    # UAT and PROD pipelines will be updated with Streamlit steps once it is tested and ready
+    UAT:
       - step:
           <<: *terraform_plan_apply
           condition:
@@ -52,7 +76,7 @@ pipelines:
           condition:
             paths:
               - snowflake/**/*.sql
-    prod:
+    PROD:
       - step:
           <<: *terraform_plan_apply
           condition: