Merged DEV into feature/streamlit_ui0_test

2024-06-13 16:30:26 +00:00
parent 23c6b1786d 4e99ece63c
commit 9d936adfe5
81 changed files with 1789 additions and 842 deletions
@@ -0,0 +1,83 @@
+
+
+from __future__ import annotations
+
+import os
+from datetime import datetime
+import logging
+
+from airflow import DAG
+from airflow.providers.snowflake.operators.snowflake import SnowflakeOperator
+from airflow.providers.snowflake.hooks.snowflake import SnowflakeHook
+from airflow.operators.python import PythonOperator
+from airflow.utils.trigger_rule import TriggerRule
+from airflow.operators.empty import EmptyOperator
+from airflow.models import Variable
+from airflow.exceptions import AirflowFailException
+
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+
+SNOWFLAKE_CONN_ID = "doczy_dev_snowflake"
+DAG_ID = "load_doczy_processed_outputs"
+DATABASE="DOCZY_DEV"
+
+TAGS=["dev","config_interface","dataload"]
+
+# Trigger rules
+ALL_SUCCESS = 'all_success'
+ALL_FAILED = 'all_failed'
+ALL_DONE = 'all_done'
+ONE_SUCCESS = 'one_success'
+ONE_FAILED = 'one_failed'
+
+# Passing empty params for now, this will be overridden by the payload from the trigger
+# These params can also be set from the Airflow UI while manually triggering the DAG
+# This will be replaced with the payload from the event after API connection is setup
+default_params = {"pipeline_processed_output": ""}
+
+def call_stored_proc(proc_name,file_type, params):
+    dwh_hook =  SnowflakeHook(snowflake_conn_id=SNOWFLAKE_CONN_ID)
+    with dwh_hook.get_conn() as conn:
+        # dwh_hook.set_autocommit(conn,autocommit=False)
+        cur = conn.cursor()
+
+        # Added new parameter file_type to determine the file name to be passed to the stored procedure
+        if file_type == 'pipeline_processed_output':
+            file_name = params['pipeline_processed_output']
+
+        cur.execute(f"CALL {DATABASE}.STG.{proc_name}('{file_name}');")
+        result = cur.fetchone()
+        if result[0] == 'Setup, Load, and Audit Complete':
+            logger.info('PROCEDURE EXECUTED SUCCESSFULLY')
+        else:
+            raise AirflowFailException("Check the DAG logs for more information. ERROR FROM SNOWFLAKE: ", result)
+        logger.info(f"QUERY EXECUTION RESULT: {str(result)}")
+
+dag = DAG(
+    DAG_ID,
+    start_date=datetime(2024, 1, 1),
+    default_args={"snowflake_conn_id": SNOWFLAKE_CONN_ID, "retries":0},
+    tags=TAGS,
+    catchup=False,
+    schedule=None,
+    params = default_params
+)
+    
+begin_job = EmptyOperator(task_id='Begin')
+
+
+load_pipeline_processed_output = PythonOperator(
+    task_id="load_pipeline_processed_output",
+    python_callable=call_stored_proc,
+    dag=dag,
+    op_kwargs={'proc_name':'LOAD_DOCZY_PIPELINE_PROCESSED_OUTPUT', 'file_type':'pipeline_processed_output'}        
+)
+
+
+end_job = EmptyOperator(task_id='End')
+
+begin_job >> load_pipeline_processed_output >> end_job
@@ -0,0 +1,94 @@
+import logging
+from airflow.exceptions import AirflowFailException
+from airflow.operators.empty import EmptyOperator
+from airflow.operators.python import PythonOperator
+from datetime import datetime
+from airflow import DAG
+from airflow.providers.snowflake.hooks.snowflake import SnowflakeHook
+# from openpyxl.workbook import Workbook
+import pandas as pd
+import boto3
+import io
+
+
+SNOWFLAKE_CONN_ID="doczy_uat_snowflake"
+TAGS=["QC","uat","etl","QC-Summery"]
+DAG_ID="qc_report_dag"
+
+bucket = "doczy-dev-infra-mwaa-resources"
+object_key = "outputs/"
+
+DATABASE="DOCZY_UAT"        
+SCHEMA = "STG"
+TABLE_NAME = "TRAINING_DATA_RAW"
+
+# Trigger rules
+ALL_SUCCESS = 'all_success'
+ALL_FAILED = 'all_failed'
+ALL_DONE = 'all_done'
+ONE_SUCCESS = 'one_success'
+ONE_FAILED = 'one_failed'
+
+
+
+
+args = {"owner": "Airflow", "start_date": datetime(2022, 1, 1), "retries":0 }
+dag = DAG(
+    dag_id=DAG_ID, default_args=args, schedule=None,
+    tags=TAGS
+)
+
+def getData():
+    # Setup connection to Snowflake
+    dwh_hook = SnowflakeHook(snowflake_conn_id=SNOWFLAKE_CONN_ID)
+    conn = dwh_hook.get_conn()  # Get the raw connection
+    
+    # Your query and the database details
+    qc_query = f"SELECT * FROM {DATABASE}.{SCHEMA}.{TABLE_NAME}"
+    
+    # Fetch data into a Pandas DataFrame
+    df = pd.read_sql(qc_query, conn)
+
+    # Initialize S3 client
+    s3 = boto3.client("s3")
+
+    # Create a buffer to hold the data
+    with io.StringIO() as csv_buffer:
+        df.to_csv(csv_buffer, index=False)
+        
+        # Save the data to S3
+        response = s3.put_object(
+            Bucket=bucket, Key=object_key+'snowflake_table_results.csv' , Body=csv_buffer.getvalue()
+        )
+
+        status = response.get("ResponseMetadata", {}).get("HTTPStatusCode")
+
+        if status == 200:
+            print(f"Successful S3 put_object response. Status - {status}")
+        else:
+            raise AirflowFailException(f"Unsuccessful S3 put_object response. Status - {status}")
+    
+    oldData = df
+    newData = df
+
+    with io.BytesIO() as output:
+        with pd.ExcelWriter(output, engine='xlsxwriter') as writer:
+            oldData.to_excel(writer, sheet_name="Old")
+            newData.to_excel(writer, sheet_name="new")
+            dat = oldData.compare(newData, align_axis=0, keep_shape=True)
+            dat.to_excel(writer, sheet_name="qc")
+        response = s3.put_object(
+                Bucket=bucket, Key=object_key+'QC_Report.xlsx' , Body=output.getvalue()
+        )
+
+    print("Dataframe is written to S3 successfully.")
+
+with dag:
+    begin_job = EmptyOperator(task_id='Begin')
+
+    get_data_from_snowflake = PythonOperator(task_id="get_data_from_snowflake", python_callable=getData)
+
+    end_job = EmptyOperator(task_id='End')
+
+
+begin_job >> get_data_from_snowflake >> end_job
@@ -0,0 +1,100 @@
+
+
+from __future__ import annotations
+
+import os
+from datetime import datetime
+import logging
+
+from airflow import DAG
+from airflow.providers.snowflake.operators.snowflake import SnowflakeOperator
+from airflow.providers.snowflake.hooks.snowflake import SnowflakeHook
+from airflow.operators.python import PythonOperator
+from airflow.utils.trigger_rule import TriggerRule
+from airflow.operators.empty import EmptyOperator
+from airflow.models import Variable
+from airflow.exceptions import AirflowFailException
+
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+
+SNOWFLAKE_CONN_ID = "doczy_uat_snowflake"
+DAG_ID = "load_request_and_contract_submissions"
+DATABASE="DOCZY_UAT"
+# bucket = "airflow-data-ingestion"
+
+TAGS=["uat","contract_config_interface","dataload"]
+
+# Trigger rules
+ALL_SUCCESS = 'all_success'
+ALL_FAILED = 'all_failed'
+ALL_DONE = 'all_done'
+ONE_SUCCESS = 'one_success'
+ONE_FAILED = 'one_failed'
+
+# Passing empty params for now, this will be overridden by the payload from the trigger
+# These params can also be set from the Airflow UI while manually triggering the DAG
+default_params = {"request_submission_file_name": "", "contract_config_file_name":""}
+# This will be replaced with the payload from the event after API connection is setup
+# training_results_file_name = "training_results_sample.csv"
+# attempt_logs_file_name = "attempt_logs_sample.csv"
+
+
+def call_stored_proc(proc_name,file_type, params):
+    dwh_hook =  SnowflakeHook(snowflake_conn_id=SNOWFLAKE_CONN_ID)
+    with dwh_hook.get_conn() as conn:
+        # dwh_hook.set_autocommit(conn,autocommit=False)
+        cur = conn.cursor()
+
+        # Added new parameter file_type to determine the file name to be passed to the stored procedure
+        # The bucket name is set by default to "doczy-dev-infra-raw-data-ingestion" and the files should be ALWAYS save under training_interface/ path for now
+        if file_type == 'request_submission':
+            file_name = params['request_submission_file_name']
+        elif file_type == 'contract_config':
+            file_name = params['contract_config_file_name']
+
+        cur.execute(f"CALL {DATABASE}.STG.{proc_name}('{file_name}');")
+        result = cur.fetchone()
+        if result[0] == 'Setup, Load, and Audit Complete':
+            logger.info('PROCEDURE EXECUTED SUCCESSFULLY')
+        else:
+            raise AirflowFailException("Check the DAG logs for more information. ERROR FROM SNOWFLAKE: ", result)
+        logger.info(f"QUERY EXECUTION RESULT: {str(result)}")
+
+dag = DAG(
+    DAG_ID,
+    start_date=datetime(2024, 1, 1),
+    default_args={"snowflake_conn_id": SNOWFLAKE_CONN_ID, "retries":0},
+    tags=TAGS,
+    catchup=False,
+    schedule=None,
+    params = default_params
+)
+    
+begin_job = EmptyOperator(task_id='Begin')
+
+
+load_request_submission = PythonOperator(
+    task_id="load_request_submissions",
+    python_callable=call_stored_proc,
+    dag=dag,
+    op_kwargs={'proc_name':'LOAD_REQUEST_SUBMISSION', 'file_type':'request_submission'}        
+)
+
+load_contract_config = PythonOperator(
+    task_id="load_contract_config",
+    python_callable=call_stored_proc,
+    dag=dag,
+    op_kwargs={'proc_name':'LOAD_CONTRACT_CONFIG', 'file_type':'contract_config'}        
+)
+
+
+
+
+end_job = EmptyOperator(task_id='End')
+
+
+begin_job >> load_request_submission >> load_contract_config >> end_job
@@ -0,0 +1,108 @@
+
+
+from __future__ import annotations
+
+import os
+from datetime import datetime
+import logging
+
+from airflow import DAG
+from airflow.providers.snowflake.operators.snowflake import SnowflakeOperator
+from airflow.providers.snowflake.hooks.snowflake import SnowflakeHook
+from airflow.operators.python import PythonOperator
+from airflow.utils.trigger_rule import TriggerRule
+from airflow.operators.empty import EmptyOperator
+from airflow.models import Variable
+from airflow.exceptions import AirflowFailException
+
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+
+SNOWFLAKE_CONN_ID = "doczy_uat_snowflake"
+DAG_ID = "load_raw_training_data"
+DATABASE="DOCZY_UAT"
+# bucket = "airflow-data-ingestion"
+
+TAGS=["uat","training_data","dataload"]
+
+# Trigger rules
+ALL_SUCCESS = 'all_success'
+ALL_FAILED = 'all_failed'
+ALL_DONE = 'all_done'
+ONE_SUCCESS = 'one_success'
+ONE_FAILED = 'one_failed'
+
+# Passing empty params for now, this will be overridden by the payload from the trigger
+# These params can also be set from the Airflow UI while manually triggering the DAG
+default_params = {"column_config_file_name": "", "raw_training_data_file_name":"", "business_config_file_name":""}
+# This will be replaced with the payload from the event after API connection is setup
+# training_results_file_name = "training_results_sample.csv"
+# attempt_logs_file_name = "attempt_logs_sample.csv"
+
+
+def call_stored_proc(proc_name,file_type, params):
+    dwh_hook =  SnowflakeHook(snowflake_conn_id=SNOWFLAKE_CONN_ID)
+    with dwh_hook.get_conn() as conn:
+        # dwh_hook.set_autocommit(conn,autocommit=False)
+        cur = conn.cursor()
+
+        # Added new parameter file_type to determine the file name to be passed to the stored procedure
+        # The bucket name is set by default to "doczy-dev-infra-raw-data-ingestion" and the files should be ALWAYS save under training_interface/ path for now
+        if file_type == 'column_config':
+            file_name = params['column_config_file_name']
+        elif file_type == 'raw_training_data':
+            file_name = params['raw_training_data_file_name']
+        elif file_type == 'business_config':
+            file_name = params['business_config_file_name']
+
+        cur.execute(f"CALL {DATABASE}.STG.{proc_name}('{file_name}');")
+        result = cur.fetchone()
+        if result[0] == 'Setup, Load, and Audit Complete':
+            logger.info('PROCEDURE EXECUTED SUCCESSFULLY')
+        else:
+            raise AirflowFailException("Check the DAG logs for more information. ERROR FROM SNOWFLAKE: ", result)
+        logger.info(f"QUERY EXECUTION RESULT: {str(result)}")
+
+dag = DAG(
+    DAG_ID,
+    start_date=datetime(2024, 1, 1),
+    default_args={"snowflake_conn_id": SNOWFLAKE_CONN_ID, "retries":0},
+    tags=TAGS,
+    catchup=False,
+    schedule=None,
+    params = default_params
+)
+    
+begin_job = EmptyOperator(task_id='Begin')
+
+
+load_training_results = PythonOperator(
+    task_id="load_column_config",
+    python_callable=call_stored_proc,
+    dag=dag,
+    op_kwargs={'proc_name':'LOAD_COLUMN_CONFIG', 'file_type':'column_config'}        
+)
+
+load_attempt_logs_sp = PythonOperator(
+    task_id="load_raw_training_data",
+    python_callable=call_stored_proc,
+    dag=dag,
+    op_kwargs={'proc_name':'LOAD_TRAINING_DATA_RAW', 'file_type':'raw_training_data'}        
+)
+
+load_business_config = PythonOperator(
+    task_id="load_business_config",
+    python_callable=call_stored_proc,
+    dag=dag,
+    op_kwargs={'proc_name':'LOAD_BUSINESS_CONFIG', 'file_type':'business_config'}        
+)
+
+
+
+end_job = EmptyOperator(task_id='End')
+
+
+begin_job >> load_training_results >> load_attempt_logs_sp >> load_business_config >> end_job
@@ -10,7 +10,7 @@ definitions:
        export AWS_REGION='us-east-2';
    - script: &aws-context-uat
        export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO_UAT:role/$OIDC_ROLE_UAT;
-        export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token;s
+        export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token;
        echo $BITBUCKET_STEP_OIDC_TOKEN > $(pwd)/web-identity-token;
        export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId');
        export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey');
@@ -50,38 +50,16 @@ definitions:
        # trigger: manual
        caches:
          - pip
-    - step: &terraform_plan_apply
-        name: "Plan and Apply Terraform"
-        image: python:3.8
-        script:
-          - python -m pip install --upgrade pip 
-          - apt-get update && apt-get install -y jq git
-          - pip install awscli
-          - pip install terraform
-          - export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO:role/$OIDC_ROLE
-          - export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token 
-          - echo $BITBUCKET_STEP_OIDC_TOKEN > $(pwd)/web-identity-token
-          - export STS_OUTPUT=$(aws sts assume-role-with-web-identity --role-arn $AWS_ROLE_ARN --role-session-name BitbucketPipeline --web-identity-token "$BITBUCKET_STEP_OIDC_TOKEN"  --duration-seconds 3600)
-          - export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId')
-          - export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey')
-          - export AWS_SESSION_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken')
-          - aws sts get-caller-identity
-          - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="region=us-east-2" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=devops-pipeline/terraform.tfstate" 
-          - terraform validate
-          - BRANCH_NAME_LOWER=$(echo $BITBUCKET_BRANCH | tr '[:upper:]' '[:lower:]')
-          - terraform apply -auto-approve -no-color -var 'access_key=$AWS_ACCESS_KEY_ID' -var 'secret_key=$AWS_SECRET_ACCESS_KEY' -var 'aws_region=us-east-2' -var 'environment=$BRANCH_NAME_LOWER'
-        trigger: automatic # uncomment after testing
-        # trigger: manual
    - step: &streamlit_deploy
        name: "Deploy streamlit to EC2"
        oidc: true
        image: python:3.8
        script:
-          - python -m pip install --upgrade pip 
+          - python -m pip install --upgrade pip
          - apt-get update && apt-get install -y jq git
          - pip install awscli
          - export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO:role/$OIDC_ROLE
-          - export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token 
+          - export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token
          - echo $BITBUCKET_STEP_OIDC_TOKEN > $(pwd)/web-identity-token
          - export STS_OUTPUT=$(aws sts assume-role-with-web-identity --role-arn $AWS_ROLE_ARN --role-session-name BitbucketPipeline --web-identity-token "$BITBUCKET_STEP_OIDC_TOKEN"  --duration-seconds 3600)
          - export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId')
@@ -95,23 +73,23 @@ definitions:
        image: python:3.8
        oidc: true
        script:
-          - python -m pip install --upgrade pip 
+          - python -m pip install --upgrade pip
          - apt-get update && apt-get install -y jq git
          - pip install awscli
          - export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO:role/$OIDC_ROLE
-          - export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token 
+          - export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token
          - echo $BITBUCKET_STEP_OIDC_TOKEN > $(pwd)/web-identity-token
          - export STS_OUTPUT=$(aws sts assume-role-with-web-identity --role-arn $AWS_ROLE_ARN --role-session-name BitbucketPipeline --web-identity-token "$BITBUCKET_STEP_OIDC_TOKEN"  --duration-seconds 3600)
          - export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId')
          - export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey')
          - export AWS_SESSION_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken')
          - aws sts get-caller-identity
-          - aws s3 sync ./airflow/dags s3://doczy-dev-infra-mwaa-resources/dags         
+          - aws s3 sync ./airflow/dags s3://doczy-dev-infra-mwaa-resources/dags
        trigger: automatic # uncomment after testing

 pipelines:
  branches:
-    DEV: 
+    DEV:
      - step:
          <<: *snowflake_deploy
          condition:
@@ -125,7 +103,7 @@ pipelines:
          name: "Deploy streamlit to EC2"
          oidc: true
          script:
-            - python -m pip install --upgrade pip 
+            - python -m pip install --upgrade pip
            - apt-get update && apt-get install -y jq git
            - pip install awscli
            - *aws-context-dev
@@ -142,94 +120,85 @@ pipelines:
              includePaths:
              - airflow/dags/*
      - step:
-          name: "Plan & Apply Textract Pipeline"
+          name: Apply textract-pipeline on DEV
          image: hugree/bitbucket-aws-python38-tf154:latest
          oidc: true
-          trigger : manual
          script:
            - *aws-context-dev
-            - cd textract-pipeline/terraform/
-            # Ensure the backend configuration for s3 obj key is corresponding to the stream name 
-            - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=terraform/textract-pipeline/terraform.tfstate" 
-            - terraform validate
-            - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=dev" 
+            - python terraform.py apply --environment=dev --module=textract-pipeline/terraform
          condition:
            changesets:
              includePaths:
+              - textract-pipeline/src/*
+              - textract-pipeline/src/**/*
              - textract-pipeline/terraform/*
-              - textract-pipeline/terraform/**/* 
+              - textract-pipeline/terraform/**/*
      - step:
-          name: "Plan & Apply DevOps Pipeline"
+          name: Apply devops-pipeline on DEV
          image: hugree/bitbucket-aws-python38-tf154:latest
          oidc: true
-          trigger : manual # Keeping it manual so that it doesnt conflict with existing resources
          script:
            - *aws-context-dev
-            - cd devops-pipeline/terraform-backend-resources/
-            # Ensure the backend configuration for s3 obj key is correct
-            - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=terraform/devops-pipeline/terraform.tfstate" 
-            - terraform validate
-            - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=dev" 
+            - python terraform.py apply --environment=dev --module=devops-pipeline/other-resources
          condition:
            changesets:
              includePaths:
-              - devops-pipeline/terraform-backend-resources/*
-              - devops-pipeline/terraform-backend-resources/**/* 
+              - devops-pipeline/other-resources/*
+              - devops-pipeline/other-resources/**/*
      - step:
-          name: "Plan & Apply Streamlit Pipeline"
+          name: Apply streamlit-server on DEV
          image: hugree/bitbucket-aws-python38-tf154:latest
          oidc: true
-          trigger : automatic 
          script:
            - *aws-context-dev
-            - cd streamlit-server/
-            # Ensure the backend configuration for s3 obj key is correct
-            - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=terraform/streamlit-pipeline/terraform.tfstate" -backend-config="region=us-east-2"
-            - terraform validate
-            - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=dev" -var "vpc_id=vpc-0f9d7c913f6522079"
+            - python terraform.py apply --environment=dev --module=streamlit-server
          condition:
            changesets:
              includePaths:
              - streamlit-server/*
-              - streamlit-server/**/* 
+              - streamlit-server/**/*
+

-                
    # UAT and PROD pipelines will be updated with Streamlit steps once it is tested and ready
    UAT:
      - step:
-          name: "Plan & Apply DevOps Pipeline"
+          name: Apply textract-pipeline on UAT
          image: hugree/bitbucket-aws-python38-tf154:latest
          oidc: true
-          trigger : automatic # Keeping it manual so that it doesnt conflict with existing resources
          script:
            - *aws-context-uat
-            - cd devops-pipeline/terraform-backend-resources/
-            - terraform validate
-            # Ensure the backend configuration for s3 obj key is correct
-            - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/devops-pipeline/terraform.tfstate" 
-            - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" 
+            - python terraform.py apply --environment=uat --module=textract-pipeline/terraform
          condition:
            changesets:
              includePaths:
-              - devops-pipeline/terraform-backend-resources/*
-              - devops-pipeline/terraform-backend-resources/**/* 
+                - textract-pipeline/src/*
+                - textract-pipeline/src/**/*
+                - textract-pipeline/terraform/*
+                - textract-pipeline/terraform/**/*
      - step:
-          name: "Plan & Apply Streamlit Pipeline"
+          name: Apply devops-pipeline on UAT
          image: hugree/bitbucket-aws-python38-tf154:latest
          oidc: true
-          trigger : manual
          script:
            - *aws-context-uat
-            - cd streamlit-server/
-            - terraform validate
-            # Ensure the backend configuration for s3 obj key is correct
-            - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/streamlit-server/terraform.tfstate" 
-            - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" 
+            - python terraform.py apply --environment=uat --module=devops-pipeline/other-resources
          condition:
            changesets:
              includePaths:
-              - streamlit-server/*
-              - streamlit-server/**/* 
+                - devops-pipeline/other-resources/*
+                - devops-pipeline/other-resources/**/*
+      - step:
+          name: Apply streamlit-server on UAT
+          image: hugree/bitbucket-aws-python38-tf154:latest
+          oidc: true
+          script:
+            - *aws-context-uat
+            - python terraform.py apply --environment=uat --module=streamlit-server
+          condition:
+            changesets:
+              includePaths:
+                - streamlit-server/*
+                - streamlit-server/**/*
      - step:
          <<: *snowflake_deploy
          condition:
@@ -242,12 +211,12 @@ pipelines:
          image: python:3.8
          name: "Deploy streamlit to EC2"
          oidc: true
-          trigger: manual # Keeping it manual as the server will be created first and then referenced in this 
+          trigger: automatic # Keeping it manual as the server will be created first and then referenced in this
          script:
-            - python -m pip install --upgrade pip 
+            - python -m pip install --upgrade pip
            - apt-get update && apt-get install -y jq git
            - pip install awscli
-            - *aws-context-dev
+            - *aws-context-uat
            - *ssm-send-command-uat
          condition:
            changesets:
@@ -261,33 +230,56 @@ pipelines:
            changesets:
              includePaths:
              - airflow/dags/*
-      - step:
-          name: "Plan & Apply Textract Pipeline"
-          image: hugree/bitbucket-aws-python38-tf154:latest
-          oidc: true
-          trigger : manual
-          script:
-            - *aws-context-uat
-            - cd textract-pipeline/terraform/
-            - terraform validate
-            # Ensure the backend configuration for s3 obj key is corresponding to the stream name 
-            - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/textract-pipeline/terraform.tfstate" 
-            - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" 
-          condition:
-            changesets:
-              includePaths:
-              - textract-pipeline/terraform/*
-              - textract-pipeline/terraform/**/* 
-

    PROD:
-      - step:
-          <<: *terraform_plan_apply
-          condition:
-            paths:
-              - terraform/prod//**/*
      - step:
          <<: *snowflake_deploy
          condition:
            paths:
              - snowflake/**/*.sql
+
+
+    feature/terraform-refactor:
+      - parallel:
+          - step:
+              name: Plan devops-pipeline on DEV
+              image: hugree/bitbucket-aws-python38-tf154:latest
+              oidc: true
+              script:
+                - *aws-context-dev
+                - python terraform.py plan --environment=dev --module=devops-pipeline/other-resources
+          - step:
+              name: Plan streamlit-server on DEV
+              image: hugree/bitbucket-aws-python38-tf154:latest
+              oidc: true
+              script:
+                - *aws-context-dev
+                - python terraform.py plan --environment=dev --module=streamlit-server
+          - step:
+              name: Plan textract-pipeline on DEV
+              image: hugree/bitbucket-aws-python38-tf154:latest
+              oidc: true
+              script:
+                - *aws-context-dev
+                - python terraform.py plan --environment=dev --module=textract-pipeline/terraform
+          - step:
+              name: Plan devops-pipeline on UAT
+              image: hugree/bitbucket-aws-python38-tf154:latest
+              oidc: true
+              script:
+                - *aws-context-uat
+                - python terraform.py plan --environment=uat --module=devops-pipeline/other-resources
+          - step:
+              name: Plan streamlit-server on UAT
+              image: hugree/bitbucket-aws-python38-tf154:latest
+              oidc: true
+              script:
+                - *aws-context-uat
+                - python terraform.py plan --environment=uat --module=streamlit-server
+          - step:
+              name: Plan textract-pipeline on UAT
+              image: hugree/bitbucket-aws-python38-tf154:latest
+              oidc: true
+              script:
+                - *aws-context-uat
+                - python terraform.py plan --environment=uat --module=textract-pipeline/terraform
@@ -6,22 +6,12 @@ terraform {
    }
  }
  backend "s3" {
-    bucket         = "doczyai-use2-d-infra-s3-terraform-state"  # Parameterize using -backend-config flag with "terraform init"
-    key            = "terraform/devops-pipeline/devops.tfstate" # Parameterize
-    region         = "us-east-2"                                # Parameterize 
-    # profile        = "doczyai"                                  # Parameterize
-    dynamodb_table = "doczyai-use2-d-infra-dyd-terraform-lock"  # Parameterize
-    encrypt        = true    
+    encrypt        = true
  }
 }

 provider "aws" {

-  # access_key = var.access_key
-  # secret_key = var.secret_key
-
-  profile = var.aws_profile
-
  default_tags {
    tags = {
      Terraform = "true"
@@ -159,7 +149,7 @@ resource "aws_iam_role" "snowflake_integration_role" {
        }
        Condition = {
          StringEquals = {
-              "sts:ExternalId" = "OQ11564_SFCRole=2_lNKzpyFn/e/xQgvGGpusVbEZA0A="
+              "sts:ExternalId" = var.storage_integration_external_id
          }
        }
      },
@@ -213,6 +203,7 @@ resource "aws_iam_role_policy_attachment" "snowflake_integration_policy_attachme

 # IAM role - cross-account-role 
 resource "aws_iam_role" "cross_account_role" {
+  count = var.environment != "dev"? 1 : 0
  name = local.cross_account_role_name

  assume_role_policy = jsonencode({
@@ -223,7 +214,7 @@ resource "aws_iam_role" "cross_account_role" {
        Effect = "Allow"
        Sid    = ""
        Principal = {
-          AWS = "arn:aws:iam::873115228912:root"
+          AWS = "arn:aws:iam::${var.cross_account_target_account_id}:root"
        }
      },
    ]
@@ -250,6 +241,7 @@ data "aws_iam_policy_document" "cross_account_policy_document" {

 # IAM Policy for cross-account-role
 resource "aws_iam_policy" "cross_account_policy" {
+  count = var.environment != "dev"? 1 : 0
  name        = local.cross_account_policy_name
  description = "This role is used for other AWS account to assume inorder to drop files to our data ingestion bucket. At the time of creation, this is being used by CODE DEV to drop all clients list."
  policy      = data.aws_iam_policy_document.cross_account_policy_document.json
@@ -257,8 +249,9 @@ resource "aws_iam_policy" "cross_account_policy" {

 # Attach IAM Policy to cross-account-role
 resource "aws_iam_role_policy_attachment" "cross_account_policy_attachment" {
-  role       = aws_iam_role.cross_account_role.name
-  policy_arn = aws_iam_policy.cross_account_policy.arn
+  count = var.environment != "dev"? 1 : 0
+  role       = aws_iam_role.cross_account_role[count.index].name
+  policy_arn = aws_iam_policy.cross_account_policy[count.index].arn
 }

 # IAM role - mwaa-exec-role
@@ -293,14 +286,14 @@ data "aws_iam_policy_document" "mwaa_exec_policy_document" {
    effect    = "Allow"
    actions   = ["airflow:CreateCliToken"]
    resources = [
-      "arn:aws:airflow:us-east-2:660131068782:environment/doczy-dev-infra-mwaa"
+      "arn:aws:airflow:us-east-2:${var.aws_account_id}:environment/doczy-${var.environment}-infra-mwaa"
    ]
  }
  statement {
    effect    = "Allow"
    actions   = ["airflow:PublishMetrics"]
    resources = [
-      "arn:aws:airflow:us-east-2:660131068782:environment/doczy-dev-infra-mwaa"
+      "arn:aws:airflow:us-east-2:${var.aws_account_id}:environment/doczy-${var.environment}-infra-mwaa"
    ]
  }
  statement {
@@ -337,7 +330,7 @@ data "aws_iam_policy_document" "mwaa_exec_policy_document" {
      "logs:GetQueryResults"
    ]
    resources = [
-      "arn:aws:logs:us-east-2:660131068782:log-group:airflow-doczy-dev-infra-mwaa-*"
+      "arn:aws:logs:us-east-2:${var.aws_account_id}:log-group:airflow-doczy-${var.environment}-infra-mwaa-*"
    ]
  }

@@ -17,7 +17,7 @@ output "snowflake_integration_role_arn" {
 }

 output "cross_account_role_arn" {
-  value = aws_iam_role.cross_account_role.id
+  value = aws_iam_role.cross_account_role[*].id
 }

 output "mwaa_exec_role_arn" {
@@ -1,19 +1,3 @@
-# required
-variable "aws_profile" {
-  type    = string
-  # default = "doczyai"
-}
-
-# # required
-# variable "secret_key" {
-#   type    = string
-# }
-
-# # required
-# variable "access_key" {
-#   type    = string
-# }
-
 # required
 variable "aws_region" {
  type    = string
@@ -66,7 +50,19 @@ variable "mwaa_resources_s3_bucket" {
  })
  default = {
    name = "mwaa-resources"
-
  }
 }

+variable "aws_account_id" {
+  type = string
+}
+
+variable "storage_integration_external_id" {
+  type = string
+}
+
+
+variable "cross_account_target_account_id" {
+  type = string
+}
+
@@ -0,0 +1,3 @@
+aws_account_id                  = 660131068782
+storage_integration_external_id = "OQ11564_SFCRole=2_lNKzpyFn/e/xQgvGGpusVbEZA0A="
+cross_account_target_account_id = ""
@@ -0,0 +1,3 @@
+aws_account_id                  = 975049960860
+storage_integration_external_id = "OQ11564_SFCRole=2_L4NUsKqd3NIBmrATa1djc/T3ECQ="
+cross_account_target_account_id = 660131068782
@@ -8,13 +8,7 @@ terraform {
 }

 provider "aws" {
-  profile = var.aws_profile
-  region = var.aws_region
-  # default_tags {
-  #   tags = {
-  #     Terraform = "true"
-  #   }
-  # }
+  region     = var.aws_region
 }

 module "aws_terraform_remote_backend" {
@@ -10,6 +10,20 @@ variable "environment" {
 variable "client_name" {
  type = string
 }
+# required
+variable "secret_key" {
+  type    = string
+}
+
+# required
+variable "access_key" {
+  type    = string
+}
+
+variable "token" {
+  type    = string
+}
+
 # dynamoDB table
 variable "terraform_dynamodb_table" {
  type = object({
@@ -9,4 +9,4 @@ output "terraform_s3_bucket_arn" {
 }
 output "terraform_s3_bucket_name" {
  value = module.aws_terraform_remote_backend.terraform_s3_bucket_name
-}
+}
@@ -1,9 +1,4 @@
 # required
-variable "aws_profile" {
-  type    = string
-  default = "doczy_uat"
-}
-# required
 variable "aws_region" {
  type    = string
  default = "us-east-2"
@@ -15,11 +10,25 @@ variable "project_name" {
 }
 # required
 variable "environment" {
-  type    = string
-  default = "uat"
+  type = string
 }
 # required
 variable "client_name" {
  type    = string
  default = "infra"
-}
+}
+
+#required
+variable "secret_key" {
+  type = string
+}
+
+#required
+variable "access_key" {
+  type = string
+}
+
+#required
+variable "token" {
+  type = string
+}
@@ -51,7 +51,7 @@ BEGIN
    batch_id := (SELECT NULLIF(TRIM($2),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1);
    request_datetime := (SELECT NULLIF(TRIM($13),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1);

-    UPDATE TABLE STG.CONTRACT_CONFIG 
+    UPDATE STG.CONTRACT_CONFIG 
    SET LATEST_FLAG = FALSE 
    WHERE BATCH_ID = :batch_id AND CONTRACT_NAME = :contract_name AND REQUEST_DATETIME < :request_datetime AND LATEST_FLAG = TRUE;  
     
@@ -32,7 +32,7 @@ BEGIN
        NULLIF(TRIM($6),'') AS SNIPPET,
        NULLIF(TRIM($7),'') AS ORIGINAL_PAGE_NUMBER,
        NULLIF(TRIM($8),'') AS NEW_PAGE_NUMBER,
-        NULLIF(TRIM($9),'') AS RESULT
+        NULLIF(TRIM($9),'') AS RESULT, 
        NULLIF(TRIM($10),'') AS BATCH_ID        

        FROM @STG.DOCZY_PIPELINE_RAW_OUTPUT_STAGE
@@ -0,0 +1,12 @@
+-- CREATING THE CONFIG TABLE
+CREATE TABLE IF NOT EXISTS STG.DOCZY_PIPELINE_PROCESSED_OUTPUT (
+    CONTRACT_NAME VARCHAR(16777216) NOT NULL,
+	FIELD_NAME VARCHAR(255),
+	RAW_VALUE VARCHAR(16777216),
+	NEW_EXTRACTED_VALUE VARCHAR(16777216),
+    CONFIDENCE_LEVEL VARCHAR(255),
+	SNIPPET VARCHAR(16777216),
+	NEW_PAGE_NUMBER VARCHAR(255),
+	BATCH_ID VARCHAR(255),
+	CREATED_TIME TIMESTAMP_NTZ(9)
+);
@@ -0,0 +1,73 @@
+-- columns to be ingested 
+-- CONTRACT_NAME VARCHAR(16777216) NOT NULL,
+-- FIELD_NAME VARCHAR(255),
+-- RAW_VALUE VARCHAR(16777216),
+-- NEW_EXTRACTED_VALUE VARCHAR(16777216),
+-- CONFIDENCE_LEVEL VARCHAR(255),
+-- SNIPPET VARCHAR(16777216),
+-- NEW_PAGE_NUMBER VARCHAR(255),
+-- BATCH_ID VARCHAR(255),
+-- CREATED_TIME TIMESTAMP_NTZ(9)
+
+CREATE OR REPLACE PROCEDURE STG.LOAD_DOCZY_PIPELINE_PROCESSED_OUTPUT(file_name VARCHAR)
+RETURNS STRING
+LANGUAGE SQL
+EXECUTE AS CALLER
+AS
+$$
+DECLARE
+    procedure_name varchar;
+BEGIN
+    
+    procedure_name := 'LOAD_DOCZY_PIPELINE_PROCESSED_OUTPUT';
+
+    call stg.log_audit(:procedure_name, 'Section 1', 99, 'START');
+        
+    -- Create or replace stage with dynamic file name
+    EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.DOCZY_PIPELINE_PROCESSED_OUTPUT_STAGE
+    STORAGE_INTEGRATION = dev_bucket_integration
+    URL = ''s3://doczy-dev-infra-raw-data-ingestion/doczy_pipeline_output/' || :file_name || '''
+    FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');';
+
+    call stg.log_audit(:procedure_name, 'Section 1', 99, 'END');
+
+    call stg.log_audit(:procedure_name, 'Section 2', 99, 'START');
+    
+    COPY INTO STG.DOCZY_PIPELINE_RAW_OUTPUT FROM (
+        SELECT 
+        NULLIF(TRIM($1),'') AS CONTRACT_NAME,
+        NULLIF(TRIM($2),'') AS FIELD_NAME,
+        NULLIF(TRIM($3),'') AS RAW_VALUE,
+        NULLIF(TRIM($4),'') AS NEW_EXTRACTED_VALUE,
+        NULLIF(TRIM($5),'') AS CONFIDENCE_LEVEL,
+        NULLIF(TRIM($6),'') AS SNIPPET,
+        NULLIF(TRIM($7),'') AS NEW_PAGE_NUMBER,
+        NULLIF(TRIM($8),'') AS BATCH_ID,
+        CURRENT_TIMESTAMP() AS CREATED_TIME       
+
+        FROM @STG.DOCZY_PIPELINE_PROCESSED_OUTPUT_STAGE
+    )
+    FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER')
+    ON_ERROR = ABORT_STATEMENT;
+    
+    call stg.log_audit(:procedure_name, 'Section 2', 99, 'END');
+ 
+    call stg.log_audit(:procedure_name, 'Section 3', 99, 'START');
+
+    INSERT INTO STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT)
+    SELECT STG.AUDIT_SID.NEXTVAL, 'STG.DOCZY_PIPELINE_PROCESSED_OUTPUT',* 
+    FROM 
+    (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER)  from @STG.DOCZY_PIPELINE_PROCESSED_OUTPUT_STAGE group by 1,2);
+
+    
+
+    RETURN 'Setup, Load, and Audit Complete';
+    
+    call stg.log_audit(:procedure_name, 'Section 3', 99, 'END');
+
+
+
+   
+END;
+$$;
+
@@ -8,4 +8,4 @@ create or replace TABLE STG.LOG_AUDIT (
 	SUB_SECTION_NAME VARCHAR(255) COLLATE 'en-ci',
 	AUDIT_SID NUMBER(38,0),
 	TOTAL_MIN NUMBER(38,0)
-);
+);
@@ -50,8 +50,8 @@ BEGIN
    contract_name := (SELECT NULLIF(TRIM($1),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1);
    batch_id := (SELECT NULLIF(TRIM($2),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1);
    request_datetime := (SELECT NULLIF(TRIM($13),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1);
-
-    UPDATE TABLE STG.CONTRACT_CONFIG 
+    
+    UPDATE STG.CONTRACT_CONFIG 
    SET LATEST_FLAG = FALSE 
    WHERE BATCH_ID = :batch_id AND CONTRACT_NAME = :contract_name AND REQUEST_DATETIME < :request_datetime AND LATEST_FLAG = TRUE;  
     
@@ -115,7 +115,8 @@ resource "aws_lb_listener" "https_8500" {
  protocol          = "HTTPS"
  ssl_policy        = "ELBSecurityPolicy-2016-08"
  # Using variable for ARN 
-  certificate_arn   = var.acm_arn_dev
+  certificate_arn   = lookup(var.acm_arns, var.environment)
+

  default_action {
    type             = "forward"
@@ -128,7 +129,7 @@ resource "aws_lb_listener" "https_8501" {
  port              = "8501"
  protocol          = "HTTPS"
  ssl_policy        = "ELBSecurityPolicy-2016-08"
-  certificate_arn   = var.acm_arn_dev
+  certificate_arn   = lookup(var.acm_arns, var.environment)

  default_action {
    type             = "forward"
@@ -141,7 +142,7 @@ resource "aws_lb_listener" "https_8502" {
  port              = "8502"
  protocol          = "HTTPS"
  ssl_policy        = "ELBSecurityPolicy-2016-08"
-  certificate_arn   = var.acm_arn_dev
+  certificate_arn   = lookup(var.acm_arns, var.environment)

  default_action {
    type             = "forward"
@@ -154,7 +155,7 @@ resource "aws_lb_listener" "https_8503" {
  port              = "8503"
  protocol          = "HTTPS"
  ssl_policy        = "ELBSecurityPolicy-2016-08"
-  certificate_arn   = var.acm_arn_dev
+  certificate_arn   = lookup(var.acm_arns, var.environment)

  default_action {
    type             = "forward"
@@ -71,7 +71,7 @@ data "aws_iam_policy_document" "combined_policy_document" {
    ]
    resources = ["*"]
  }
-  
+
  statement {
    sid       = "APIGatewayInvokeFullAccess"
    effect    = "Allow"
@@ -131,3 +131,6 @@ resource "aws_iam_role_policy_attachment" "combined_policy_attachment" {
  policy_arn = aws_iam_policy.combined_policy.arn
 }

+output "ec2_role_name" {
+  value = local.ec2_role_name
+}
@@ -1,5 +1,5 @@
 provider "aws" {
-  region = "us-east-2" 
+  region = "us-east-2"
 }

 terraform {
@@ -11,11 +11,6 @@ terraform {
  }

  backend "s3" {
-    # bucket         = "doczyai-use2-d-infra-s3-terraform-state"                       # Parameterize using -backend-config flag with "terraform init"
-    # key            = "terraform/streamlit-server/terraform.tfstate" # Parameterize
-    # region         = "us-east-2"                               # Parameterize 
-    # profile        = "temp_cred"                                 # Parameterize
-    # dynamodb_table = "doczyai-use2-d-infra-dyd-terraform-lock"                          # Parameterize
    encrypt        = true
  }
 }
@@ -43,7 +38,7 @@ locals {

  # global prefix
  global_prefix = "${var.project_name}-${local.region_short_name}-${local.environment_short_name}"
-  
+
  # Resource prefix
  iam_policy_prefix = "${local.global_prefix}-pol"

@@ -65,14 +60,14 @@ data "aws_subnets" "subnets" {
    values = [var.vpc_id]
  }
 }
- 
+
 # vpc security groups
 data "aws_security_groups" "security_groups" {
  filter {
    name   = "group-name"
    values = ["*default*"]
  }
- 
+
  filter {
    name   = "vpc-id"
    values = [var.vpc_id]
@@ -86,27 +81,27 @@ resource "aws_iam_instance_profile" "ec2_instance_profile" {
 }

 resource "aws_instance" "streamlit_server" {
-  ami                    = var.ubuntu_ami
+  ami                    = lookup(var.ubuntu_ami, var.environment)
  instance_type          = var.ec2_instance_type
  subnet_id              = data.aws_subnets.subnets.ids[0]
-  vpc_security_group_ids = data.aws_security_groups.security_groups.ids 
-  iam_instance_profile = aws_iam_instance_profile.ec2_instance_profile.name  
+  vpc_security_group_ids = data.aws_security_groups.security_groups.ids
+  iam_instance_profile = aws_iam_instance_profile.ec2_instance_profile.name
  associate_public_ip_address = false
-  key_name = "tf-test-key" # Created using TF and uploaded to S3. Prerequisite
+  key_name = var.key_name

  root_block_device {
    volume_size = 30 # Variable
    encrypted = true # Mandatory
  }
-#    Setup SSH authentication for bitbucket access to pull code 
-#    Setup SSL certificate for HTTPS access for SSO to work 
+#    Setup SSH authentication for bitbucket access to pull code
+#    Setup SSL certificate for HTTPS access for SSO to work
  user_data = <<-EOF
                #!/bin/bash
                echo '${file("${path.module}/requirements.txt")}' > /tmp/requirements.txt
                apt-get update
                apt-get install -y python3 python3-pip
                sudo apt-get install libffi-dev
-                python3 -m pip3 install -r /tmp/requirements.txt
+                python3 -m pip install -r /tmp/requirements.txt
                sudo apt install openssh-client
                sudo apt install net-tools

@@ -202,3 +197,4 @@ resource "aws_instance" "streamlit_server" {
 }


+
@@ -12,7 +12,6 @@ variable "project_name" {
 # required
 variable "environment" {
  type    = string
-  default = "dev"
 }


@@ -31,10 +30,18 @@ variable "vpc_id" {
  type = string
 }

+# variable "ubuntu_ami" {
+#     type = string
+#     default = "ami-0de7e97fedfbc6ef6" # Switch to this ami-0b8b44ec9a8f90422
+
+# }
+
 variable "ubuntu_ami" {
-    type = string
-    default = "ami-0de7e97fedfbc6ef6" # Switch to this ami-0b8b44ec9a8f90422
-  
+  type = map(string)
+  default = {
+    dev  = "ami-0de7e97fedfbc6ef6"
+    uat  = "ami-0b8b44ec9a8f90422"
+  }
 }

 variable "ec2_instance_type" {
@@ -42,12 +49,6 @@ variable "ec2_instance_type" {
    default = "t2.large" 
 }

-# EC2 iam role name
-variable "ec2_iam_role_name" {
-  type    = string
-  default = "ec2-iam-role"
-}
-
 # Cerate an array of ARNS
 variable "SecretsNames"{
  type = list(string)
@@ -59,16 +60,15 @@ variable "acm_arn_dev" {
  default = "arn:aws:acm:us-east-2:660131068782:certificate/947e04b3-be22-43bf-bea2-a0ad3b221d70"
 }

-variable "secret_key" {
-  type    = string
+variable "acm_arns" {
+  type = map(string)
+  default = {
+    dev  = "arn:aws:acm:us-east-2:660131068782:certificate/947e04b3-be22-43bf-bea2-a0ad3b221d70"
+    uat  = "arn:aws:acm:us-east-2:975049960860:certificate/e036e76d-1f4f-496e-ac97-5f65ec4a34cc"
+    prod = "arn:aws:acm:region:account-id:certificate/prod-cert-id"
+  }
 }

-# required
-variable "access_key" {
-  type    = string
+variable "key_name" {
 }

-# required
-variable "token" {
-  type    = string
-}
@@ -0,0 +1,2 @@
+vpc_id = "vpc-0f9d7c913f6522079"
+key_name = "tf-test-key"
@@ -0,0 +1,2 @@
+vpc_id = "vpc-0392396d0e7bdd77f"
+key_name = "tf-uat-key"
@@ -156,6 +156,7 @@ with buttons[1]:



+
    # @st.cache_data 
    # def convert_df(df):
    #    return df.to_csv(index=False).encode('utf-8')
@@ -8,6 +8,7 @@ from datetime import datetime
 import boto3
 import util
 import requests
+import time
 from sf_conn import get_client_names, get_secret, save_to_sf
 from constants import USER_LIST, DOCZY_PIPELINE_URL_DEV

@@ -74,10 +75,11 @@ client_row = st.columns([0.1, 0.8])
 with client_row[0]:
    st.write("**Client Name**")
 with client_row[1]:
-    client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed")
+    client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344

 client_bucket = client_s3_paths.get(client)
-# # to be deleted when buckets for different clients are ready; below line is added only for testing the corresponding DAG
+
+# to be deleted when buckets for different clients are ready; below line is added only for testing the corresponding DAG
 client_bucket = 'doczy-ai-client-1'

 batch_objects = s3_client.list_objects_v2(Bucket=client_bucket
@@ -87,17 +89,51 @@ batch_list = []
 for prefix in batch_objects['CommonPrefixes']:
    batch_list.append(prefix['Prefix'][:-1].split('/')[-1])

+# Hardcoded batch_list for testing purposes
+# batch_list = ['batch_020524103737', 'batch_090524131433', 'batch_090524131607', 'batch_100524123000', 'batch_130524064322', 
+#               'batch_160524071331', 'batch_200524213550', 'batch_250424112237', 'batch_280524120530', 'batch_280524121721', 'batch_280524144222',
+#               'batch_290524123926', 'batch_290524164044', 'batch_310524102029', 'batch_310524124050', 'batch_310524162346', 'batch_310524162631']
+
+if 'sorted_list' not in st.session_state:
+    st.session_state.sorted_list = batch_list
+
+def sort_list(ex_list, sort_by, order):
+    if sort_by == 'Alphabetical':
+        ex_list = sorted(ex_list, reverse=(order == 'Descending'))
+    elif sort_by == 'Create Date':
+        ex_list = ex_list if order == 'Ascending' else list(reversed(ex_list))
+
+    return ex_list
+
+col1, col2, col3, col4 = st.columns([0.5, 0.5, 0.5, 0.5])
+
+
+with col1: 
+    sort_by = st.radio("**Sort Batch_IDs**", ('Alphabetical', 'Create Date'))
+
+with col2:
+    order = st.radio('', ('Ascending','Descending'))
+
+with col3: 
+    add_vertical_space(2)
+    if st.button('Apply'):
+        st.session_state.sorted_list = sort_list(batch_list, sort_by, order)
+
 path_row = st.columns([0.1, 0.8])
 with path_row[0]:
    st.write("**Batch ID**")
 with path_row[1]:
-    batch_id = st.selectbox('**Batch ID**', batch_list, label_visibility = "collapsed")
+    batch_id = st.selectbox('**Batch ID**', st.session_state.sorted_list, label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344
+
+if not batch_id:
+    batch_id = "None"

 checks = st.columns([0.1, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12])
 with checks[0]:
    st.write("**Group No.**")
+
 with checks[1]:
-    a = st.checkbox('Unique Key', key = str(1))
+    a = st.checkbox('Unique Key', key = str(1), args="Unique")
 with checks[2]:
    b = st.checkbox('Pricing Before Carveouts', key = str(2))
 with checks[3]:
@@ -119,11 +155,15 @@ file_list = []
 file_objects = s3_client.list_objects_v2(Bucket=client_bucket
    , Prefix="contracts_landing_zone/"+batch_id+"/", Delimiter='/')

+# Hardcoded file_list for testing purposes
+# file_list = ['Boilerplate_TX Amendment Mission Health Network effective_040114 MU.pdf', 'Custom_TX - MP AMENDMENT - MISSION HEALTH NETWORK - MU.pdf', 
+#              'Delaware First Health_First State Homecare Agency_212260_7 MU.pdf', 'Molina Healthcare of Texas, Inc. Amendment 4 - HIX ACA__EFF 01012016_MU.pdf']
+
 if st.button("Read the contracts from Path"):
    for obj in file_objects.get('Contents',[]):
        if not obj['Key'].endswith('/'):
            file_list.append(obj['Key'].split('/')[-1])
-
+            
    df['Contract Name'] = file_list
    # df['Request ID'] = range(len(file_list))
    # df['Contract ID'] = file_list
@@ -160,23 +200,33 @@ additional_info = pd.DataFrame(columns=['CLIENT_NAME', 'BATCH_ID', 'REQUEST_USER
 additional_info.loc[0] = [client, batch_id, st.session_state.user_info['mail'], datetime.now().strftime("%Y-%m-%d %H:%M:%S")]
 st.write(additional_info)

+st.session_state.contract_count = 0
 contract_list = []
 for index, row in edited_df.iterrows():
+    allow_run_for_contract = False
    group_list = []
    if row['Unique Key']:
        group_list.append('Unique Key')
+        allow_run_for_contract = True
    if row['Pricing Before Carveouts']:
        group_list.append('Pricing Before Carveouts')
+        allow_run_for_contract = True
    if row['Contract Related']:
        group_list.append('Contract Related')
+        allow_run_for_contract = True
    if row['Provider']:
        group_list.append('Provider')
+        allow_run_for_contract = True
    if row['Timeline']:
        group_list.append('Timeline')
+        allow_run_for_contract = True
    if row['Carveout Indicator']:
        group_list.append('Carveout Indicator')
+        allow_run_for_contract = True
    if row['Carveout Methodology']:
        group_list.append('Carveout Methodology')
+        allow_run_for_contract = True
+    if allow_run_for_contract: st.session_state.contract_count += 1
    entry_dict = {
        "contract_name": row['Contract Name'],
        "groups": group_list,
@@ -197,23 +247,27 @@ with buttons[0]:
    st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv')
 with buttons[1]:
    if st.button("Run Doczy.AI Pipeline"):
-        # csv_buf = StringIO()
-        # additional_info.to_csv(csv_buf, header=True, index=False)
-        # csv_buf.seek(0)
-        # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/request_submission.csv')
-        # csv_buf = StringIO()
-        # edited_df.to_csv(csv_buf, header=True, index=False)
-        # csv_buf.seek(0)
-        # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/contract_config.csv')       
-        # try:
-        #     save_to_sf('load_request_and_contract_submissions', request_submission_file_name = "request_submission.csv", contract_config_file_name = "contract_config.csv")
-        # except Exception as e:
-        #     st.write(e)
-        response = requests.post(doczy_pipeline, json = myobj)
-        if response.status_code >= 200 and response.status_code < 300:
-            st.write("Success")
-            # st.write(myobj)
+        if not st.session_state.contract_count == len(edited_df):
+            st.error("Select at least one Group No. for every Contract")
        else:
-            st.write("Failed")
-        # st.write(response.text)
+            with st.spinner('Running...'):      
+                # csv_buf = StringIO()
+                # additional_info.to_csv(csv_buf, header=True, index=False)
+                # csv_buf.seek(0)
+                # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/request_submission.csv')
+                # csv_buf = StringIO()
+                # edited_df.to_csv(csv_buf, header=True, index=False)
+                # csv_buf.seek(0)
+                # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/contract_config.csv')       
+                # try:
+                #     save_to_sf('load_request_and_contract_submissions', request_submission_file_name = "request_submission.csv", contract_config_file_name = "contract_config.csv")
+                # except Exception as e:
+                #     st.write(e)
+                response = requests.post(doczy_pipeline, json = myobj)
+            if response.status_code >= 200 and response.status_code < 300:
+                st.write("Success")
+                # st.write(myobj)
+            else:
+                st.write("Failed")
+            # st.write(response.text)

@@ -10,6 +10,7 @@ from langchain.chains import RetrievalQA

 import streamlit as st
 from streamlit_extras.add_vertical_space import add_vertical_space
+from streamlit_pdf_viewer import pdf_viewer     # needs to be installed on the server
 import os
 import pandas as pd
 import numpy as np
@@ -18,24 +19,26 @@ import anthropic
 from pydantic import BaseModel
 from typing import List
 import re
+import base64
 from sf_conn import get_snowflake_conn
+import io

 REDIRECT_URI = 'https://doczydev.aarete.com:8502'
 user_list = USER_LIST

 st.set_page_config(layout = "wide")
 # Sidebar contents
-with st.sidebar:
-    st.title("Doczy.AI ™")
-    st.markdown(
-        """
-    ## About
-    This app extracts data from contracts
+# with st.sidebar:
+#     st.title("Doczy.AI ™")
+#     st.markdown(
+#         """
+#     ## About
+#     This app extracts data from contracts
 
-    """
-    )
-    add_vertical_space(15)
-    # st.write("Doczy")
+#     """
+#     )
+#     add_vertical_space(15)
+#     # st.write("Doczy")

 # AARETE LOGO
 x,y,z = st.columns([15,2,15])
@@ -104,13 +107,11 @@ for obj in objects['Contents']:

 contract_list = sorted(file_list)

-
-
 file_row = st.columns([0.2, 0.7, 0.1])
 with file_row[0]:
    st.write("**Contract Name**")
 with file_row[1]:
-    file_name = st.selectbox('Select a file', contract_list + ['All'], label_visibility = "collapsed")
+    file_name = st.selectbox('Select a file', ['All'] + contract_list, label_visibility = "collapsed", index= None) # MODIFIED - Append 'All' in the front instead of at the end

 field_row = st.columns([0.2, 0.7, 0.1])
 with field_row[0]:
@@ -120,7 +121,7 @@ with field_row[1]:
        , 'Pricing Before Carveouts - II', 'Carveout Indicator, Code Type and Code #s - I'
        , 'Carveout Indicator, Code Type and Code #s - II', 'Carveout Indicator, Code Type and Code #s - III'
        , 'Optimize Carving Indic.', 'Carveout Method - I', 'Carveout Method - II', 'Provider'
-        , 'Timeline'), label_visibility = "collapsed")
+        , 'Timeline'), label_visibility = "collapsed", index = None)

 if field_group == 'Unique Key':
    fields = fields[fields['PRIORITY'] == 'A']
@@ -164,10 +165,51 @@ if st.button("Show Results"):
    df2 = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description])

    # get this dataframe from snowflake table
-    # df2 = pd.DataFrame(columns=['Contract Name','Field Name', 'SF_DB_COL_NAME', 'Snippet','Page Number'
-        # , 'Field Extracted Value', 'Actual Value','Imputed Value'])
+    df2 = pd.DataFrame(columns=['Contract Name','Field Name', 'SF_DB_COL_NAME', 'Snippet','Page Number'
+        , 'Field Extracted Value', 'Actual Value','Imputed Value'])
    df2.to_csv('temp2.csv', index=False)

+if st.button("Show PDF"):
+    if file_name == None or file_name == "All":
+        st.error("Choose one specific file.")
+    else:
+        with st.sidebar:
+            st.markdown(
+                """
+                <style>
+                    section[data-testid="stSidebar"] {
+                        width: 550px !important; # Set the width to your desired value
+                    }
+                </style>
+                """,
+                unsafe_allow_html=True,
+            )
+            s3_client.Object(bucket,file_name)
+            data=obj.get()['Body'].read()
+            pdf_viewer(io.BytesIO(data), width=1500)
+
+# if st.button("Show PDF"):
+#     if file_name == None or file_name == "All":
+#         st.error("Choose one specific file.")
+#     else:
+#         with st.sidebar:
+#             with open(file_name, "rb") as f:
+#                 base64_pdf = base64.b64encode(f.read()).decode('utf-8')
+#             # Embedding PDF in HTML
+#             pdf_display = F'<iframe src="data:application/pdf;base64,{base64_pdf}" width="500" height="1000" type="application/pdf"></iframe>'
+#             # Displaying File
+#             st.markdown(
+#                 """
+#                 <style>
+#                     section[data-testid="stSidebar"] {
+#                         width: 600px !important; # Set the width to your desired value
+#                     }
+#                 </style>
+#                 """,
+#                 unsafe_allow_html=True,
+#             )
+#             st.markdown(pdf_display, unsafe_allow_html=True)
+
 df2 = pd.read_csv('temp2.csv')
 df2['Imputed Value'] = ''
 edited_df = st.data_editor(df2)
@@ -0,0 +1,259 @@
+import streamlit as st
+from streamlit_extras.add_vertical_space import add_vertical_space
+import os
+import streamlit as st
+import pandas as pd
+from io import StringIO
+from datetime import datetime
+import boto3
+# import util
+import requests
+import time
+# from sf_conn import get_client_names, get_secret, save_to_sf
+# from constants import USER_LIST, DOCZY_PIPELINE_URL_DEV
+
+# doczy_pipeline = DOCZY_PIPELINE_URL_DEV
+
+# REDIRECT_URI = 'https://doczydev.aarete.com:8501'
+# user_list = USER_LIST
+st.set_page_config(layout = "wide")
+# # # Sidebar contents
+# # with st.sidebar:
+# #     st.title("Doczy.AI ™")
+# #     st.markdown(
+# #         """
+# #     ## About
+# #     This app extracts data from contracts
+ 
+# #     """
+# #     )
+# #     add_vertical_space(15)
+# #     # st.write("Doczy")
+
+_,c1= st.columns([5,1])
+# try:
+#     util.setup_page(REDIRECT_URI)
+# except:
+#     st.write("SSO Failed")
+#     st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'}
+st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'}
+try:
+    c1.write(f"User: **{st.session_state.user_info['displayName']}**")
+    user_mail = st.session_state.user_info['mail']
+except KeyError as e:
+    # Do we add a link to get to the login page here? 
+    st.write("Session Expired.")
+    st.stop()
+
+
+
+s3_client = boto3.client('s3',
+    region_name="us-east-2",
+)
+
+# # to be replaced with snowflake data
+client_list = ['doczy-ai-client-1', 'Delaware First Health, Inc.', 'Community Health Choice, Inc','CareSource Network Partners LLC',
+    'HealthNet of Cali', 'Oklahoma Complete Health, Inc', 'HealthFirst', 'Molina Healthcare of TX', 'AvMed', 'Arizona Care1st',
+    'WellCare New Jersey']
+
+# client_list, s3_paths = get_client_names()
+# client_s3_paths = dict(zip(client_list, s3_paths))
+
+client_row = st.columns([0.1, 0.8])
+with client_row[0]:
+    st.write("**Client Name**")
+with client_row[1]:
+    client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344
+
+# client_bucket = client_s3_paths.get(client)
+
+# # to be deleted when buckets for different clients are ready; below line is added only for testing the corresponding DAG
+client_bucket = 'doczy-ai-client-1'
+
+# batch_objects = s3_client.list_objects_v2(Bucket=client_bucket
+#     , Prefix="contracts_landing_zone/", Delimiter='/')      
+
+# batch_list = []
+# for prefix in batch_objects['CommonPrefixes']:
+#     batch_list.append(prefix['Prefix'][:-1].split('/')[-1])
+
+batch_list = ['batch_020524103737', 'batch_090524131433', 'batch_090524131607', 'batch_100524123000', 'batch_130524064322', 
+              'batch_160524071331', 'batch_200524213550', 'batch_250424112237', 'batch_280524120530', 'batch_280524121721', 'batch_280524144222',
+              'batch_290524123926', 'batch_290524164044', 'batch_310524102029', 'batch_310524124050', 'batch_310524162346', 'batch_310524162631']
+
+if 'sorted_list' not in st.session_state:
+    st.session_state.sorted_list = batch_list
+
+def sort_list(ex_list, sort_by, order):
+    if sort_by == 'Alphabetical':
+        ex_list = sorted(ex_list, reverse=(order == 'Descending'))
+    elif sort_by == 'Create Date':
+        ex_list = ex_list if order == 'Ascending' else list(reversed(ex_list))
+
+    return ex_list
+
+col1, col2, col3, col4 = st.columns([0.5, 0.5, 0.5, 0.5])
+
+
+with col1: 
+    sort_by = st.radio("**Sort Batch_IDs**", ('Alphabetical', 'Create Date'))
+
+with col2:
+    order = st.radio('', ('Ascending','Descending'))
+
+with col3: 
+    add_vertical_space(2)
+    if st.button('Apply'):
+        st.session_state.sorted_list = sort_list(batch_list, sort_by, order)
+
+path_row = st.columns([0.1, 0.8])
+with path_row[0]:
+    st.write("**Batch ID**")
+with path_row[1]:
+    batch_id = st.selectbox('**Batch ID**', st.session_state.sorted_list, label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344
+
+if not batch_id:
+    batch_id = "None"
+
+checks = st.columns([0.1, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12])
+with checks[0]:
+    st.write("**Group No.**")
+
+with checks[1]:
+    a = st.checkbox('Unique Key', key = str(1), args="Unique")
+with checks[2]:
+    b = st.checkbox('Pricing Before Carveouts', key = str(2))
+with checks[3]:
+    c = st.checkbox('Contract Related', key = str(3))
+with checks[4]:
+    d = st.checkbox('Provider', key = str(4))
+with checks[5]:
+    e = st.checkbox('Timeline', key = str(5))
+with checks[6]:
+    f = st.checkbox('Carveout Indicator', key = str(6))
+with checks[7]:
+    g = st.checkbox('Carveout Methodology', key = str(7))
+
+add_vertical_space(1)
+
+df = pd.DataFrame(columns=['Contract Name', 'Unique Key','Pricing Before Carveouts'
+    , 'Contract Related', 'Provider', 'Timeline', 'Carveout Indicator', 'Carveout Methodology'])
+# file_list = []
+# file_objects = s3_client.list_objects_v2(Bucket=client_bucket
+#     , Prefix="contracts_landing_zone/"+batch_id+"/", Delimiter='/')
+file_list = ['Boilerplate_TX Amendment Mission Health Network effective_040114 MU.pdf', 'Custom_TX - MP AMENDMENT - MISSION HEALTH NETWORK - MU.pdf', 
+             'Delaware First Health_First State Homecare Agency_212260_7 MU.pdf', 'Molina Healthcare of Texas, Inc. Amendment 4 - HIX ACA__EFF 01012016_MU.pdf']
+
+if st.button("Read the contracts from Path"):
+    # for obj in file_objects.get('Contents',[]):
+    #     if not obj['Key'].endswith('/'):
+    #         file_list.append(obj['Key'].split('/')[-1])
+    df['Contract Name'] = file_list
+    # df['Request ID'] = range(len(file_list))
+    # df['Contract ID'] = file_list
+    df['Unique Key'] = a
+    df['Pricing Before Carveouts'] = b
+    df['Contract Related'] = c
+    df['Provider'] = d
+    df['Timeline'] = e
+    df['Carveout Indicator'] = f
+    df['Carveout Methodology'] = g
+    dir_path = os.path.dirname(os.path.realpath(__file__))
+    print(f'DEBUGGING: PWD= {dir_path}')
+    df.to_csv('temp1.csv', index=False)
+
+add_vertical_space(1)
+df2 = pd.read_csv('temp1.csv')
+edited_df = st.data_editor(df2)
+
+edited_df['REQUEST_USER'] = user_mail
+edited_df['LATEST_FLAG BOOLEAN'] = True
+edited_df['PIPELINE_KICKOFF_DATETIME'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+edited_df['REQUEST_DATETIME'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+@st.cache_data 
+def convert_df(df):
+    return df.to_csv(index=False).encode('utf-8')
+
+csv = convert_df(edited_df)
+# edited_df = edited_df.reset_index()  # make sure indexes pair with number of rows
+
+# additional_info = pd.DataFrame(columns=['REQUEST_ID','T_DRIVE_PATH','CLIENT_NAME'
+# , 'GROUP_NAME', 'REQUEST_USERNAME', 'REQUEST_DATETIME'])
+additional_info = pd.DataFrame(columns=['CLIENT_NAME', 'BATCH_ID', 'REQUEST_USERNAME', 'REQUEST_DATETIME'])
+additional_info.loc[0] = [client, batch_id, st.session_state.user_info['mail'], datetime.now().strftime("%Y-%m-%d %H:%M:%S")]
+st.write(additional_info)
+
+st.session_state.contract_count = 0
+contract_list = []
+for index, row in edited_df.iterrows():
+    allow_run_for_contract = False
+    group_list = []
+    if row['Unique Key']:
+        group_list.append('Unique Key')
+        allow_run_for_contract = True
+    if row['Pricing Before Carveouts']:
+        group_list.append('Pricing Before Carveouts')
+        allow_run_for_contract = True
+    if row['Contract Related']:
+        group_list.append('Contract Related')
+        allow_run_for_contract = True
+    if row['Provider']:
+        group_list.append('Provider')
+        allow_run_for_contract = True
+    if row['Timeline']:
+        group_list.append('Timeline')
+        allow_run_for_contract = True
+    if row['Carveout Indicator']:
+        group_list.append('Carveout Indicator')
+        allow_run_for_contract = True
+    if row['Carveout Methodology']:
+        group_list.append('Carveout Methodology')
+        allow_run_for_contract = True
+    if allow_run_for_contract: st.session_state.contract_count += 1
+    entry_dict = {
+        "contract_name": row['Contract Name'],
+        "groups": group_list,
+        "contract_source_path": "contracts_landing_zone/"+batch_id+"/"+row['Contract Name']
+    }
+    contract_list.append(entry_dict)
+
+myobj = {
+    "s3_bucket": client_bucket,
+    "batch_id": batch_id,
+    "client_name": client,
+    "username": user_mail,
+    "contract_list": contract_list
+}
+
+buttons = st.columns([0.8, 0.2])
+with buttons[0]:
+    st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv')
+with buttons[1]:
+    if st.button("Run Doczy.AI Pipeline"):
+        if not st.session_state.contract_count == len(edited_df):
+            st.error("Select at least one Group No. for every Contract")
+        else:
+            with st.spinner('Running...'):              # Feedback to User while API endpoint sends response for DOC-342
+                # csv_buf = StringIO()
+                # additional_info.to_csv(csv_buf, header=True, index=False)
+                # csv_buf.seek(0)
+                # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/request_submission.csv')
+                # csv_buf = StringIO()
+                # edited_df.to_csv(csv_buf, header=True, index=False)
+                # csv_buf.seek(0)
+                # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/contract_config.csv')       
+                # try:
+                #     save_to_sf('load_request_and_contract_submissions', request_submission_file_name = "request_submission.csv", contract_config_file_name = "contract_config.csv")
+                # except Exception as e:
+                #     st.write(e)
+                # response = requests.post(doczy_pipeline, json = myobj)
+                # if response.status_code >= 200 and response.status_code < 300:
+                #     st.write("Success")
+                #     # st.write(myobj)
+                # else:
+                #     st.write("Failed")
+                # st.write(response.text)
+                time.sleep(5)
+            st.write("Success!")
+
@@ -0,0 +1,228 @@
+import json
+
+import boto3
+from langchain.prompts import PromptTemplate
+from langchain.embeddings.bedrock import BedrockEmbeddings
+from langchain.llms.bedrock import Bedrock
+from langchain_community.vectorstores import Chroma
+# from constants import CHROMA_SETTINGS, EMBEDDING_MODEL_NAME, PERSIST_DIRECTORY, MODEL_ID, MODEL_BASENAME, SOURCE_DIRECTORY, USER_LIST
+from langchain.chains import RetrievalQA
+
+import streamlit as st
+from streamlit_extras.add_vertical_space import add_vertical_space
+from streamlit_pdf_viewer import pdf_viewer     # needs to be installed on the server
+import os
+import pandas as pd
+import numpy as np
+# import util
+import anthropic
+from pydantic import BaseModel
+from typing import List
+import re
+import base64
+# from sf_conn import get_snowflake_conn
+
+REDIRECT_URI = 'https://doczydev.aarete.com:8502'
+# user_list = USER_LIST
+
+st.set_page_config(layout = "wide")
+# Sidebar contents
+# with st.sidebar:
+#     st.title("Doczy.AI ™")
+#     st.markdown(
+#         """
+#     ## About
+#     This app extracts data from contracts
+ 
+#     """
+#     )
+#     add_vertical_space(15)
+#     # st.write("Doczy")
+
+_,c1= st.columns([5,1])
+# try:
+#     util.setup_page(REDIRECT_URI)
+# except:
+#     st.write("SSO Failed")
+#     st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'}
+st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'}
+try:
+    c1.write(f"User: **{st.session_state.user_info['displayName']}**")
+    user_mail = st.session_state.user_info['mail']
+except KeyError as e:
+    st.write("Session Expired.")
+    st.stop()
+
+# remove below try except statement if comparison with actual vales is not required
+# try:
+#     conn = get_snowflake_conn('STG')
+#     cur = conn.cursor()
+#     query = 'select * from "TRAINING_DATA_RAW"'
+#     cur.execute(query)
+#     field_values = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description])
+#     field_values['Document_Name'] = field_values['DOCUMENT_NAME']
+# except:
+#     # field_values = pd.read_csv('contract_field_values.csv', encoding='unicode_escape', skipinitialspace=True)
+#     # field_values = field_values.loc[:, ~field_values.columns.str.contains('Unnamed:')]
+#     st.write("Conn failed, unable to fetch data from training data table in Snowflake")
+
+field_values = pd.read_csv('contract_field_values.csv', encoding='unicode_escape', skipinitialspace=True)
+field_values = field_values.loc[:, ~field_values.columns.str.contains('Unnamed:')]
+
+# try:
+#     query = 'select * from "PROMPT_CONFIG"'
+#     cur.execute(query)
+#     fields = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description])
+
+#     fields.rename(columns={'FIELD_DESC': 'Field Name'}, inplace = True)
+#     fields.rename(columns={'PROMPT': 'Interrogation Question?'}, inplace = True)
+#     fields.rename(columns={'GROUP_ID': 'PRIORITY'}, inplace = True)
+#     fields.rename(columns={'FIELD_NAME': 'SF_DB_COL_NAME'}, inplace = True)
+#     fields.rename(columns={'FM_MODEL_ID': 'llm_selected'}, inplace = True)    
+# except Exception as e:
+#     st.write("Unable to fetch data from Snowflake: ",e)
+#     # fields = pd.read_csv('contract_fields.csv', encoding='unicode_escape', skipinitialspace=True)
+#     # fields = fields[~fields['SF_COL_NAME'].str.endswith('_PG', na=None)]
+
+fields = pd.read_csv('contract_fields.csv', encoding='unicode_escape', skipinitialspace=True)
+# fields = fields[fields['SF_DB_COL_NAME'].str.endswith('_PG', na=None)]
+
+# change the code below if contract list is fetched from snowflake
+s3_client = boto3.client('s3',
+    region_name="us-east-2"
+)
+bucket = 'doczy-dev-infra-textract'
+# objects = s3_client.list_objects_v2(Bucket=bucket, Prefix="training-data/")
+# file_list = []
+# for obj in objects['Contents']:
+#     if not obj['Key'].endswith('/'):
+#         file_list.append(obj['Key'])
+
+file_list = ['Contract_Training_Exercise_Pricing.pdf', 'Contract_Training_Exercise_SLA.pdf']
+contract_list = sorted(file_list)
+
+file_row = st.columns([0.2, 0.7, 0.1])
+with file_row[0]:
+    st.write("**Contract Name**")
+with file_row[1]:
+    file_name = st.selectbox('Select a file', ['All'] + contract_list, label_visibility = "collapsed", index= None) # MODIFIED - Append 'All' in the front instead of at the end
+
+field_row = st.columns([0.2, 0.7, 0.1])
+with field_row[0]:
+    st.write("**Field Group**")
+with field_row[1]:
+    field_group = st.selectbox('Field Group',('Unique Key', 'Contract Related', 'Pricing Before Carveouts - I'
+        , 'Pricing Before Carveouts - II', 'Carveout Indicator, Code Type and Code #s - I'
+        , 'Carveout Indicator, Code Type and Code #s - II', 'Carveout Indicator, Code Type and Code #s - III'
+        , 'Optimize Carving Indic.', 'Carveout Method - I', 'Carveout Method - II', 'Provider'
+        , 'Timeline'), label_visibility = "collapsed", index = None)
+
+if field_group == 'Unique Key':
+    fields = fields[fields['PRIORITY'] == 'A']
+elif field_group == 'Contract Related':
+    fields = fields[fields['PRIORITY'] == 'C']
+elif field_group == 'Pricing Before Carveouts - I':
+    fields = fields[fields['PRIORITY'] == 'B']
+    fields = np.array_split(fields, 2)[0]
+elif field_group == 'Pricing Before Carveouts - II':
+    fields = fields[fields['PRIORITY'] == 'B']
+    fields = np.array_split(fields, 2)[1]
+elif field_group == 'Carveout Indicator, Code Type and Code #s - I':
+    fields = fields[fields['PRIORITY'] == 'F']
+    fields = np.array_split(fields, 3)[0]
+elif field_group == 'Carveout Indicator, Code Type and Code #s - II':
+    fields = fields[fields['PRIORITY'] == 'F']
+    fields = np.array_split(fields, 3)[1]
+elif field_group == 'Carveout Indicator, Code Type and Code #s - III':
+    fields = fields[fields['PRIORITY'] == 'F']
+    fields = np.array_split(fields, 3)[2]
+elif field_group == 'Carveout Methodology - I':
+    fields = fields[fields['PRIORITY'] == 'G']
+    fields = np.array_split(fields, 4)[0]
+elif field_group == 'Carveout Methodology - II':
+    fields = fields[fields['PRIORITY'] == 'G']
+    fields = np.array_split(fields, 4)[1]
+elif field_group == 'Carveout Method - III':
+    fields = fields[fields['PRIORITY'] == 'G']
+    fields = np.array_split(fields, 4)[2]
+elif field_group == 'Carveout Method - IV':
+    fields = fields[fields['PRIORITY'] == 'G']
+    fields = np.array_split(fields, 4)[3]
+elif field_group == 'Provider':
+    fields = fields[fields['PRIORITY'] == 'D']
+elif field_group == 'Timeline':
+    fields = fields[fields['PRIORITY'] == 'E']
+
+if st.button("Show Results"):
+    # query = 'select * from "DOCZY_PIPELINE_RAW_OUTPUT"'
+    # cur.execute(query)
+    # df2 = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description])
+
+    # get this dataframe from snowflake table
+    df2 = pd.DataFrame(columns=['Contract Name','Field Name', 'SF_DB_COL_NAME', 'Snippet','Page Number'
+        , 'Field Extracted Value', 'Actual Value','Imputed Value'])
+    df2.to_csv('temp2.csv', index=False)
+
+if st.button("Show PDF"):
+    if file_name == None or file_name == "All":
+        st.error("Choose one specific file.")
+    else:
+        with st.sidebar:
+            st.markdown(
+                """
+                <style>
+                    section[data-testid="stSidebar"] {
+                        width: 550px !important; # Set the width to your desired value
+                    }
+                </style>
+                """,
+                unsafe_allow_html=True,
+            )
+            pdf_viewer(file_name, width=1500)
+
+# if st.button("Show PDF"):
+#     if file_name == None or file_name == "All":
+#         st.error("Choose one specific file.")
+#     else:
+#         with st.sidebar:
+#             with open(file_name, "rb") as f:
+#                 base64_pdf = base64.b64encode(f.read()).decode('utf-8')
+#             # Embedding PDF in HTML
+#             pdf_display = F'<iframe src="data:application/pdf;base64,{base64_pdf}" width="500" height="1000" type="application/pdf"></iframe>'
+#             # Displaying File
+#             st.markdown(
+#                 """
+#                 <style>
+#                     section[data-testid="stSidebar"] {
+#                         width: 600px !important; # Set the width to your desired value
+#                     }
+#                 </style>
+#                 """,
+#                 unsafe_allow_html=True,
+#             )
+#             st.markdown(pdf_display, unsafe_allow_html=True)
+
+df2 = pd.read_csv('temp2.csv')
+df2['Imputed Value'] = ''
+edited_df = st.data_editor(df2)
+
+@st.cache_data 
+def convert_df(df):
+    return df.to_csv(index=False).encode('utf-8')
+
+csv = convert_df(edited_df)
+
+buttons = st.columns(3)
+with buttons[0]:
+    # st.button("Save All Imputations") 
+    st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv')
+with buttons[1]:
+    # st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv')
+    st.write("")
+with buttons[2]:
+    if st.button("Kickoff Database Integration"):
+        st.write("Stored in DB")
+
+
+
+
@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+import os
+import re
+import subprocess
+import logging
+import signal
+from typing import Callable, Optional, Dict, Tuple, List
+
+LOGGER = logging.getLogger(__name__)
+LOGGER.setLevel(logging.INFO)
+handler = logging.StreamHandler()
+handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
+LOGGER.addHandler(handler)
+
+
+class SensitiveFormatter(logging.Formatter):
+    """Formatter that removes sensitive information from logs."""
+
+    @staticmethod
+    def mask_password(log: str) -> str:
+        return re.sub(r'password=([^\s]+)', r'password=*****', log)
+
+    @staticmethod
+    def mask_api_key(log: str) -> str:
+        return re.sub(r'api_key=([^\s]+)', r'api_key=*****', log)
+
+    @staticmethod
+    def _mask(s: str) -> str:
+        filtered = SensitiveFormatter.mask_password(s)
+        filtered = SensitiveFormatter.mask_api_key(filtered)
+        return filtered
+
+    def format(self, record) -> str:
+        original = super().format(record)
+        return self._mask(original)
+
+
+def prepare_logger() -> logging.Logger:
+    global LOGGER
+    if LOGGER is not None:
+        return LOGGER
+    LOGGER = logging.getLogger(__name__)
+    LOGGER.setLevel(logging.INFO)
+    log_format = '%(asctime)s %(filename)s:%(lineno)-4s [%(levelname)s] %(message)s'
+    handler = logging.StreamHandler()
+    handler.setFormatter(SensitiveFormatter(log_format))
+    LOGGER.addHandler(handler)
+    return LOGGER
+
+
+def get_blue_shade(log_prefix: str) -> str:
+    """Returns an ANSI color code for a shade of blue based on the hash of the log_prefix."""
+    blue_shades = [81, 87, 117, 153, 159, 195, 111, 45, 39]
+    hash_value = hash(log_prefix)
+    blue_index = hash_value % len(blue_shades)
+    return f"\033[38;5;{blue_shades[blue_index]}m"
+
+
+def signal_handler(sig, frame, process):
+    print("Ctrl+C pressed! Sending SIGTERM to Terraform process...")
+    process.terminate()
+
+
+def run_command(command: str, log_output: bool = False, decorate_logs: bool = True, log_cmd: bool = False,
+                log_prefix: str = "", envs: Optional[str] = None, secrets: Optional[Dict[str, str]] = None,
+                failure_callback: Optional[Callable[[str], None]] = None,
+                cwd: Optional[str] = None) -> Tuple[int, List[str]]:
+    new_env = os.environ.copy()
+    if secrets:
+        for key, value in secrets.items():
+            new_env[key] = os.path.expandvars(value)
+
+    full_command = f"{envs} {command}" if envs else command
+    process = subprocess.Popen(full_command, shell=True, cwd=cwd, env=new_env,
+                               stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+
+    # Setup signal handler
+    original_sigint_handler = signal.getsignal(signal.SIGINT)
+    signal.signal(signal.SIGINT, lambda sig, frame: signal_handler(sig, frame, process))
+
+    log_prefix_env = os.getenv("LOG_PREFIX", "")
+    log_prefix_full = f"[{log_prefix_env}] {log_prefix}" if log_prefix_env else log_prefix
+
+    if log_cmd:
+        LOGGER.info(
+            f"{get_blue_shade(log_prefix_full)}{log_prefix_full}\033[0m: Running command: \033[1;34m{command}\033[0m")
+        if envs:
+            LOGGER.info(f"Using additional envs: \033[1;34m{envs}\033[0m")
+        if secrets:
+            LOGGER.info(f"Using additional environment variables with secrets: \033[1;34m{list(secrets.keys())}\033[0m")
+
+    cmd_output = []
+    while True:
+        output = process.stdout.readline() if process.stdout else ''
+        if output:
+            output_line = output.strip()
+            if log_output:
+                log_statement = (
+                    f"{get_blue_shade(log_prefix_full)}{log_prefix_full}\033[0m: "
+                    f"{output_line}") if log_prefix else output_line
+                LOGGER.info(log_statement) if decorate_logs else print(log_statement)
+            cmd_output.append(output_line)
+        elif process.poll() is not None:
+            break
+
+    # Restore original signal handler
+    signal.signal(signal.SIGINT, original_sigint_handler)
+
+    rc = process.poll() or 0  # Ensure rc is an int, default to 0 if None
+    if rc != 0 and failure_callback:
+        failure_callback(f"Command returned code: {rc}")
+
+    return rc, cmd_output
+
+
+def decorate_successful(text: str) -> str:
+    return f"\033[32;1m{text}\033[0m"
+
+
+def decorate_white_bold(text: str) -> str:
+    return f"\033[37;1m{text}\033[0m"
+
+
+def decorate_warn(text: str) -> str:
+    return f"\033[33;1m{text}\033[0m"
@@ -1,3 +0,0 @@
-# AWS Lambda handler function
-def lambda_handler(event, context):
-    print("Receiver Lambda")
@@ -1,3 +0,0 @@
-# AWS Lambda handler function
-def lambda_handler(event, context):
-    print("Sender Lambda")
@@ -1,42 +0,0 @@
-terraform {
-  required_providers {
-    aws = {
-      source = "hashicorp/aws"
-      version = "5.39.0"
-    }
-  }
-
-  backend "s3" {
-    bucket         = "doczy-test-bucket" # Parameterize using -backend-config flag with "terraform init"
-    key            = "terraform/terraform_new2.tfstate" # Parameterize
-    region         = "us-east-2" # Parameterize 
-    # profile        = "doczyai" # Parameterize
-    dynamodb_table = "terraform-lock" # Parameterize
-    encrypt        = true
-  }
-}
-
-provider "aws" {
-  profile = var.aws_profile
-}
-
-module "textract_pipeline" {
-  source = "./modules/aws-textract-pipeline"
-  project_name = var.project_name
-  aws_region = var.aws_region
-  environment = var.environment
-
-  # for each for multiple resources
-  for_each = var.client_list
-
-  client_name = each.value.client_name
-  s3_bucket_name = each.value.s3_bucket_name
-
-  lambda_role = "arn:aws:iam::660131068782:role/doczy-dev-infra-lambda-universal-role"
-  
-}
-
-
-# List - multiple client, separate state files
-# client abbrv
-# stream-wise folders or repo
@@ -1,212 +0,0 @@
-locals {
-  region_map = {
-    us-east-1 = "use1"
-    us-east-2 = "use2"
-    us-west-1 = "usw1"
-    us-west-2 = "usw2"
-  }
-  environment_map = {
-    prod    = "p"
-    uat     = "u"
-    qa      = "q"
-    dev     = "d"
-  }
-
-  common_tags = {
-    environment = var.environment
-
-  }
-
-  # region abbreviation
-  region_short_name   = local.region_map[var.aws_region]
-
-  # environment abbreviation
-  environment_short_name   = local.environment_map[var.environment]
-
-  # global prefix
-  global_prefix = "${var.project_name}-${local.region_short_name}-${local.environment_short_name}-${var.client_name}"
-  
-  # Resource prefix
-  lambda_prefix = "${local.global_prefix}-lmb"
-  sqs_prefix = "${local.global_prefix}-sqs"
-  sns_prefix = "${local.global_prefix}-sns"
-  
-  # Lambda names
-  sender_lambda_name = "${local.lambda_prefix}-${var.textract_sender_lambda.name}"
-  receiver_lambda_name = "${local.lambda_prefix}-${var.textract_receiver_lambda.name}"
-
-  # SQS names
-  sender_queue_name = "${local.sqs_prefix}-${var.sender_sqs_queue.name}"
-  receiver_queue_name = "${local.sqs_prefix}-${var.receiver_sqs_queue.name}"
-
-  # SNS names
-  textract_notification_sns_name = "${local.sns_prefix}-textract-notification"
-
-  # property file path
-  property_file_s3_path = "${var.s3_bucket_name}/config.properties"
-}
-
-# sender lambda function
-module "textract_sender_lambda_function" {
-  source = "terraform-aws-modules/lambda/aws"
-  version = "7.2.2"
-
-  function_name = local.sender_lambda_name
-  description   = var.textract_sender_lambda.description
-  handler       = var.textract_sender_lambda.handler
-  runtime       = var.textract_sender_lambda.runtime
-  timeout       = var.textract_sender_lambda.timeout
-  memory_size   = var.textract_sender_lambda.memory_size
-  ephemeral_storage_size  = var.textract_sender_lambda.ephemeral_storage_size
-  architectures = var.textract_sender_lambda.architectures
-  
-  environment_variables = {
-    PROPERTY_FILE_S3_PATH = local.property_file_s3_path
-  }
-
-  source_path   = var.textract_sender_lambda.source_path
-
-  event_source_mapping = {
-    sqs = {
-      event_source_arn        = module.textract_sender_sqs.queue_arn
-      function_response_types = ["ReportBatchItemFailures"]
-      scaling_config = {
-        maximum_concurrency = 20
-        
-      }
-      batch_size = 1
-    }
-  }
-
-  lambda_role  = var.lambda_role
-  create_role = false
-
-  tags    = local.common_tags
-}
-
-# Sender SQS Queue
-module "textract_sender_sqs" {
-  source  = "terraform-aws-modules/sqs/aws"
-  version = "4.1.1"
-  name    = local.sender_queue_name
-  delay_seconds             = var.sender_sqs_queue.delay_seconds
-  max_message_size          = var.sender_sqs_queue.max_message_size
-  message_retention_seconds = var.sender_sqs_queue.message_retention_seconds
-  visibility_timeout_seconds= var.sender_sqs_queue.visibility_timeout_seconds
-  tags    = local.common_tags
-}
-
-# receiver lambda function
-module "textract_receiver_lambda_function" {
-  source = "terraform-aws-modules/lambda/aws"
-  version = "7.2.2"
-
-  function_name = local.receiver_lambda_name
-  description   = var.textract_receiver_lambda.description
-  handler       = var.textract_receiver_lambda.handler
-  runtime       = var.textract_receiver_lambda.runtime
-  timeout       = var.textract_receiver_lambda.timeout
-  memory_size   = var.textract_receiver_lambda.memory_size
-  ephemeral_storage_size  = var.textract_receiver_lambda.ephemeral_storage_size
-  architectures = var.textract_receiver_lambda.architectures
-  
-  environment_variables = {
-    PROPERTY_FILE_S3_PATH = local.property_file_s3_path
-  }
-
-  source_path   = var.textract_receiver_lambda.source_path
-
-  event_source_mapping = {
-    sqs = {
-      event_source_arn        = module.textract_receiver_sqs.queue_arn
-      function_response_types = ["ReportBatchItemFailures"]
-      scaling_config = {
-        maximum_concurrency = 20 
-      }
-      batch_size = 1
-    }
-  }
-
-  lambda_role  = var.lambda_role
-  create_role = false
-
-  tags    = local.common_tags
-}
-
-# textract notification sns 
-module "textract_notification_sns" {
-  source  = "terraform-aws-modules/sns/aws"
-  version = ">= 5.0"
-
-  name = local.textract_notification_sns_name
-
-  topic_policy_statements = {
-    sqs = {
-      sid = "SQSSubscribe"
-      actions = [
-        "sns:Subscribe",
-        "sns:Receive",
-      ]
-
-      principals = [{
-        type        = "AWS"
-        identifiers = ["*"]
-      }]
-
-      conditions = [{
-        test     = "StringLike"
-        variable = "sns:Endpoint"
-        values   = [module.textract_receiver_sqs.queue_arn]
-      }]
-    }
-  }
-
-  subscriptions = {
-    sqs = {
-      protocol = "sqs"
-      endpoint = module.textract_receiver_sqs.queue_arn
-    }
-  }
-
-  tags = {
-    Environment = "dev"
-  }
-}
-
-# Receiver SQS Queue
-module "textract_receiver_sqs" {
-  source  = "terraform-aws-modules/sqs/aws"
-  version = "4.1.1"
-  name    = local.receiver_queue_name
-  delay_seconds             = var.receiver_sqs_queue.delay_seconds
-  max_message_size          = var.receiver_sqs_queue.max_message_size
-  message_retention_seconds = var.receiver_sqs_queue.message_retention_seconds
-  visibility_timeout_seconds= var.receiver_sqs_queue.visibility_timeout_seconds
-  tags    = local.common_tags
-
-  create_queue_policy = true
-  queue_policy_statements = {
-    sns = {
-      sid     = "SNSPublish"
-      actions = ["sqs:SendMessage"]
-
-      principals = [
-        {
-          type        = "Service"
-          identifiers = ["sns.amazonaws.com"]
-        }
-      ]
-
-      conditions = [{
-        test     = "ArnEquals"
-        variable = "aws:SourceArn"
-        values   = [module.textract_notification_sns.topic_arn]
-      }]
-    }
-  }
-}
-
-
-
-
-
@@ -1,26 +0,0 @@
-# Output variable definitions
-
-output "sender_lambda_arn" {
-  description = "Sender lambda arn"
-  value       = module.textract_sender_lambda_function.lambda_function_arn
-}
-
-output "receiver_lambda_arn" {
-  description = "Receiver lambda arn"
-  value       = module.textract_receiver_lambda_function.lambda_function_arn
-}
-
-output "sender_queue_arn" {
-  description = "Sender queue arn"
-  value       = module.textract_sender_sqs.queue_arn
-}
-
-output "receiver_queue_arn" {
-  description = "Receiver queue arn"
-  value       = module.textract_receiver_sqs.queue_arn
-}
-
-output "textract_notification_sns_arn" {
-  description = "value"
-  value = module.textract_notification_sns.topic_arn
-}
@@ -1,130 +0,0 @@
-# Required
-variable "project_name" {
-  type = string
-}
-
-# Required
-variable "client_name" {
-  type = string
-}
-
-
-# Required
-variable "environment" {
-  type = string
-}
-
-# Required
-variable "aws_region" {
-  type = string
-}
-
-# Required
-variable "lambda_role" {
-  type = string
-}
-
-# Required
-variable "s3_bucket_name" {
-  type = string
-}
-
-# textract_sender_lambda
-variable "textract_sender_lambda" {
-  type = object({
-    name    = string
-    description = string
-    handler = string
-    runtime = string
-    timeout = number
-    memory_size = number
-    ephemeral_storage_size = number
-    architectures = list(string)
-    layers = list(string)
-    environment_variables = map(string)
-    source_path = string
-  })
-  default = {
-    name = "textract-sender"
-    description = "This function sends async request to TEXTRACT service"
-    handler = "index.lambda_handler"
-    runtime = "python3.12"
-    timeout = 60
-    memory_size = 256
-    ephemeral_storage_size = 512
-    architectures = ["x86_64"]
-    layers = []
-    environment_variables = {
-      PROPERTY_FILE_S3_PATH = "doczy-dev-infra-textract/config.properties"
-    }
-    source_path = "../src/lambda/textract-sender"
-  }
-}
-
-# textract sender queue
-variable "sender_sqs_queue" {
-    type = object({
-      name = string
-      delay_seconds = number
-      max_message_size          = number
-      message_retention_seconds = number
-      visibility_timeout_seconds= number
-    })
-    default = {
-      name                      = "textract-sender-queue"
-      delay_seconds             = 90
-      max_message_size          = 2048
-      message_retention_seconds = 86400
-      visibility_timeout_seconds= 901
-    } 
-}
-
-# textract_receiver_lambda
-variable "textract_receiver_lambda" {
-  type = object({
-    name    = string
-    description = string
-    handler = string
-    runtime = string
-    timeout = number
-    memory_size = number
-    ephemeral_storage_size = number
-    architectures = list(string)
-    layers = list(string)
-    environment_variables = map(string)
-    source_path = string
-  })
-  default = {
-    name = "textract-receiver"
-    description = "This function receive async request to TEXTRACT service"
-    handler = "index.lambda_handler"
-    runtime = "python3.12"
-    timeout = 60
-    memory_size = 256
-    ephemeral_storage_size = 512
-    architectures = ["x86_64"]
-    layers = []
-    environment_variables = {
-      PROPERTY_FILE_S3_PATH = "doczy-dev-infra-textract/config.properties"
-    }
-    source_path = "../src/lambda/textract-receiver"
-  }
-}
-
-# textract receiver queue 
-variable "receiver_sqs_queue" {
-    type = object({
-      name = string
-      delay_seconds = number
-      max_message_size          = number
-      message_retention_seconds = number
-      visibility_timeout_seconds= number
-    })
-    default = {
-      name                      = "textract-receiver-queue"
-      delay_seconds             = 90
-      max_message_size          = 2048
-      message_retention_seconds = 86400
-      visibility_timeout_seconds= 901
-    } 
-}
@@ -1,25 +0,0 @@
-# Output variable definitions
-
-output "sender_lambda_arn" {
-  description = "Sender lambda arn"
-  value = [ for output_list in module.textract_pipeline : output_list.sender_lambda_arn]
-}
-
-output "receiver_lambda_arn" {
-  description = "Receiver lambda arn"
-  value = [ for output_list in module.textract_pipeline : output_list.receiver_lambda_arn]
-}
-
-output "sender_queue_arn" {
-  description = "Sender queue arn"
-  value = [ for output_list in module.textract_pipeline : output_list.sender_queue_arn]
-}
-
-output "receiver_queue_arn" {
-  description = "Receiver queue arn"
-  value = [ for output_list in module.textract_pipeline : output_list.receiver_queue_arn]
-}
-
-output "textract_notification_sns_arn" {
-    value = [ for output_list in module.textract_pipeline : output_list.textract_notification_sns_arn]
-}
@@ -1,37 +0,0 @@
-variable "aws_profile" {
-  type = string
-  # default = "doczyai"
-}
-
-variable "aws_region" {
-  type = string
-  default = "us-east-2"
-}
-
-variable "project_name" {
-  type = string
-  default = "doczyai"
-}
-
-variable "environment" {
-  type = string
-  default = "dev"
-}
-
-variable "client_list" {
-  type = map(object({
-    client_name = string
-    s3_bucket_name = string
-  }))
-  default = {
-    client1 = {
-      s3_bucket_name  = "doczy-dev-infra-textract"
-      client_name = "cn1"
-    },
-    client2 = {
-      s3_bucket_name  = "doczy-dev-infra-textract"
-      client_name = "cn2"
-    }
-    
-  }
-}
@@ -0,0 +1,227 @@
+from terminal import run_command, prepare_logger
+
+import os
+import argparse
+import sys
+import subprocess
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+script_dir = os.path.dirname(os.path.abspath(__file__))
+
+LOGGER = prepare_logger()
+
+LOGGER.info(sys.argv)
+
+parser = argparse.ArgumentParser(description='Run Terraform init with dynamic backend configuration.')
+parser.add_argument('terraform_command', type=str, choices=['plan', 'apply'],
+                    help='The Terraform command to execute (plan, apply, init).')
+parser.add_argument('--environment', type=str, required=True, help='The environment name i.e. dev, prod.')
+parser.add_argument('--module', type=str, required=False, help='The module name.')
+parser.add_argument('--init-extra-args', type=str, required=False, help='Terraform init extra arguments')
+parser.add_argument('--plan-extra-args', type=str, required=False, help='Terraform plan extra arguments')
+parser.add_argument('--apply-extra-args', type=str, required=False, help='Terraform plan extra arguments')
+parser.add_argument('--threads', type=int, default=4, help='Number of threads to use (default: 4)')
+
+args = parser.parse_args()
+
+raw_terraform_command = args.terraform_command
+environment = args.environment
+init_extra_args = args.init_extra_args
+plan_extra_args = args.plan_extra_args
+apply_extra_args = args.apply_extra_args
+terraform_threads = args.threads
+module_name = args.module
+
+bitbucket_ci = os.getenv('BITBUCKET_CI')
+
+
+def change_dir(path):
+    if path:
+        _, git_root = run_command('git rev-parse --show-toplevel', log_cmd=True, log_output=True)
+        LOGGER.info(f"git_root={git_root}")
+        absolute_path = os.path.join(git_root[0], path)
+        return absolute_path
+        # os.chdir(absolute_path)
+    else:
+        LOGGER.error("TF_ROOT_MODULE_PATH environment variable is not set.")
+        sys.exit(1)
+
+
+def select_vars_file():
+    tf_vars_file = os.getenv('TFVARS_FILE_PATH', f'vars-{environment}.tfvars')
+    if tf_vars_file:
+        LOGGER.info(f"Using {tf_vars_file} as vars-file...")
+    else:
+        LOGGER.error("TFVARS_FILE_PATH variable is not set.")
+        sys.exit(1)
+    return tf_vars_file
+
+
+def prepare_tf_init_command(stack_path, init_extra_args, state_environment, tf_state_bucket_name):
+    command = (f'terraform init --reconfigure '
+               f'-backend-config="dynamodb_table=doczyai-use2-{get_environment_value(environment)}-infra-dyd-terraform-lock" '
+               f'-backend-config="bucket=doczyai-use2-{get_environment_value(environment)}-infra-s3-terraform-state" '
+               f'-backend-config="key=terraform/{stack_path}/terraform.tfstate" -backend-config="region=us-east-2"')
+    if init_extra_args:
+        command = f"{command} {init_extra_args}"
+    return command
+
+
+def get_environment_value(environment):
+    environment_map = {
+        'prod': 'p',
+        'uat': 'u',
+        'qa': 'q',
+        'dev': 'd'
+    }
+
+    return environment_map.get(environment, None)  # Returns None if the environment is not found
+
+
+def prepare_tf_main_command(environment, stack_path, state_environment, tf_state_bucket_name, vars_file,
+                            plan_extra_args,
+                            apply_extra_args):
+    common_command = f'-var-file=vars-{environment}.tfvars -var "aws_region=us-east-2" -var "environment={environment}"'
+    if raw_terraform_command == "plan":
+        path_part = stack_path.split('/')[1] if '/' in stack_path else stack_path
+        command = f"terraform plan -out .{path_part}.{environment}.tfplan {common_command}"
+        if plan_extra_args:
+            command = f"{command} {plan_extra_args}"
+    elif raw_terraform_command == "apply":
+        command = f"terraform apply -auto-approve {common_command}"
+        if apply_extra_args:
+            command = f"{command} {apply_extra_args}"
+    return command
+
+
+tf_root_module_path = os.getenv('TF_ROOT_MODULE_PATH') if not module_name else module_name
+
+
+def do_terraform(stack_path):
+    LOGGER.info(f"stack_path={stack_path}")
+    absolute_path = change_dir(stack_path)
+    # TF INIT
+    state_environment = "dev-new" if environment == "dev" else environment
+    tf_state_bucket_name = f"pi2new-{state_environment}-terraform-state-file"
+
+    terraform_init_command = prepare_tf_init_command(stack_path, init_extra_args, state_environment,
+                                                     tf_state_bucket_name)
+    rc_init, init_result = run_command(terraform_init_command, log_output=True, log_cmd=True, log_prefix=stack_path,
+                                       cwd=absolute_path)
+
+
+    if rc_init > 0:
+        raise Exception(f"{stack_path}: Failed to run terraform init")
+
+    vars_file = select_vars_file()
+    terraform_main_command = prepare_tf_main_command(environment, stack_path, state_environment, tf_state_bucket_name,
+                                                     vars_file, plan_extra_args, apply_extra_args)
+
+    rc_main, command_result = run_command(terraform_main_command, log_output=True, log_cmd=True,
+                                          log_prefix=stack_path, cwd=absolute_path)
+    if rc_init or rc_main > 0:
+        raise RuntimeError(f"Unexpected Terraform error for command: {terraform_main_command}")
+    else:
+        return True, "OK"
+
+    # return True, "OK"
+
+
+def find_paths_with_file(file_name):
+    try:
+        completed_process = subprocess.run(['git', 'ls-files', f'*{file_name}'], check=True, text=True,
+                                           stdout=subprocess.PIPE)
+
+        paths = set()
+        for file_path in completed_process.stdout.splitlines():
+            dir_path = '/'.join(file_path.split('/')[:-1])
+            if dir_path.startswith("modules/"):
+                continue
+            if dir_path.startswith("infra/"):
+                dir_path = dir_path[len("infra/"):]
+            if dir_path.startswith("kinesis") or dir_path.startswith("aws-transfer-sftp"):
+                paths.add(dir_path)
+
+        return paths
+    except subprocess.CalledProcessError as e:
+        LOGGER.error(f"Error executing git command: {e}")
+        return set()
+
+
+def read_dependencies(path):
+    depends_on_file = os.path.join(path, ".depends-on.txt")
+    if os.path.exists(depends_on_file):
+        with open(depends_on_file, "r") as f:
+            dependencies = [line.strip() for line in f.readlines()]
+        return dependencies
+    return []
+
+
+def wait_for_dependencies(dependencies, completed, failed, path):
+    """
+    Waits for all dependencies to be satisfied before returning.
+    Now also checks for failed dependencies and raises an exception if found.
+    """
+    while dependencies:
+        pending_dependencies = [dep for dep in dependencies if dep not in completed]
+        if any(dep in failed for dep in pending_dependencies):
+            failed_deps = [dep for dep in pending_dependencies if dep in failed]
+            raise Exception(f"{path}: Cannot proceed. Dependencies failed: {failed_deps}")
+        if not pending_dependencies:
+            return
+        LOGGER.warning(f"{path}: Waiting for dependencies to complete: {pending_dependencies}")
+        time.sleep(5)
+
+
+def execute_with_dependencies(paths):
+    completed = set()  # Track completed paths
+    errored_modules = set()  # Track paths of modules where an error occurred
+    failed = {}  # Track failures
+
+    with ThreadPoolExecutor(max_workers=terraform_threads) as executor:
+        future_to_path = {}
+
+        for path in paths:
+            dependencies = read_dependencies(path)
+            if dependencies:
+                # Submit a future for waiting dependencies
+                dep_future = executor.submit(wait_for_dependencies, dependencies, completed, failed, path)
+                # Submit the main task to be executed after dependencies are resolved
+                future = executor.submit(lambda p=dep_future, x=path: do_terraform(x) if p.result() is None else None)
+            else:
+                future = executor.submit(do_terraform, path)
+            future_to_path[future] = path
+
+        for future in as_completed(future_to_path):
+            path = future_to_path[future]
+            try:
+                result = future.result()
+                if result is None or not result[0]:  # Assuming do_terraform returns (success: bool, message: str)
+                    errored_modules.add(path)
+                    failed[path] = "Failed due to internal error"  # Mark as failed
+                else:
+                    completed.add(path)  # Mark as completed
+                LOGGER.info(f"Thread result for {path}: {result}")
+            except Exception as exc:
+                errored_modules.add(path)
+                failed[path] = str(exc)  # Mark as failed
+                LOGGER.error(f"{path} generated an exception: {exc}")
+
+    # Check if any errors occurred and print the errored module paths
+    if errored_modules:
+        LOGGER.error("One or more modules failed to process correctly.")
+        for module_path in errored_modules:
+            LOGGER.error(f"Module with error: {module_path}")
+        sys.exit(1)
+
+
+# Your main execution logic here
+if __name__ == "__main__":
+    if tf_root_module_path:
+        do_terraform(tf_root_module_path)
+    else:
+        paths = find_paths_with_file('provider.tf')
+        for path in paths:
+            LOGGER.warning(f"Discovered: {path}")
+        execute_with_dependencies(paths)
@@ -1,3 +0,0 @@
-# AWS Lambda handler function
-def lambda_handler(event, context):
-    print("Receiver Lambda")
@@ -1,3 +0,0 @@
-# AWS Lambda handler function
-def lambda_handler(event, context):
-    print("Sender Lambda")
@@ -13,12 +13,12 @@ sqs = boto3.client('sqs')
 # Initialize S3 & Textract clients
 s3_client = boto3.client('s3')

-def lambda_handler(event, context):

+def lambda_handler(event, context):
    try:
        # Read environment variables
        property_file_path = os.environ.get('PROPERTY_FILE_S3_PATH', '')
-        
+
        # Read config.properties
        file_path_array = property_file_path.split("/")

@@ -36,14 +36,14 @@ def lambda_handler(event, context):
            for record in event['Records']:
                # Extract the message body from the record
                record_body = json.loads(record['body'])
-                #logger.info('Message Count: ', str(len(record_body['Records'])) )
+                # logger.info('Message Count: ', str(len(record_body['Records'])) )
                for sqs_record in record_body['Records']:
                    # Construct the source paths
                    source_path = unquote_plus(sqs_record['s3']['object']['key'])
                    document_id = get_filename_from_path(source_path)
                    document_id = os.path.splitext(document_id)[0]
-                    
-                    prompt_config_query = "SELECT DISTINCT PC.FIELD_NAME,LENGTH(PC.PROMPT) as PROMPT_LENGTH, PC.FM_MODEL_ID,PC.GROUP_ID FROM PROMPT_CONFIG PC JOIN DOCUMENT_LOGS DL ON DL.GROUP_ID LIKE '%'||PC.GROUP_ID ||'%' AND DL.DOCUMENT_ID = '"+document_id+"' ORDER BY PROMPT_LENGTH"
+
+                    prompt_config_query = "SELECT DISTINCT PC.FIELD_NAME,LENGTH(PC.PROMPT) as PROMPT_LENGTH, PC.FM_MODEL_ID,PC.GROUP_ID FROM PROMPT_CONFIG PC JOIN DOCUMENT_LOGS DL ON DL.GROUP_ID LIKE '%'||PC.GROUP_ID ||'%' AND DL.DOCUMENT_ID = '" + document_id + "' ORDER BY PROMPT_LENGTH"
                    field_Prompt_group_response = call_snowflake_db_select_lambda(prompt_config_query)

                    # Maximum prompt length per group
@@ -56,7 +56,11 @@ def lambda_handler(event, context):
                    for i, (key, batch_list) in enumerate(batches.items(), 1):
                        group_id, fm_model_id = key
                        for j, batch in enumerate(batch_list, 1):
-                            send_batch_to_sqs(batch,"https://sqs.us-east-2.amazonaws.com/660131068782/doczy-prompt-processor-sqs-dev",S3_BUCKET_NAME+"/"+source_path,i)
+                            AWS_REGION_PARAM = os.getenv('AWS_REGION_PARAM')
+                            AWS_ACCOUNT_ID = os.getenv('AWS_ACCOUNT_ID')
+                            ENVIRONMENT = os.getenv('ENVIRONMENT')
+                            sqs_url = f"https://sqs.{AWS_REGION_PARAM}.amazonaws.com/{AWS_ACCOUNT_ID}/doczy-prompt-processor-sqs-{ENVIRONMENT}"
+                            send_batch_to_sqs(batch, sqs_url, S3_BUCKET_NAME + "/" + source_path, i)
                    return {
                        'statusCode': 200,
                        'body': json.dumps('Message sent to SQS queue')
@@ -68,12 +72,12 @@ def lambda_handler(event, context):
        return {
            'statusCode': 500,
            'body': error_message
-        }        
-                
+        }
+
+    # Function to retrieve configuration values from S3
+

-# Function to retrieve configuration values from S3
 def load_config_from_s3(bucket_name, file_key):
-    
    # Download the config file from S3
    response = s3_client.get_object(Bucket=bucket_name, Key=file_key)
    config_content = response['Body'].read().decode('utf-8')
@@ -89,12 +93,13 @@ def load_config_from_s3(bucket_name, file_key):

    return config_dict

+
 def get_filename_from_path(full_path):
    return os.path.basename(full_path)

+
 # Function to get s3 object tags
 def get_s3_object_tags(bucket_name, object_key):
-
    try:
        # Get object tags
        response = s3_client.get_object_tagging(
@@ -110,15 +115,15 @@ def get_s3_object_tags(bucket_name, object_key):
    except Exception as e:
        logger.exception(f"Error: {e}")
        return None
-    
+

 def call_snowflake_db_select_lambda(query):
    # Create a Lambda client
    lambda_client = boto3.client('lambda')
-    
+
    # Specify your Lambda function name
    lambda_function_name = 'doczy-dev-get-snowflake-logs'
-    
+
    # Prepare payload
    payload = {
        'query': query
@@ -130,14 +135,15 @@ def call_snowflake_db_select_lambda(query):
            InvocationType='RequestResponse',  # Synchronous invocation
            Payload=json.dumps(payload)
        )
-        
+
        # Return response body
        response_body = response['Payload'].read().decode('utf-8')
        return response_body
    except Exception as e:
        logger.exception(f"Error: {e}")
        return None
-    
+
+
 def extract_group_id(json_string):
    try:
        logger.info(f"json_string: {json_string}")
@@ -146,13 +152,14 @@ def extract_group_id(json_string):
        group_id = body[0]['GROUP_ID']
        group_ids = group_id.split(',')
        group_ids_quoted = ["'" + group_id.strip() + "'" for group_id in group_ids]
-    
+
        # Join the elements back together with commas
        return ','.join(group_ids_quoted)
    except (json.JSONDecodeError, KeyError, IndexError) as e:
        logger.error("Error parsing JSON or extracting GROUP_ID:", e)
        return None
-    
+
+
 def extract_body_tags(json_data):
    try:
        # Parse the JSON string
@@ -167,14 +174,13 @@ def extract_body_tags(json_data):

        logger.info(f"tags_list: {tags_list}")
        return tags_list
-    
+
    except (json.JSONDecodeError, KeyError) as e:
        logger.error("Error parsing JSON or extracting 'body' tag:", e)
        return None


 def divide_into_batches(data, max_prompt_length_per_group):
-
    try:
        # Parse the input data
        data_dict = json.loads(data)
@@ -189,7 +195,7 @@ def divide_into_batches(data, max_prompt_length_per_group):

        # Divide data into batches based on group_id and fm_model_id
        for item in body_dict:
-            
+
            group_id = item["GROUP_ID"]
            fm_model_id = item["FM_MODEL_ID"]
            prompt_length = item["PROMPT_LENGTH"]
@@ -212,8 +218,7 @@ def divide_into_batches(data, max_prompt_length_per_group):

            # Retrieve the current batch (last sub-batch)
            current_batch = batches[(group_id, fm_model_id)][-1]
-            
-            
+
            # Append the current item to the current batch
            current_batch.append(item)
            # Update prompt length for the current group and model id
@@ -227,16 +232,16 @@ def divide_into_batches(data, max_prompt_length_per_group):
        raise e


-def send_batch_to_sqs(batch, queue_url,source_path,prompt_batch_id):
+def send_batch_to_sqs(batch, queue_url, source_path, prompt_batch_id):
    # Initialize SQS client
    sqs = boto3.client('sqs')

    # Convert batch to JSON string
-    batch_json = json.dumps(batch) 
+    batch_json = json.dumps(batch)
    try:
        result = {"field_list": batch_json,
-                    "file_name": source_path,
-                    "prompt_batch_id" : prompt_batch_id
+                  "file_name": source_path,
+                  "prompt_batch_id": prompt_batch_id
                  }
        logger.info(f"Message sent to SQS queue: {json.dumps(result)}")
        # Send message to SQS queue
@@ -250,4 +255,4 @@ def send_batch_to_sqs(batch, queue_url,source_path,prompt_batch_id):

    except Exception as e:
        logger.error('Error sending message to SQS queue:', e)
-        return False
+        return False
@@ -40,9 +40,9 @@ locals {

    uat = {
      vpc_id = "vpc-0392396d0e7bdd77f"
-      secret_manager_name = ""
-      devops_s3_bucket_name = ""
-      api_access_required_roles_arns = []
+      secret_manager_name = "doczy-dev-db-svc-acc"
+      devops_s3_bucket_name = "doczyai-use2-u-infra-s3-devops-resources"
+      api_access_required_roles_arns = ["arn:aws:iam::975049960860:role/doczyai-use2-u-rol-streamlit-server"]
    }
  }

@@ -54,7 +54,7 @@ locals {

  # global prefix
  global_prefix = "${var.project_name}-${local.region_short_name}-${local.environment_short_name}-infra"
-  
+
  # Resource prefix
  iam_role_prefix = "${local.global_prefix}-rol"

@@ -68,10 +68,6 @@ locals {
  }
 }
 provider "aws" {
-  profile = var.aws_profile
-
-  # access_key = var.access_key
-  # secret_key = var.secret_key

  default_tags {
    tags = local.common_tags
@@ -86,6 +82,8 @@ module "textract_pipeline" {

  client_name = each.value.client_name

+  aws_account_id = var.aws_account_id
+
  project_name        = var.project_name
  aws_region          = var.aws_region
  environment         = var.environment
@@ -99,6 +97,8 @@ module "textract_pipeline" {

  textract_role_name = aws_iam_role.textract_role.name
  lambda_role_name = aws_iam_role.lambda_role.name
+
+  text_creation_lambda_additional_layer_arn = var.text_creation_lambda_additional_layer_arn
 }

 # Create lambda_role
@@ -25,12 +25,12 @@ locals {

  # global prefix
  global_prefix = "${var.project_name}-${local.region_short_name}-${local.environment_short_name}-${var.client_name}"
-  
+
  # Resource prefix
  lambda_prefix = "${local.global_prefix}-lmb"
  api_gateway_prefix = "${local.global_prefix}-api"
  lambda_layer_prefix = "${local.global_prefix}-lyr"
-  
+
  # Lambda names
  s3_folder_details_lambda_name = "${local.lambda_prefix}-${var.s3_folder_detail_lambda.name}"
  batch_creation_lambda_name = "${local.lambda_prefix}-${var.batch_creation_lambda.name}"
@@ -42,7 +42,7 @@ locals {
  sender_lambda_name = "${local.lambda_prefix}-${var.textract_sender_lambda.name}"
  receiver_lambda_name = "${local.lambda_prefix}-${var.textract_receiver_lambda.name}"
  text_creation_lambda_name = "${local.lambda_prefix}-${var.text_creation_lambda.name}"
-  prompt_orchestrator_lambda_name = "${local.lambda_prefix}-${var.prompt_orchestrator_lambda.name}"
+  prompt_orchestrator_lambda_name = "${local.lambda_prefix}-${local.prompt_orchestrator_lambda.name}"
  prompt_processor_lambda_name = "${local.lambda_prefix}-${var.prompt_processor_lambda.name}"
  call_bedrock_lambda_name = "${local.lambda_prefix}-${var.call_bedrock_lambda.name}"
  database_interface_lambda_name = "${local.lambda_prefix}-${var.database_interface_lambda.name}"
@@ -183,7 +183,7 @@ module "s3_folder_detail_lambda_function" {
  memory_size   = var.s3_folder_detail_lambda.memory_size
  ephemeral_storage_size  = var.s3_folder_detail_lambda.ephemeral_storage_size
  architectures = var.s3_folder_detail_lambda.architectures
-  
+
  environment_variables = {}

  source_path   = var.s3_folder_detail_lambda.source_path
@@ -210,7 +210,7 @@ module "batch_creation_lambda_function" {
  memory_size   = var.batch_creation_lambda.memory_size
  ephemeral_storage_size  = var.batch_creation_lambda.ephemeral_storage_size
  architectures = var.batch_creation_lambda.architectures
-  
+
  environment_variables = local.environment_variables

  source_path   = var.batch_creation_lambda.source_path
@@ -237,7 +237,7 @@ module "trigger_pipeline_lambda_function" {
  memory_size   = var.trigger_pipeline_lambda.memory_size
  ephemeral_storage_size  = var.trigger_pipeline_lambda.ephemeral_storage_size
  architectures = var.trigger_pipeline_lambda.architectures
-  
+
  environment_variables = merge(local.environment_variables,{
    INITIATE_DB_SQS_URL = var.insert_record_queue_url
  })
@@ -266,7 +266,7 @@ module "insert_record_lambda_function" {
  memory_size   = var.insert_record_lambda.memory_size
  ephemeral_storage_size  = var.insert_record_lambda.ephemeral_storage_size
  architectures = var.insert_record_lambda.architectures
-  
+
  environment_variables = local.environment_variables

  source_path   = var.insert_record_lambda.source_path
@@ -277,7 +277,7 @@ module "insert_record_lambda_function" {
      function_response_types = ["ReportBatchItemFailures"]
      scaling_config = {
        maximum_concurrency = 20
-        
+
      }
      batch_size = 1
    }
@@ -310,7 +310,7 @@ module "docx_to_pdf_lambda_function" {
    module.lambda_layer_libre_office.lambda_layer_arn,
    module.lambda_layer_brotli.lambda_layer_arn,
  ]
-  
+
  environment_variables = local.environment_variables

  source_path   = var.docx_to_pdf_lambda.source_path
@@ -321,7 +321,7 @@ module "docx_to_pdf_lambda_function" {
      function_response_types = ["ReportBatchItemFailures"]
      scaling_config = {
        maximum_concurrency = 20
-        
+
      }
      batch_size = 1
    }
@@ -349,7 +349,7 @@ module "tiff_to_pdf_lambda_function" {
  memory_size   = var.tiff_to_pdf_lambda.memory_size
  ephemeral_storage_size  = var.tiff_to_pdf_lambda.ephemeral_storage_size
  architectures = var.tiff_to_pdf_lambda.architectures
-  
+
  environment_variables = local.environment_variables

  source_path   = var.tiff_to_pdf_lambda.source_path
@@ -360,7 +360,7 @@ module "tiff_to_pdf_lambda_function" {
      function_response_types = ["ReportBatchItemFailures"]
      scaling_config = {
        maximum_concurrency = 20
-        
+
      }
      batch_size = 1
    }
@@ -388,7 +388,7 @@ module "pdf_validation_lambda_function" {
  memory_size   = var.pdf_validation_lambda.memory_size
  ephemeral_storage_size  = var.pdf_validation_lambda.ephemeral_storage_size
  architectures = var.pdf_validation_lambda.architectures
-  
+
  layers = [
    module.lambda_layer_pypdf2.lambda_layer_arn
  ]
@@ -403,7 +403,7 @@ module "pdf_validation_lambda_function" {
      function_response_types = ["ReportBatchItemFailures"]
      scaling_config = {
        maximum_concurrency = 20
-        
+
      }
      batch_size = 1
    }
@@ -431,7 +431,7 @@ module "textract_sender_lambda_function" {
  memory_size   = var.textract_sender_lambda.memory_size
  ephemeral_storage_size  = var.textract_sender_lambda.ephemeral_storage_size
  architectures = var.textract_sender_lambda.architectures
-  
+
  environment_variables = merge(local.environment_variables,{
    TEXTRACT_ROLE_ARN = var.textract_role_arn
    SNS_TOPIC_ARN = var.textract_sns_topic_arn
@@ -445,7 +445,7 @@ module "textract_sender_lambda_function" {
      function_response_types = ["ReportBatchItemFailures"]
      scaling_config = {
        maximum_concurrency = 20
-        
+
      }
      batch_size = 1
    }
@@ -473,7 +473,7 @@ module "textract_receiver_lambda_function" {
  memory_size   = var.textract_receiver_lambda.memory_size
  ephemeral_storage_size  = var.textract_receiver_lambda.ephemeral_storage_size
  architectures = var.textract_receiver_lambda.architectures
-  
+
  environment_variables = local.environment_variables

  source_path   = var.textract_receiver_lambda.source_path
@@ -484,7 +484,7 @@ module "textract_receiver_lambda_function" {
      function_response_types = ["ReportBatchItemFailures"]
      scaling_config = {
        maximum_concurrency = 20
-        
+
      }
      batch_size = 1
    }
@@ -514,9 +514,10 @@ module "text_creation_lambda_function" {
  architectures = var.text_creation_lambda.architectures

  layers = [
-    module.lambda_layer_pypdf2.lambda_layer_arn
+    module.lambda_layer_pypdf2.lambda_layer_arn,
+    var.text_creation_lambda_additional_layer_arn
  ]
-  
+
  environment_variables = local.environment_variables

  source_path   = var.text_creation_lambda.source_path
@@ -527,7 +528,7 @@ module "text_creation_lambda_function" {
      function_response_types = ["ReportBatchItemFailures"]
      scaling_config = {
        maximum_concurrency = 20
-        
+
      }
      batch_size = 1
    }
@@ -548,22 +549,22 @@ module "prompt_orchestrator_lambda_function" {
  version = "7.2.2"

  function_name = local.prompt_orchestrator_lambda_name
-  description   = var.prompt_orchestrator_lambda.description
-  handler       = var.prompt_orchestrator_lambda.handler
-  runtime       = var.prompt_orchestrator_lambda.runtime
-  timeout       = var.prompt_orchestrator_lambda.timeout
-  memory_size   = var.prompt_orchestrator_lambda.memory_size
-  ephemeral_storage_size  = var.prompt_orchestrator_lambda.ephemeral_storage_size
-  architectures = var.prompt_orchestrator_lambda.architectures
+  description   = local.prompt_orchestrator_lambda.description
+  handler       = local.prompt_orchestrator_lambda.handler
+  runtime       = local.prompt_orchestrator_lambda.runtime
+  timeout       = local.prompt_orchestrator_lambda.timeout
+  memory_size   = local.prompt_orchestrator_lambda.memory_size
+  ephemeral_storage_size  = local.prompt_orchestrator_lambda.ephemeral_storage_size
+  architectures = local.prompt_orchestrator_lambda.architectures

  layers = []
-  
-  environment_variables = merge(local.environment_variables,{
+
+  environment_variables = merge(local.environment_variables, local.prompt_orchestrator_lambda.environment_variables, {
    PROMPT_PROCESSOR_QUEUE_URL = var.prompt_processor_queue_url
    MAX_PROMPT_LENGTH_PER_GROUP = "5000"
  })

-  source_path   = var.prompt_orchestrator_lambda.source_path
+  source_path   = local.prompt_orchestrator_lambda.source_path

  event_source_mapping = {
    sqs = {
@@ -571,7 +572,7 @@ module "prompt_orchestrator_lambda_function" {
      function_response_types = ["ReportBatchItemFailures"]
      scaling_config = {
        maximum_concurrency = 20
-        
+
      }
      batch_size = 1
      enabled = false
@@ -602,7 +603,7 @@ module "prompt_processor_lambda_function" {
  architectures = var.prompt_processor_lambda.architectures

  layers = []
-  
+
  environment_variables = merge(local.environment_variables,{
    CALL_BEDROCK_LAMBDA_FUNCTION_NAME = local.call_bedrock_lambda_name
  })
@@ -615,7 +616,7 @@ module "prompt_processor_lambda_function" {
      function_response_types = ["ReportBatchItemFailures"]
      scaling_config = {
        maximum_concurrency = 20
-        
+
      }
      batch_size = 1
    }
@@ -645,7 +646,7 @@ module "call_bedrock_lambda_function" {
  architectures = var.call_bedrock_lambda.architectures

  layers = []
-  
+
  environment_variables = local.environment_variables

  source_path   = var.call_bedrock_lambda.source_path
@@ -672,7 +673,7 @@ module "database_interface_lambda_function" {
  memory_size   = var.database_interface_lambda.memory_size
  ephemeral_storage_size  = var.database_interface_lambda.ephemeral_storage_size
  architectures = var.database_interface_lambda.architectures
-  
+
  environment_variables = merge(local.environment_variables,{
    SECRET_MANAGER_NAME = var.secret_manager_name
  })
@@ -705,7 +706,7 @@ module "database_interface_get_lambda_function" {
  memory_size   = var.database_interface_get_lambda.memory_size
  ephemeral_storage_size  = var.database_interface_get_lambda.ephemeral_storage_size
  architectures = var.database_interface_get_lambda.architectures
-  
+
  environment_variables = merge(local.environment_variables,{
    SECRET_MANAGER_NAME = var.secret_manager_name
  })
@@ -775,7 +776,7 @@ resource "aws_s3_bucket_notification" "bucket_notification" {
    events        = ["s3:ObjectCreated:*"]
    filter_prefix = "textract-output-json/"
    filter_suffix = ".json"
-  }  
+  }

   queue {
    id            = "text-file-upload-event"
@@ -847,9 +848,9 @@ resource "aws_api_gateway_method" "create_batch_method" {
  http_method   = "POST"
  authorization = "AWS_IAM"

-  depends_on = [ 
-    aws_api_gateway_rest_api.project_api, 
-    aws_api_gateway_resource.create_batch 
+  depends_on = [
+    aws_api_gateway_rest_api.project_api,
+    aws_api_gateway_resource.create_batch
  ]
 }

@@ -863,8 +864,8 @@ resource "aws_api_gateway_method_response" "create_batch_method_response_200" {
    "application/json" = "Empty"
  }

-  depends_on = [ 
-    aws_api_gateway_rest_api.project_api, 
+  depends_on = [
+    aws_api_gateway_rest_api.project_api,
    aws_api_gateway_resource.create_batch,
    aws_api_gateway_method.create_batch_method
  ]
@@ -879,8 +880,8 @@ resource "aws_api_gateway_integration" "create_batch_integration" {
  type                    = "AWS"
  uri                     = module.batch_creation_lambda_function.lambda_function_invoke_arn

-  depends_on = [ 
-    aws_api_gateway_rest_api.project_api, 
+  depends_on = [
+    aws_api_gateway_rest_api.project_api,
    aws_api_gateway_resource.create_batch,
    aws_api_gateway_method.create_batch_method,
    module.batch_creation_lambda_function
@@ -894,8 +895,8 @@ resource "aws_api_gateway_integration_response" "create_batch_integration_respon
  http_method = aws_api_gateway_method.create_batch_method.http_method
  status_code = aws_api_gateway_method_response.create_batch_method_response_200.status_code

-  depends_on = [ 
-    aws_api_gateway_rest_api.project_api, 
+  depends_on = [
+    aws_api_gateway_rest_api.project_api,
    aws_api_gateway_resource.create_batch,
    aws_api_gateway_method.create_batch_method,
    aws_api_gateway_method_response.create_batch_method_response_200,
@@ -912,8 +913,8 @@ resource "aws_lambda_permission" "create_batch_lambda_permission" {
  principal     = "apigateway.amazonaws.com"
  source_arn = "${aws_api_gateway_rest_api.project_api.execution_arn}/*/${aws_api_gateway_method.create_batch_method.http_method}${aws_api_gateway_resource.create_batch.path}"

-  depends_on = [ 
-    aws_api_gateway_rest_api.project_api, 
+  depends_on = [
+    aws_api_gateway_rest_api.project_api,
    aws_api_gateway_resource.create_batch,
    aws_api_gateway_method.create_batch_method,
    module.batch_creation_lambda_function
@@ -963,8 +964,8 @@ resource "aws_api_gateway_integration_response" "s3_folder_detail_integration_re
  http_method = aws_api_gateway_method.s3_folder_detail_method.http_method
  status_code = aws_api_gateway_method_response.s3_folder_detail_method_response_200.status_code

-  depends_on = [ 
-    aws_api_gateway_rest_api.project_api, 
+  depends_on = [
+    aws_api_gateway_rest_api.project_api,
    aws_api_gateway_resource.s3_folder_detail,
    aws_api_gateway_method.s3_folder_detail_method,
    aws_api_gateway_method_response.s3_folder_detail_method_response_200,
@@ -1023,8 +1024,8 @@ resource "aws_api_gateway_integration_response" "trigger_pipeline_integration_re
  http_method = aws_api_gateway_method.trigger_pipeline_method.http_method
  status_code = aws_api_gateway_method_response.trigger_pipeline_method_response_200.status_code

-  depends_on = [ 
-    aws_api_gateway_rest_api.project_api, 
+  depends_on = [
+    aws_api_gateway_rest_api.project_api,
    aws_api_gateway_resource.trigger_pipeline,
    aws_api_gateway_method.trigger_pipeline_method,
    aws_api_gateway_method_response.trigger_pipeline_method_response_200,
@@ -18,6 +18,11 @@ variable "aws_region" {
  type = string
 }

+# Required
+variable "aws_account_id" {
+  type = string
+}
+
 # Required
 variable "vpc_id" {
  type = string
@@ -480,22 +485,15 @@ variable "text_creation_lambda" {
  }
 }

+variable "text_creation_lambda_additional_layer_arn" {
+  type = string
+}
+
 # prompt-orchestrator lambda
-variable "prompt_orchestrator_lambda" {
-  type = object({
-    name    = string
-    description = string
-    handler = string
-    runtime = string
-    timeout = number
-    memory_size = number
-    ephemeral_storage_size = number
-    architectures = list(string)
-    layers = list(string)
-    environment_variables = map(string)
-    source_path = string
-  })
-  default = {
+
+# Use locals to merge default values with variable values
+locals {
+  prompt_orchestrator_lambda_default = {
    name = "prompt-orchestrator"
    description = "This Lambda functions breaks prompts into chunks and send messages into SQS"
    handler = "index.lambda_handler"
@@ -505,9 +503,15 @@ variable "prompt_orchestrator_lambda" {
    ephemeral_storage_size = 512
    architectures = ["x86_64"]
    layers = []
-    environment_variables = {}
+    environment_variables = {
+      AWS_REGION_PARAM = var.aws_region
+      AWS_ACCOUNT_ID = var.aws_account_id
+      ENVIRONMENT = var.environment
+    }
    source_path = "../src/lambda/prompt-orchestrator"
  }
+
+  prompt_orchestrator_lambda = merge(local.prompt_orchestrator_lambda_default)
 }

 # prompt-processor lambda
@@ -52,6 +52,7 @@ module "textract_pipeline_part3" {

  project_name = var.project_name
  aws_region   = var.aws_region
+  aws_account_id = var.aws_account_id
  environment  = var.environment
  vpc_id       = var.vpc_id
  secret_manager_name = var.secret_manager_name
@@ -68,6 +69,7 @@ module "textract_pipeline_part3" {
  textract_sender_queue_arn = module.textract_pipeline_part1.sender_queue_arn
  textract_receiver_queue_arn = module.textract_pipeline_part1.receiver_queue_arn
  text_creation_queue_arn = module.textract_pipeline_part1.text_creation_queue_arn
+  text_creation_lambda_additional_layer_arn = var.text_creation_lambda_additional_layer_arn
  prompt_orchestrator_queue_arn = module.textract_pipeline_part1.prompt_orchestrator_queue_arn
  prompt_processor_queue_arn = module.textract_pipeline_part1.prompt_processor_queue_arn
  prompt_processor_queue_url = module.textract_pipeline_part1.prompt_processor_queue_url
@@ -198,7 +200,7 @@ data "aws_iam_policy_document" "lambda_policy" {
    ]

    effect = "Allow"
-    resources = ["arn:aws:secretsmanager:*:660131068782:secret:*"] 
+    resources = ["arn:aws:secretsmanager:*:${var.aws_account_id}:secret:*"]
  }
 }

@@ -56,4 +56,12 @@ variable "textract_role_name" {
 # Required
 variable "lambda_role_name" {
  type = string
+}
+
+variable "aws_account_id" {
+  type = string
+}
+
+variable "text_creation_lambda_additional_layer_arn" {
+  type = string
 }
@@ -1,19 +1,3 @@
-# # required
-# variable "secret_key" {
-#   type    = string
-# }
-
-# # required
-# variable "access_key" {
-#   type    = string
-# }
-
-
-# required
-variable "aws_profile" {
-  type    = string
-  # default = "doczyai"
-}
 # required
 variable "aws_region" {
  type    = string
@@ -40,4 +24,12 @@ variable "client_list" {
      client_name = "cn1"
    }
  }
+}
+
+variable "aws_account_id" {
+  type = string
+}
+
+variable "text_creation_lambda_additional_layer_arn" {
+  type = string
 }
@@ -0,0 +1,2 @@
+aws_account_id                            = 660131068782
+text_creation_lambda_additional_layer_arn = "arn:aws:lambda:us-east-2:336392948345:layer:AWSSDKPandas-Python312:8"
@@ -0,0 +1,3 @@
+aws_account_id                            = 975049960860
+# TODO provide new layer arn for UAT? 336392948345 was used in DEV but not sure what account that is?!
+text_creation_lambda_additional_layer_arn = "arn:aws:lambda:us-east-2:336392948345:layer:AWSSDKPandas-Python312:8"