diff --git a/airflow/dags/pipeline_processed_outputs_dag.py b/airflow/dags/pipeline_processed_outputs_dag.py new file mode 100644 index 0000000..cc41556 --- /dev/null +++ b/airflow/dags/pipeline_processed_outputs_dag.py @@ -0,0 +1,83 @@ + + +from __future__ import annotations + +import os +from datetime import datetime +import logging + +from airflow import DAG +from airflow.providers.snowflake.operators.snowflake import SnowflakeOperator +from airflow.providers.snowflake.hooks.snowflake import SnowflakeHook +from airflow.operators.python import PythonOperator +from airflow.utils.trigger_rule import TriggerRule +from airflow.operators.empty import EmptyOperator +from airflow.models import Variable +from airflow.exceptions import AirflowFailException + + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + + +SNOWFLAKE_CONN_ID = "doczy_dev_snowflake" +DAG_ID = "load_doczy_processed_outputs" +DATABASE="DOCZY_DEV" + +TAGS=["dev","config_interface","dataload"] + +# Trigger rules +ALL_SUCCESS = 'all_success' +ALL_FAILED = 'all_failed' +ALL_DONE = 'all_done' +ONE_SUCCESS = 'one_success' +ONE_FAILED = 'one_failed' + +# Passing empty params for now, this will be overridden by the payload from the trigger +# These params can also be set from the Airflow UI while manually triggering the DAG +# This will be replaced with the payload from the event after API connection is setup +default_params = {"pipeline_processed_output": ""} + +def call_stored_proc(proc_name,file_type, params): + dwh_hook = SnowflakeHook(snowflake_conn_id=SNOWFLAKE_CONN_ID) + with dwh_hook.get_conn() as conn: + # dwh_hook.set_autocommit(conn,autocommit=False) + cur = conn.cursor() + + # Added new parameter file_type to determine the file name to be passed to the stored procedure + if file_type == 'pipeline_processed_output': + file_name = params['pipeline_processed_output'] + + cur.execute(f"CALL {DATABASE}.STG.{proc_name}('{file_name}');") + result = cur.fetchone() + if result[0] == 'Setup, Load, and Audit Complete': + logger.info('PROCEDURE EXECUTED SUCCESSFULLY') + else: + raise AirflowFailException("Check the DAG logs for more information. ERROR FROM SNOWFLAKE: ", result) + logger.info(f"QUERY EXECUTION RESULT: {str(result)}") + +dag = DAG( + DAG_ID, + start_date=datetime(2024, 1, 1), + default_args={"snowflake_conn_id": SNOWFLAKE_CONN_ID, "retries":0}, + tags=TAGS, + catchup=False, + schedule=None, + params = default_params +) + +begin_job = EmptyOperator(task_id='Begin') + + +load_pipeline_processed_output = PythonOperator( + task_id="load_pipeline_processed_output", + python_callable=call_stored_proc, + dag=dag, + op_kwargs={'proc_name':'LOAD_DOCZY_PIPELINE_PROCESSED_OUTPUT', 'file_type':'pipeline_processed_output'} +) + + +end_job = EmptyOperator(task_id='End') + +begin_job >> load_pipeline_processed_output >> end_job diff --git a/airflow/dags/uat_Qa_Dag.py b/airflow/dags/uat_Qa_Dag.py new file mode 100644 index 0000000..bfda702 --- /dev/null +++ b/airflow/dags/uat_Qa_Dag.py @@ -0,0 +1,94 @@ +import logging +from airflow.exceptions import AirflowFailException +from airflow.operators.empty import EmptyOperator +from airflow.operators.python import PythonOperator +from datetime import datetime +from airflow import DAG +from airflow.providers.snowflake.hooks.snowflake import SnowflakeHook +# from openpyxl.workbook import Workbook +import pandas as pd +import boto3 +import io + + +SNOWFLAKE_CONN_ID="doczy_uat_snowflake" +TAGS=["QC","uat","etl","QC-Summery"] +DAG_ID="qc_report_dag" + +bucket = "doczy-dev-infra-mwaa-resources" +object_key = "outputs/" + +DATABASE="DOCZY_UAT" +SCHEMA = "STG" +TABLE_NAME = "TRAINING_DATA_RAW" + +# Trigger rules +ALL_SUCCESS = 'all_success' +ALL_FAILED = 'all_failed' +ALL_DONE = 'all_done' +ONE_SUCCESS = 'one_success' +ONE_FAILED = 'one_failed' + + + + +args = {"owner": "Airflow", "start_date": datetime(2022, 1, 1), "retries":0 } +dag = DAG( + dag_id=DAG_ID, default_args=args, schedule=None, + tags=TAGS +) + +def getData(): + # Setup connection to Snowflake + dwh_hook = SnowflakeHook(snowflake_conn_id=SNOWFLAKE_CONN_ID) + conn = dwh_hook.get_conn() # Get the raw connection + + # Your query and the database details + qc_query = f"SELECT * FROM {DATABASE}.{SCHEMA}.{TABLE_NAME}" + + # Fetch data into a Pandas DataFrame + df = pd.read_sql(qc_query, conn) + + # Initialize S3 client + s3 = boto3.client("s3") + + # Create a buffer to hold the data + with io.StringIO() as csv_buffer: + df.to_csv(csv_buffer, index=False) + + # Save the data to S3 + response = s3.put_object( + Bucket=bucket, Key=object_key+'snowflake_table_results.csv' , Body=csv_buffer.getvalue() + ) + + status = response.get("ResponseMetadata", {}).get("HTTPStatusCode") + + if status == 200: + print(f"Successful S3 put_object response. Status - {status}") + else: + raise AirflowFailException(f"Unsuccessful S3 put_object response. Status - {status}") + + oldData = df + newData = df + + with io.BytesIO() as output: + with pd.ExcelWriter(output, engine='xlsxwriter') as writer: + oldData.to_excel(writer, sheet_name="Old") + newData.to_excel(writer, sheet_name="new") + dat = oldData.compare(newData, align_axis=0, keep_shape=True) + dat.to_excel(writer, sheet_name="qc") + response = s3.put_object( + Bucket=bucket, Key=object_key+'QC_Report.xlsx' , Body=output.getvalue() + ) + + print("Dataframe is written to S3 successfully.") + +with dag: + begin_job = EmptyOperator(task_id='Begin') + + get_data_from_snowflake = PythonOperator(task_id="get_data_from_snowflake", python_callable=getData) + + end_job = EmptyOperator(task_id='End') + + +begin_job >> get_data_from_snowflake >> end_job \ No newline at end of file diff --git a/airflow/dags/uat_config_interface_dag .py b/airflow/dags/uat_config_interface_dag .py new file mode 100644 index 0000000..bcca34f --- /dev/null +++ b/airflow/dags/uat_config_interface_dag .py @@ -0,0 +1,100 @@ + + +from __future__ import annotations + +import os +from datetime import datetime +import logging + +from airflow import DAG +from airflow.providers.snowflake.operators.snowflake import SnowflakeOperator +from airflow.providers.snowflake.hooks.snowflake import SnowflakeHook +from airflow.operators.python import PythonOperator +from airflow.utils.trigger_rule import TriggerRule +from airflow.operators.empty import EmptyOperator +from airflow.models import Variable +from airflow.exceptions import AirflowFailException + + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + + +SNOWFLAKE_CONN_ID = "doczy_uat_snowflake" +DAG_ID = "load_request_and_contract_submissions" +DATABASE="DOCZY_UAT" +# bucket = "airflow-data-ingestion" + +TAGS=["uat","contract_config_interface","dataload"] + +# Trigger rules +ALL_SUCCESS = 'all_success' +ALL_FAILED = 'all_failed' +ALL_DONE = 'all_done' +ONE_SUCCESS = 'one_success' +ONE_FAILED = 'one_failed' + +# Passing empty params for now, this will be overridden by the payload from the trigger +# These params can also be set from the Airflow UI while manually triggering the DAG +default_params = {"request_submission_file_name": "", "contract_config_file_name":""} +# This will be replaced with the payload from the event after API connection is setup +# training_results_file_name = "training_results_sample.csv" +# attempt_logs_file_name = "attempt_logs_sample.csv" + + +def call_stored_proc(proc_name,file_type, params): + dwh_hook = SnowflakeHook(snowflake_conn_id=SNOWFLAKE_CONN_ID) + with dwh_hook.get_conn() as conn: + # dwh_hook.set_autocommit(conn,autocommit=False) + cur = conn.cursor() + + # Added new parameter file_type to determine the file name to be passed to the stored procedure + # The bucket name is set by default to "doczy-dev-infra-raw-data-ingestion" and the files should be ALWAYS save under training_interface/ path for now + if file_type == 'request_submission': + file_name = params['request_submission_file_name'] + elif file_type == 'contract_config': + file_name = params['contract_config_file_name'] + + cur.execute(f"CALL {DATABASE}.STG.{proc_name}('{file_name}');") + result = cur.fetchone() + if result[0] == 'Setup, Load, and Audit Complete': + logger.info('PROCEDURE EXECUTED SUCCESSFULLY') + else: + raise AirflowFailException("Check the DAG logs for more information. ERROR FROM SNOWFLAKE: ", result) + logger.info(f"QUERY EXECUTION RESULT: {str(result)}") + +dag = DAG( + DAG_ID, + start_date=datetime(2024, 1, 1), + default_args={"snowflake_conn_id": SNOWFLAKE_CONN_ID, "retries":0}, + tags=TAGS, + catchup=False, + schedule=None, + params = default_params +) + +begin_job = EmptyOperator(task_id='Begin') + + +load_request_submission = PythonOperator( + task_id="load_request_submissions", + python_callable=call_stored_proc, + dag=dag, + op_kwargs={'proc_name':'LOAD_REQUEST_SUBMISSION', 'file_type':'request_submission'} +) + +load_contract_config = PythonOperator( + task_id="load_contract_config", + python_callable=call_stored_proc, + dag=dag, + op_kwargs={'proc_name':'LOAD_CONTRACT_CONFIG', 'file_type':'contract_config'} +) + + + + +end_job = EmptyOperator(task_id='End') + + +begin_job >> load_request_submission >> load_contract_config >> end_job \ No newline at end of file diff --git a/airflow/dags/uat_raw_training_data_dag.py b/airflow/dags/uat_raw_training_data_dag.py new file mode 100644 index 0000000..c78fda5 --- /dev/null +++ b/airflow/dags/uat_raw_training_data_dag.py @@ -0,0 +1,108 @@ + + +from __future__ import annotations + +import os +from datetime import datetime +import logging + +from airflow import DAG +from airflow.providers.snowflake.operators.snowflake import SnowflakeOperator +from airflow.providers.snowflake.hooks.snowflake import SnowflakeHook +from airflow.operators.python import PythonOperator +from airflow.utils.trigger_rule import TriggerRule +from airflow.operators.empty import EmptyOperator +from airflow.models import Variable +from airflow.exceptions import AirflowFailException + + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + + +SNOWFLAKE_CONN_ID = "doczy_uat_snowflake" +DAG_ID = "load_raw_training_data" +DATABASE="DOCZY_UAT" +# bucket = "airflow-data-ingestion" + +TAGS=["uat","training_data","dataload"] + +# Trigger rules +ALL_SUCCESS = 'all_success' +ALL_FAILED = 'all_failed' +ALL_DONE = 'all_done' +ONE_SUCCESS = 'one_success' +ONE_FAILED = 'one_failed' + +# Passing empty params for now, this will be overridden by the payload from the trigger +# These params can also be set from the Airflow UI while manually triggering the DAG +default_params = {"column_config_file_name": "", "raw_training_data_file_name":"", "business_config_file_name":""} +# This will be replaced with the payload from the event after API connection is setup +# training_results_file_name = "training_results_sample.csv" +# attempt_logs_file_name = "attempt_logs_sample.csv" + + +def call_stored_proc(proc_name,file_type, params): + dwh_hook = SnowflakeHook(snowflake_conn_id=SNOWFLAKE_CONN_ID) + with dwh_hook.get_conn() as conn: + # dwh_hook.set_autocommit(conn,autocommit=False) + cur = conn.cursor() + + # Added new parameter file_type to determine the file name to be passed to the stored procedure + # The bucket name is set by default to "doczy-dev-infra-raw-data-ingestion" and the files should be ALWAYS save under training_interface/ path for now + if file_type == 'column_config': + file_name = params['column_config_file_name'] + elif file_type == 'raw_training_data': + file_name = params['raw_training_data_file_name'] + elif file_type == 'business_config': + file_name = params['business_config_file_name'] + + cur.execute(f"CALL {DATABASE}.STG.{proc_name}('{file_name}');") + result = cur.fetchone() + if result[0] == 'Setup, Load, and Audit Complete': + logger.info('PROCEDURE EXECUTED SUCCESSFULLY') + else: + raise AirflowFailException("Check the DAG logs for more information. ERROR FROM SNOWFLAKE: ", result) + logger.info(f"QUERY EXECUTION RESULT: {str(result)}") + +dag = DAG( + DAG_ID, + start_date=datetime(2024, 1, 1), + default_args={"snowflake_conn_id": SNOWFLAKE_CONN_ID, "retries":0}, + tags=TAGS, + catchup=False, + schedule=None, + params = default_params +) + +begin_job = EmptyOperator(task_id='Begin') + + +load_training_results = PythonOperator( + task_id="load_column_config", + python_callable=call_stored_proc, + dag=dag, + op_kwargs={'proc_name':'LOAD_COLUMN_CONFIG', 'file_type':'column_config'} +) + +load_attempt_logs_sp = PythonOperator( + task_id="load_raw_training_data", + python_callable=call_stored_proc, + dag=dag, + op_kwargs={'proc_name':'LOAD_TRAINING_DATA_RAW', 'file_type':'raw_training_data'} +) + +load_business_config = PythonOperator( + task_id="load_business_config", + python_callable=call_stored_proc, + dag=dag, + op_kwargs={'proc_name':'LOAD_BUSINESS_CONFIG', 'file_type':'business_config'} +) + + + +end_job = EmptyOperator(task_id='End') + + +begin_job >> load_training_results >> load_attempt_logs_sp >> load_business_config >> end_job \ No newline at end of file diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index ffaecb2..ba20121 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -10,7 +10,7 @@ definitions: export AWS_REGION='us-east-2'; - script: &aws-context-uat export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO_UAT:role/$OIDC_ROLE_UAT; - export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token;s + export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token; echo $BITBUCKET_STEP_OIDC_TOKEN > $(pwd)/web-identity-token; export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId'); export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey'); @@ -50,38 +50,16 @@ definitions: # trigger: manual caches: - pip - - step: &terraform_plan_apply - name: "Plan and Apply Terraform" - image: python:3.8 - script: - - python -m pip install --upgrade pip - - apt-get update && apt-get install -y jq git - - pip install awscli - - pip install terraform - - export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO:role/$OIDC_ROLE - - export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token - - echo $BITBUCKET_STEP_OIDC_TOKEN > $(pwd)/web-identity-token - - export STS_OUTPUT=$(aws sts assume-role-with-web-identity --role-arn $AWS_ROLE_ARN --role-session-name BitbucketPipeline --web-identity-token "$BITBUCKET_STEP_OIDC_TOKEN" --duration-seconds 3600) - - export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId') - - export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey') - - export AWS_SESSION_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken') - - aws sts get-caller-identity - - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="region=us-east-2" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=devops-pipeline/terraform.tfstate" - - terraform validate - - BRANCH_NAME_LOWER=$(echo $BITBUCKET_BRANCH | tr '[:upper:]' '[:lower:]') - - terraform apply -auto-approve -no-color -var 'access_key=$AWS_ACCESS_KEY_ID' -var 'secret_key=$AWS_SECRET_ACCESS_KEY' -var 'aws_region=us-east-2' -var 'environment=$BRANCH_NAME_LOWER' - trigger: automatic # uncomment after testing - # trigger: manual - step: &streamlit_deploy name: "Deploy streamlit to EC2" oidc: true image: python:3.8 script: - - python -m pip install --upgrade pip + - python -m pip install --upgrade pip - apt-get update && apt-get install -y jq git - pip install awscli - export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO:role/$OIDC_ROLE - - export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token + - export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token - echo $BITBUCKET_STEP_OIDC_TOKEN > $(pwd)/web-identity-token - export STS_OUTPUT=$(aws sts assume-role-with-web-identity --role-arn $AWS_ROLE_ARN --role-session-name BitbucketPipeline --web-identity-token "$BITBUCKET_STEP_OIDC_TOKEN" --duration-seconds 3600) - export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId') @@ -95,23 +73,23 @@ definitions: image: python:3.8 oidc: true script: - - python -m pip install --upgrade pip + - python -m pip install --upgrade pip - apt-get update && apt-get install -y jq git - pip install awscli - export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO:role/$OIDC_ROLE - - export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token + - export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token - echo $BITBUCKET_STEP_OIDC_TOKEN > $(pwd)/web-identity-token - export STS_OUTPUT=$(aws sts assume-role-with-web-identity --role-arn $AWS_ROLE_ARN --role-session-name BitbucketPipeline --web-identity-token "$BITBUCKET_STEP_OIDC_TOKEN" --duration-seconds 3600) - export AWS_ACCESS_KEY_ID=$(echo $STS_OUTPUT | jq -r '.Credentials.AccessKeyId') - export AWS_SECRET_ACCESS_KEY=$(echo $STS_OUTPUT | jq -r '.Credentials.SecretAccessKey') - export AWS_SESSION_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken') - aws sts get-caller-identity - - aws s3 sync ./airflow/dags s3://doczy-dev-infra-mwaa-resources/dags + - aws s3 sync ./airflow/dags s3://doczy-dev-infra-mwaa-resources/dags trigger: automatic # uncomment after testing pipelines: branches: - DEV: + DEV: - step: <<: *snowflake_deploy condition: @@ -125,7 +103,7 @@ pipelines: name: "Deploy streamlit to EC2" oidc: true script: - - python -m pip install --upgrade pip + - python -m pip install --upgrade pip - apt-get update && apt-get install -y jq git - pip install awscli - *aws-context-dev @@ -142,94 +120,85 @@ pipelines: includePaths: - airflow/dags/* - step: - name: "Plan & Apply Textract Pipeline" + name: Apply textract-pipeline on DEV image: hugree/bitbucket-aws-python38-tf154:latest oidc: true - trigger : manual script: - *aws-context-dev - - cd textract-pipeline/terraform/ - # Ensure the backend configuration for s3 obj key is corresponding to the stream name - - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=terraform/textract-pipeline/terraform.tfstate" - - terraform validate - - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=dev" + - python terraform.py apply --environment=dev --module=textract-pipeline/terraform condition: changesets: includePaths: + - textract-pipeline/src/* + - textract-pipeline/src/**/* - textract-pipeline/terraform/* - - textract-pipeline/terraform/**/* + - textract-pipeline/terraform/**/* - step: - name: "Plan & Apply DevOps Pipeline" + name: Apply devops-pipeline on DEV image: hugree/bitbucket-aws-python38-tf154:latest oidc: true - trigger : manual # Keeping it manual so that it doesnt conflict with existing resources script: - *aws-context-dev - - cd devops-pipeline/terraform-backend-resources/ - # Ensure the backend configuration for s3 obj key is correct - - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=terraform/devops-pipeline/terraform.tfstate" - - terraform validate - - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=dev" + - python terraform.py apply --environment=dev --module=devops-pipeline/other-resources condition: changesets: includePaths: - - devops-pipeline/terraform-backend-resources/* - - devops-pipeline/terraform-backend-resources/**/* + - devops-pipeline/other-resources/* + - devops-pipeline/other-resources/**/* - step: - name: "Plan & Apply Streamlit Pipeline" + name: Apply streamlit-server on DEV image: hugree/bitbucket-aws-python38-tf154:latest oidc: true - trigger : automatic script: - *aws-context-dev - - cd streamlit-server/ - # Ensure the backend configuration for s3 obj key is correct - - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=terraform/streamlit-pipeline/terraform.tfstate" -backend-config="region=us-east-2" - - terraform validate - - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=dev" -var "vpc_id=vpc-0f9d7c913f6522079" + - python terraform.py apply --environment=dev --module=streamlit-server condition: changesets: includePaths: - streamlit-server/* - - streamlit-server/**/* + - streamlit-server/**/* + - # UAT and PROD pipelines will be updated with Streamlit steps once it is tested and ready UAT: - step: - name: "Plan & Apply DevOps Pipeline" + name: Apply textract-pipeline on UAT image: hugree/bitbucket-aws-python38-tf154:latest oidc: true - trigger : automatic # Keeping it manual so that it doesnt conflict with existing resources script: - *aws-context-uat - - cd devops-pipeline/terraform-backend-resources/ - - terraform validate - # Ensure the backend configuration for s3 obj key is correct - - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/devops-pipeline/terraform.tfstate" - - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" + - python terraform.py apply --environment=uat --module=textract-pipeline/terraform condition: changesets: includePaths: - - devops-pipeline/terraform-backend-resources/* - - devops-pipeline/terraform-backend-resources/**/* + - textract-pipeline/src/* + - textract-pipeline/src/**/* + - textract-pipeline/terraform/* + - textract-pipeline/terraform/**/* - step: - name: "Plan & Apply Streamlit Pipeline" + name: Apply devops-pipeline on UAT image: hugree/bitbucket-aws-python38-tf154:latest oidc: true - trigger : manual script: - *aws-context-uat - - cd streamlit-server/ - - terraform validate - # Ensure the backend configuration for s3 obj key is correct - - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/streamlit-server/terraform.tfstate" - - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" + - python terraform.py apply --environment=uat --module=devops-pipeline/other-resources condition: changesets: includePaths: - - streamlit-server/* - - streamlit-server/**/* + - devops-pipeline/other-resources/* + - devops-pipeline/other-resources/**/* + - step: + name: Apply streamlit-server on UAT + image: hugree/bitbucket-aws-python38-tf154:latest + oidc: true + script: + - *aws-context-uat + - python terraform.py apply --environment=uat --module=streamlit-server + condition: + changesets: + includePaths: + - streamlit-server/* + - streamlit-server/**/* - step: <<: *snowflake_deploy condition: @@ -242,12 +211,12 @@ pipelines: image: python:3.8 name: "Deploy streamlit to EC2" oidc: true - trigger: manual # Keeping it manual as the server will be created first and then referenced in this + trigger: automatic # Keeping it manual as the server will be created first and then referenced in this script: - - python -m pip install --upgrade pip + - python -m pip install --upgrade pip - apt-get update && apt-get install -y jq git - pip install awscli - - *aws-context-dev + - *aws-context-uat - *ssm-send-command-uat condition: changesets: @@ -261,33 +230,56 @@ pipelines: changesets: includePaths: - airflow/dags/* - - step: - name: "Plan & Apply Textract Pipeline" - image: hugree/bitbucket-aws-python38-tf154:latest - oidc: true - trigger : manual - script: - - *aws-context-uat - - cd textract-pipeline/terraform/ - - terraform validate - # Ensure the backend configuration for s3 obj key is corresponding to the stream name - - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/textract-pipeline/terraform.tfstate" - - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" - condition: - changesets: - includePaths: - - textract-pipeline/terraform/* - - textract-pipeline/terraform/**/* - PROD: - - step: - <<: *terraform_plan_apply - condition: - paths: - - terraform/prod//**/* - step: <<: *snowflake_deploy condition: paths: - snowflake/**/*.sql + + + feature/terraform-refactor: + - parallel: + - step: + name: Plan devops-pipeline on DEV + image: hugree/bitbucket-aws-python38-tf154:latest + oidc: true + script: + - *aws-context-dev + - python terraform.py plan --environment=dev --module=devops-pipeline/other-resources + - step: + name: Plan streamlit-server on DEV + image: hugree/bitbucket-aws-python38-tf154:latest + oidc: true + script: + - *aws-context-dev + - python terraform.py plan --environment=dev --module=streamlit-server + - step: + name: Plan textract-pipeline on DEV + image: hugree/bitbucket-aws-python38-tf154:latest + oidc: true + script: + - *aws-context-dev + - python terraform.py plan --environment=dev --module=textract-pipeline/terraform + - step: + name: Plan devops-pipeline on UAT + image: hugree/bitbucket-aws-python38-tf154:latest + oidc: true + script: + - *aws-context-uat + - python terraform.py plan --environment=uat --module=devops-pipeline/other-resources + - step: + name: Plan streamlit-server on UAT + image: hugree/bitbucket-aws-python38-tf154:latest + oidc: true + script: + - *aws-context-uat + - python terraform.py plan --environment=uat --module=streamlit-server + - step: + name: Plan textract-pipeline on UAT + image: hugree/bitbucket-aws-python38-tf154:latest + oidc: true + script: + - *aws-context-uat + - python terraform.py plan --environment=uat --module=textract-pipeline/terraform diff --git a/devops-pipeline/other-resources/main.tf b/devops-pipeline/other-resources/main.tf index bc17c4e..11e41b4 100644 --- a/devops-pipeline/other-resources/main.tf +++ b/devops-pipeline/other-resources/main.tf @@ -6,22 +6,12 @@ terraform { } } backend "s3" { - bucket = "doczyai-use2-d-infra-s3-terraform-state" # Parameterize using -backend-config flag with "terraform init" - key = "terraform/devops-pipeline/devops.tfstate" # Parameterize - region = "us-east-2" # Parameterize - # profile = "doczyai" # Parameterize - dynamodb_table = "doczyai-use2-d-infra-dyd-terraform-lock" # Parameterize - encrypt = true + encrypt = true } } provider "aws" { - # access_key = var.access_key - # secret_key = var.secret_key - - profile = var.aws_profile - default_tags { tags = { Terraform = "true" @@ -159,7 +149,7 @@ resource "aws_iam_role" "snowflake_integration_role" { } Condition = { StringEquals = { - "sts:ExternalId" = "OQ11564_SFCRole=2_lNKzpyFn/e/xQgvGGpusVbEZA0A=" + "sts:ExternalId" = var.storage_integration_external_id } } }, @@ -213,6 +203,7 @@ resource "aws_iam_role_policy_attachment" "snowflake_integration_policy_attachme # IAM role - cross-account-role resource "aws_iam_role" "cross_account_role" { + count = var.environment != "dev"? 1 : 0 name = local.cross_account_role_name assume_role_policy = jsonencode({ @@ -223,7 +214,7 @@ resource "aws_iam_role" "cross_account_role" { Effect = "Allow" Sid = "" Principal = { - AWS = "arn:aws:iam::873115228912:root" + AWS = "arn:aws:iam::${var.cross_account_target_account_id}:root" } }, ] @@ -250,6 +241,7 @@ data "aws_iam_policy_document" "cross_account_policy_document" { # IAM Policy for cross-account-role resource "aws_iam_policy" "cross_account_policy" { + count = var.environment != "dev"? 1 : 0 name = local.cross_account_policy_name description = "This role is used for other AWS account to assume inorder to drop files to our data ingestion bucket. At the time of creation, this is being used by CODE DEV to drop all clients list." policy = data.aws_iam_policy_document.cross_account_policy_document.json @@ -257,8 +249,9 @@ resource "aws_iam_policy" "cross_account_policy" { # Attach IAM Policy to cross-account-role resource "aws_iam_role_policy_attachment" "cross_account_policy_attachment" { - role = aws_iam_role.cross_account_role.name - policy_arn = aws_iam_policy.cross_account_policy.arn + count = var.environment != "dev"? 1 : 0 + role = aws_iam_role.cross_account_role[count.index].name + policy_arn = aws_iam_policy.cross_account_policy[count.index].arn } # IAM role - mwaa-exec-role @@ -293,14 +286,14 @@ data "aws_iam_policy_document" "mwaa_exec_policy_document" { effect = "Allow" actions = ["airflow:CreateCliToken"] resources = [ - "arn:aws:airflow:us-east-2:660131068782:environment/doczy-dev-infra-mwaa" + "arn:aws:airflow:us-east-2:${var.aws_account_id}:environment/doczy-${var.environment}-infra-mwaa" ] } statement { effect = "Allow" actions = ["airflow:PublishMetrics"] resources = [ - "arn:aws:airflow:us-east-2:660131068782:environment/doczy-dev-infra-mwaa" + "arn:aws:airflow:us-east-2:${var.aws_account_id}:environment/doczy-${var.environment}-infra-mwaa" ] } statement { @@ -337,7 +330,7 @@ data "aws_iam_policy_document" "mwaa_exec_policy_document" { "logs:GetQueryResults" ] resources = [ - "arn:aws:logs:us-east-2:660131068782:log-group:airflow-doczy-dev-infra-mwaa-*" + "arn:aws:logs:us-east-2:${var.aws_account_id}:log-group:airflow-doczy-${var.environment}-infra-mwaa-*" ] } diff --git a/devops-pipeline/other-resources/outputs.tf b/devops-pipeline/other-resources/outputs.tf index a8e15a4..73b55c5 100644 --- a/devops-pipeline/other-resources/outputs.tf +++ b/devops-pipeline/other-resources/outputs.tf @@ -17,7 +17,7 @@ output "snowflake_integration_role_arn" { } output "cross_account_role_arn" { - value = aws_iam_role.cross_account_role.id + value = aws_iam_role.cross_account_role[*].id } output "mwaa_exec_role_arn" { diff --git a/devops-pipeline/other-resources/variables.tf b/devops-pipeline/other-resources/variables.tf index 1035b4d..7905f6b 100644 --- a/devops-pipeline/other-resources/variables.tf +++ b/devops-pipeline/other-resources/variables.tf @@ -1,19 +1,3 @@ -# required -variable "aws_profile" { - type = string - # default = "doczyai" -} - -# # required -# variable "secret_key" { -# type = string -# } - -# # required -# variable "access_key" { -# type = string -# } - # required variable "aws_region" { type = string @@ -66,7 +50,19 @@ variable "mwaa_resources_s3_bucket" { }) default = { name = "mwaa-resources" - } } +variable "aws_account_id" { + type = string +} + +variable "storage_integration_external_id" { + type = string +} + + +variable "cross_account_target_account_id" { + type = string +} + diff --git a/devops-pipeline/other-resources/vars-dev.tfvars b/devops-pipeline/other-resources/vars-dev.tfvars new file mode 100644 index 0000000..4de60b8 --- /dev/null +++ b/devops-pipeline/other-resources/vars-dev.tfvars @@ -0,0 +1,3 @@ +aws_account_id = 660131068782 +storage_integration_external_id = "OQ11564_SFCRole=2_lNKzpyFn/e/xQgvGGpusVbEZA0A=" +cross_account_target_account_id = "" \ No newline at end of file diff --git a/devops-pipeline/other-resources/vars-uat.tfvars b/devops-pipeline/other-resources/vars-uat.tfvars new file mode 100644 index 0000000..1d307e6 --- /dev/null +++ b/devops-pipeline/other-resources/vars-uat.tfvars @@ -0,0 +1,3 @@ +aws_account_id = 975049960860 +storage_integration_external_id = "OQ11564_SFCRole=2_L4NUsKqd3NIBmrATa1djc/T3ECQ=" +cross_account_target_account_id = 660131068782 \ No newline at end of file diff --git a/devops-pipeline/terraform-backend-resources/main.tf b/devops-pipeline/terraform-backend-resources/main.tf index 885ad4e..56aec03 100644 --- a/devops-pipeline/terraform-backend-resources/main.tf +++ b/devops-pipeline/terraform-backend-resources/main.tf @@ -8,13 +8,7 @@ terraform { } provider "aws" { - profile = var.aws_profile - region = var.aws_region - # default_tags { - # tags = { - # Terraform = "true" - # } - # } + region = var.aws_region } module "aws_terraform_remote_backend" { diff --git a/devops-pipeline/terraform-backend-resources/modules/aws-terraform-remote-backend/variables.tf b/devops-pipeline/terraform-backend-resources/modules/aws-terraform-remote-backend/variables.tf index 48cfca4..3483195 100644 --- a/devops-pipeline/terraform-backend-resources/modules/aws-terraform-remote-backend/variables.tf +++ b/devops-pipeline/terraform-backend-resources/modules/aws-terraform-remote-backend/variables.tf @@ -10,6 +10,20 @@ variable "environment" { variable "client_name" { type = string } +# required +variable "secret_key" { + type = string +} + +# required +variable "access_key" { + type = string +} + +variable "token" { + type = string +} + # dynamoDB table variable "terraform_dynamodb_table" { type = object({ diff --git a/devops-pipeline/terraform-backend-resources/outputs.tf b/devops-pipeline/terraform-backend-resources/outputs.tf index 58a5ddb..2192baf 100644 --- a/devops-pipeline/terraform-backend-resources/outputs.tf +++ b/devops-pipeline/terraform-backend-resources/outputs.tf @@ -9,4 +9,4 @@ output "terraform_s3_bucket_arn" { } output "terraform_s3_bucket_name" { value = module.aws_terraform_remote_backend.terraform_s3_bucket_name -} \ No newline at end of file +} diff --git a/devops-pipeline/terraform-backend-resources/variables.tf b/devops-pipeline/terraform-backend-resources/variables.tf index 25c303f..3e52526 100644 --- a/devops-pipeline/terraform-backend-resources/variables.tf +++ b/devops-pipeline/terraform-backend-resources/variables.tf @@ -1,9 +1,4 @@ # required -variable "aws_profile" { - type = string - default = "doczy_uat" -} -# required variable "aws_region" { type = string default = "us-east-2" @@ -15,11 +10,25 @@ variable "project_name" { } # required variable "environment" { - type = string - default = "uat" + type = string } # required variable "client_name" { type = string default = "infra" -} \ No newline at end of file +} + +#required +variable "secret_key" { + type = string +} + +#required +variable "access_key" { + type = string +} + +#required +variable "token" { + type = string +} diff --git a/terraform-approach/terraform/config.tfvars b/devops-pipeline/terraform-backend-resources/vars-dev.tfvars similarity index 100% rename from terraform-approach/terraform/config.tfvars rename to devops-pipeline/terraform-backend-resources/vars-dev.tfvars diff --git a/terraform/evironments/dev/backend.tf b/devops-pipeline/terraform-backend-resources/vars-uat.tfvars similarity index 100% rename from terraform/evironments/dev/backend.tf rename to devops-pipeline/terraform-backend-resources/vars-uat.tfvars diff --git a/snowflake/DEV/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql b/snowflake/DEV/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql index 4e5562c..bd1b22f 100644 --- a/snowflake/DEV/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql +++ b/snowflake/DEV/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql @@ -51,7 +51,7 @@ BEGIN batch_id := (SELECT NULLIF(TRIM($2),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1); request_datetime := (SELECT NULLIF(TRIM($13),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1); - UPDATE TABLE STG.CONTRACT_CONFIG + UPDATE STG.CONTRACT_CONFIG SET LATEST_FLAG = FALSE WHERE BATCH_ID = :batch_id AND CONTRACT_NAME = :contract_name AND REQUEST_DATETIME < :request_datetime AND LATEST_FLAG = TRUE; diff --git a/snowflake/DEV/config_interface/R__007_LOAD_DOCZY_PIPELINE_RAW_OPUTPUTS.sql b/snowflake/DEV/config_interface/R__007_LOAD_DOCZY_PIPELINE_RAW_OPUTPUTS.sql index 4c7d5d7..4309d00 100644 --- a/snowflake/DEV/config_interface/R__007_LOAD_DOCZY_PIPELINE_RAW_OPUTPUTS.sql +++ b/snowflake/DEV/config_interface/R__007_LOAD_DOCZY_PIPELINE_RAW_OPUTPUTS.sql @@ -32,7 +32,7 @@ BEGIN NULLIF(TRIM($6),'') AS SNIPPET, NULLIF(TRIM($7),'') AS ORIGINAL_PAGE_NUMBER, NULLIF(TRIM($8),'') AS NEW_PAGE_NUMBER, - NULLIF(TRIM($9),'') AS RESULT + NULLIF(TRIM($9),'') AS RESULT, NULLIF(TRIM($10),'') AS BATCH_ID FROM @STG.DOCZY_PIPELINE_RAW_OUTPUT_STAGE diff --git a/snowflake/DEV/config_interface/R__008_PIPELINE_PROCESSED_OUTPUT.sql b/snowflake/DEV/config_interface/R__008_PIPELINE_PROCESSED_OUTPUT.sql new file mode 100644 index 0000000..daa0ecf --- /dev/null +++ b/snowflake/DEV/config_interface/R__008_PIPELINE_PROCESSED_OUTPUT.sql @@ -0,0 +1,12 @@ +-- CREATING THE CONFIG TABLE +CREATE TABLE IF NOT EXISTS STG.DOCZY_PIPELINE_PROCESSED_OUTPUT ( + CONTRACT_NAME VARCHAR(16777216) NOT NULL, + FIELD_NAME VARCHAR(255), + RAW_VALUE VARCHAR(16777216), + NEW_EXTRACTED_VALUE VARCHAR(16777216), + CONFIDENCE_LEVEL VARCHAR(255), + SNIPPET VARCHAR(16777216), + NEW_PAGE_NUMBER VARCHAR(255), + BATCH_ID VARCHAR(255), + CREATED_TIME TIMESTAMP_NTZ(9) +); \ No newline at end of file diff --git a/snowflake/DEV/config_interface/R__009_LOAD_PIPELINE_PROCESSED_OUTPUTS_SP.sql b/snowflake/DEV/config_interface/R__009_LOAD_PIPELINE_PROCESSED_OUTPUTS_SP.sql new file mode 100644 index 0000000..374c3b4 --- /dev/null +++ b/snowflake/DEV/config_interface/R__009_LOAD_PIPELINE_PROCESSED_OUTPUTS_SP.sql @@ -0,0 +1,73 @@ +-- columns to be ingested +-- CONTRACT_NAME VARCHAR(16777216) NOT NULL, +-- FIELD_NAME VARCHAR(255), +-- RAW_VALUE VARCHAR(16777216), +-- NEW_EXTRACTED_VALUE VARCHAR(16777216), +-- CONFIDENCE_LEVEL VARCHAR(255), +-- SNIPPET VARCHAR(16777216), +-- NEW_PAGE_NUMBER VARCHAR(255), +-- BATCH_ID VARCHAR(255), +-- CREATED_TIME TIMESTAMP_NTZ(9) + +CREATE OR REPLACE PROCEDURE STG.LOAD_DOCZY_PIPELINE_PROCESSED_OUTPUT(file_name VARCHAR) +RETURNS STRING +LANGUAGE SQL +EXECUTE AS CALLER +AS +$$ +DECLARE + procedure_name varchar; +BEGIN + + procedure_name := 'LOAD_DOCZY_PIPELINE_PROCESSED_OUTPUT'; + + call stg.log_audit(:procedure_name, 'Section 1', 99, 'START'); + + -- Create or replace stage with dynamic file name + EXECUTE IMMEDIATE 'CREATE OR REPLACE STAGE STG.DOCZY_PIPELINE_PROCESSED_OUTPUT_STAGE + STORAGE_INTEGRATION = dev_bucket_integration + URL = ''s3://doczy-dev-infra-raw-data-ingestion/doczy_pipeline_output/' || :file_name || ''' + FILE_FORMAT = (FORMAT_NAME = ''STG.CSV_HEADER'');'; + + call stg.log_audit(:procedure_name, 'Section 1', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 2', 99, 'START'); + + COPY INTO STG.DOCZY_PIPELINE_RAW_OUTPUT FROM ( + SELECT + NULLIF(TRIM($1),'') AS CONTRACT_NAME, + NULLIF(TRIM($2),'') AS FIELD_NAME, + NULLIF(TRIM($3),'') AS RAW_VALUE, + NULLIF(TRIM($4),'') AS NEW_EXTRACTED_VALUE, + NULLIF(TRIM($5),'') AS CONFIDENCE_LEVEL, + NULLIF(TRIM($6),'') AS SNIPPET, + NULLIF(TRIM($7),'') AS NEW_PAGE_NUMBER, + NULLIF(TRIM($8),'') AS BATCH_ID, + CURRENT_TIMESTAMP() AS CREATED_TIME + + FROM @STG.DOCZY_PIPELINE_PROCESSED_OUTPUT_STAGE + ) + FILE_FORMAT = (FORMAT_NAME = 'STG.CSV_HEADER') + ON_ERROR = ABORT_STATEMENT; + + call stg.log_audit(:procedure_name, 'Section 2', 99, 'END'); + + call stg.log_audit(:procedure_name, 'Section 3', 99, 'START'); + + INSERT INTO STG.DIM_AUDIT (AUDIT_SID, TABLE_NAME, SOURCE_FILE_NAME, LOAD_DATE, SOURCE_COUNT) + SELECT STG.AUDIT_SID.NEXTVAL, 'STG.DOCZY_PIPELINE_PROCESSED_OUTPUT',* + FROM + (SELECT DISTINCT METADATA$FILENAME, CURRENT_TIMESTAMP(), max(METADATA$FILE_ROW_NUMBER) from @STG.DOCZY_PIPELINE_PROCESSED_OUTPUT_STAGE group by 1,2); + + + + RETURN 'Setup, Load, and Audit Complete'; + + call stg.log_audit(:procedure_name, 'Section 3', 99, 'END'); + + + + +END; +$$; + diff --git a/snowflake/UAT/R__1002_LOG_AUDIT_TABLE.sql b/snowflake/UAT/R__1002_LOG_AUDIT_TABLE.sql index 1cc0768..d3ce540 100644 --- a/snowflake/UAT/R__1002_LOG_AUDIT_TABLE.sql +++ b/snowflake/UAT/R__1002_LOG_AUDIT_TABLE.sql @@ -8,4 +8,4 @@ create or replace TABLE STG.LOG_AUDIT ( SUB_SECTION_NAME VARCHAR(255) COLLATE 'en-ci', AUDIT_SID NUMBER(38,0), TOTAL_MIN NUMBER(38,0) -); \ No newline at end of file +); diff --git a/snowflake/UAT/config_interface/R__1004_LOAD_CONTRACT_CONFIG_SP.sql b/snowflake/UAT/config_interface/R__1004_LOAD_CONTRACT_CONFIG_SP.sql index 4e5562c..e00a89c 100644 --- a/snowflake/UAT/config_interface/R__1004_LOAD_CONTRACT_CONFIG_SP.sql +++ b/snowflake/UAT/config_interface/R__1004_LOAD_CONTRACT_CONFIG_SP.sql @@ -50,8 +50,8 @@ BEGIN contract_name := (SELECT NULLIF(TRIM($1),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1); batch_id := (SELECT NULLIF(TRIM($2),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1); request_datetime := (SELECT NULLIF(TRIM($13),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1); - - UPDATE TABLE STG.CONTRACT_CONFIG + + UPDATE STG.CONTRACT_CONFIG SET LATEST_FLAG = FALSE WHERE BATCH_ID = :batch_id AND CONTRACT_NAME = :contract_name AND REQUEST_DATETIME < :request_datetime AND LATEST_FLAG = TRUE; diff --git a/streamlit-server/alb.tf b/streamlit-server/alb.tf index a94a31c..48b6bdb 100644 --- a/streamlit-server/alb.tf +++ b/streamlit-server/alb.tf @@ -115,7 +115,8 @@ resource "aws_lb_listener" "https_8500" { protocol = "HTTPS" ssl_policy = "ELBSecurityPolicy-2016-08" # Using variable for ARN - certificate_arn = var.acm_arn_dev + certificate_arn = lookup(var.acm_arns, var.environment) + default_action { type = "forward" @@ -128,7 +129,7 @@ resource "aws_lb_listener" "https_8501" { port = "8501" protocol = "HTTPS" ssl_policy = "ELBSecurityPolicy-2016-08" - certificate_arn = var.acm_arn_dev + certificate_arn = lookup(var.acm_arns, var.environment) default_action { type = "forward" @@ -141,7 +142,7 @@ resource "aws_lb_listener" "https_8502" { port = "8502" protocol = "HTTPS" ssl_policy = "ELBSecurityPolicy-2016-08" - certificate_arn = var.acm_arn_dev + certificate_arn = lookup(var.acm_arns, var.environment) default_action { type = "forward" @@ -154,7 +155,7 @@ resource "aws_lb_listener" "https_8503" { port = "8503" protocol = "HTTPS" ssl_policy = "ELBSecurityPolicy-2016-08" - certificate_arn = var.acm_arn_dev + certificate_arn = lookup(var.acm_arns, var.environment) default_action { type = "forward" diff --git a/streamlit-server/iam.tf b/streamlit-server/iam.tf index e1aaa0b..5fce91b 100644 --- a/streamlit-server/iam.tf +++ b/streamlit-server/iam.tf @@ -71,7 +71,7 @@ data "aws_iam_policy_document" "combined_policy_document" { ] resources = ["*"] } - + statement { sid = "APIGatewayInvokeFullAccess" effect = "Allow" @@ -131,3 +131,6 @@ resource "aws_iam_role_policy_attachment" "combined_policy_attachment" { policy_arn = aws_iam_policy.combined_policy.arn } +output "ec2_role_name" { + value = local.ec2_role_name +} diff --git a/streamlit-server/main.tf b/streamlit-server/main.tf index 1fd8316..75fc049 100644 --- a/streamlit-server/main.tf +++ b/streamlit-server/main.tf @@ -1,5 +1,5 @@ provider "aws" { - region = "us-east-2" + region = "us-east-2" } terraform { @@ -11,11 +11,6 @@ terraform { } backend "s3" { - # bucket = "doczyai-use2-d-infra-s3-terraform-state" # Parameterize using -backend-config flag with "terraform init" - # key = "terraform/streamlit-server/terraform.tfstate" # Parameterize - # region = "us-east-2" # Parameterize - # profile = "temp_cred" # Parameterize - # dynamodb_table = "doczyai-use2-d-infra-dyd-terraform-lock" # Parameterize encrypt = true } } @@ -43,7 +38,7 @@ locals { # global prefix global_prefix = "${var.project_name}-${local.region_short_name}-${local.environment_short_name}" - + # Resource prefix iam_policy_prefix = "${local.global_prefix}-pol" @@ -65,14 +60,14 @@ data "aws_subnets" "subnets" { values = [var.vpc_id] } } - + # vpc security groups data "aws_security_groups" "security_groups" { filter { name = "group-name" values = ["*default*"] } - + filter { name = "vpc-id" values = [var.vpc_id] @@ -86,27 +81,27 @@ resource "aws_iam_instance_profile" "ec2_instance_profile" { } resource "aws_instance" "streamlit_server" { - ami = var.ubuntu_ami + ami = lookup(var.ubuntu_ami, var.environment) instance_type = var.ec2_instance_type subnet_id = data.aws_subnets.subnets.ids[0] - vpc_security_group_ids = data.aws_security_groups.security_groups.ids - iam_instance_profile = aws_iam_instance_profile.ec2_instance_profile.name + vpc_security_group_ids = data.aws_security_groups.security_groups.ids + iam_instance_profile = aws_iam_instance_profile.ec2_instance_profile.name associate_public_ip_address = false - key_name = "tf-test-key" # Created using TF and uploaded to S3. Prerequisite + key_name = var.key_name root_block_device { volume_size = 30 # Variable encrypted = true # Mandatory } -# Setup SSH authentication for bitbucket access to pull code -# Setup SSL certificate for HTTPS access for SSO to work +# Setup SSH authentication for bitbucket access to pull code +# Setup SSL certificate for HTTPS access for SSO to work user_data = <<-EOF #!/bin/bash echo '${file("${path.module}/requirements.txt")}' > /tmp/requirements.txt apt-get update apt-get install -y python3 python3-pip sudo apt-get install libffi-dev - python3 -m pip3 install -r /tmp/requirements.txt + python3 -m pip install -r /tmp/requirements.txt sudo apt install openssh-client sudo apt install net-tools @@ -202,3 +197,4 @@ resource "aws_instance" "streamlit_server" { } + diff --git a/streamlit-server/variables.tf b/streamlit-server/variables.tf index 79f9735..e919d93 100644 --- a/streamlit-server/variables.tf +++ b/streamlit-server/variables.tf @@ -12,7 +12,6 @@ variable "project_name" { # required variable "environment" { type = string - default = "dev" } @@ -31,10 +30,18 @@ variable "vpc_id" { type = string } +# variable "ubuntu_ami" { +# type = string +# default = "ami-0de7e97fedfbc6ef6" # Switch to this ami-0b8b44ec9a8f90422 + +# } + variable "ubuntu_ami" { - type = string - default = "ami-0de7e97fedfbc6ef6" # Switch to this ami-0b8b44ec9a8f90422 - + type = map(string) + default = { + dev = "ami-0de7e97fedfbc6ef6" + uat = "ami-0b8b44ec9a8f90422" + } } variable "ec2_instance_type" { @@ -42,12 +49,6 @@ variable "ec2_instance_type" { default = "t2.large" } -# EC2 iam role name -variable "ec2_iam_role_name" { - type = string - default = "ec2-iam-role" -} - # Cerate an array of ARNS variable "SecretsNames"{ type = list(string) @@ -59,16 +60,15 @@ variable "acm_arn_dev" { default = "arn:aws:acm:us-east-2:660131068782:certificate/947e04b3-be22-43bf-bea2-a0ad3b221d70" } -variable "secret_key" { - type = string +variable "acm_arns" { + type = map(string) + default = { + dev = "arn:aws:acm:us-east-2:660131068782:certificate/947e04b3-be22-43bf-bea2-a0ad3b221d70" + uat = "arn:aws:acm:us-east-2:975049960860:certificate/e036e76d-1f4f-496e-ac97-5f65ec4a34cc" + prod = "arn:aws:acm:region:account-id:certificate/prod-cert-id" + } } -# required -variable "access_key" { - type = string +variable "key_name" { } -# required -variable "token" { - type = string -} \ No newline at end of file diff --git a/streamlit-server/vars-dev.tfvars b/streamlit-server/vars-dev.tfvars new file mode 100644 index 0000000..50fcb55 --- /dev/null +++ b/streamlit-server/vars-dev.tfvars @@ -0,0 +1,2 @@ +vpc_id = "vpc-0f9d7c913f6522079" +key_name = "tf-test-key" \ No newline at end of file diff --git a/streamlit-server/vars-uat.tfvars b/streamlit-server/vars-uat.tfvars new file mode 100644 index 0000000..96cb6a8 --- /dev/null +++ b/streamlit-server/vars-uat.tfvars @@ -0,0 +1,2 @@ +vpc_id = "vpc-0392396d0e7bdd77f" +key_name = "tf-uat-key" \ No newline at end of file diff --git a/streamlit/interface_0.py b/streamlit/interface_0.py index c7c0bb5..95db3a6 100644 --- a/streamlit/interface_0.py +++ b/streamlit/interface_0.py @@ -156,6 +156,7 @@ with buttons[1]: + # @st.cache_data # def convert_df(df): # return df.to_csv(index=False).encode('utf-8') diff --git a/streamlit/interface_1.py b/streamlit/interface_1.py index 0f7db6c..08ed07b 100644 --- a/streamlit/interface_1.py +++ b/streamlit/interface_1.py @@ -8,6 +8,7 @@ from datetime import datetime import boto3 import util import requests +import time from sf_conn import get_client_names, get_secret, save_to_sf from constants import USER_LIST, DOCZY_PIPELINE_URL_DEV @@ -74,10 +75,11 @@ client_row = st.columns([0.1, 0.8]) with client_row[0]: st.write("**Client Name**") with client_row[1]: - client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed") + client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344 client_bucket = client_s3_paths.get(client) -# # to be deleted when buckets for different clients are ready; below line is added only for testing the corresponding DAG + +# to be deleted when buckets for different clients are ready; below line is added only for testing the corresponding DAG client_bucket = 'doczy-ai-client-1' batch_objects = s3_client.list_objects_v2(Bucket=client_bucket @@ -87,17 +89,51 @@ batch_list = [] for prefix in batch_objects['CommonPrefixes']: batch_list.append(prefix['Prefix'][:-1].split('/')[-1]) +# Hardcoded batch_list for testing purposes +# batch_list = ['batch_020524103737', 'batch_090524131433', 'batch_090524131607', 'batch_100524123000', 'batch_130524064322', +# 'batch_160524071331', 'batch_200524213550', 'batch_250424112237', 'batch_280524120530', 'batch_280524121721', 'batch_280524144222', +# 'batch_290524123926', 'batch_290524164044', 'batch_310524102029', 'batch_310524124050', 'batch_310524162346', 'batch_310524162631'] + +if 'sorted_list' not in st.session_state: + st.session_state.sorted_list = batch_list + +def sort_list(ex_list, sort_by, order): + if sort_by == 'Alphabetical': + ex_list = sorted(ex_list, reverse=(order == 'Descending')) + elif sort_by == 'Create Date': + ex_list = ex_list if order == 'Ascending' else list(reversed(ex_list)) + + return ex_list + +col1, col2, col3, col4 = st.columns([0.5, 0.5, 0.5, 0.5]) + + +with col1: + sort_by = st.radio("**Sort Batch_IDs**", ('Alphabetical', 'Create Date')) + +with col2: + order = st.radio('', ('Ascending','Descending')) + +with col3: + add_vertical_space(2) + if st.button('Apply'): + st.session_state.sorted_list = sort_list(batch_list, sort_by, order) + path_row = st.columns([0.1, 0.8]) with path_row[0]: st.write("**Batch ID**") with path_row[1]: - batch_id = st.selectbox('**Batch ID**', batch_list, label_visibility = "collapsed") + batch_id = st.selectbox('**Batch ID**', st.session_state.sorted_list, label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344 + +if not batch_id: + batch_id = "None" checks = st.columns([0.1, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12]) with checks[0]: st.write("**Group No.**") + with checks[1]: - a = st.checkbox('Unique Key', key = str(1)) + a = st.checkbox('Unique Key', key = str(1), args="Unique") with checks[2]: b = st.checkbox('Pricing Before Carveouts', key = str(2)) with checks[3]: @@ -119,11 +155,15 @@ file_list = [] file_objects = s3_client.list_objects_v2(Bucket=client_bucket , Prefix="contracts_landing_zone/"+batch_id+"/", Delimiter='/') +# Hardcoded file_list for testing purposes +# file_list = ['Boilerplate_TX Amendment Mission Health Network effective_040114 MU.pdf', 'Custom_TX - MP AMENDMENT - MISSION HEALTH NETWORK - MU.pdf', +# 'Delaware First Health_First State Homecare Agency_212260_7 MU.pdf', 'Molina Healthcare of Texas, Inc. Amendment 4 - HIX ACA__EFF 01012016_MU.pdf'] + if st.button("Read the contracts from Path"): for obj in file_objects.get('Contents',[]): if not obj['Key'].endswith('/'): file_list.append(obj['Key'].split('/')[-1]) - + df['Contract Name'] = file_list # df['Request ID'] = range(len(file_list)) # df['Contract ID'] = file_list @@ -160,23 +200,33 @@ additional_info = pd.DataFrame(columns=['CLIENT_NAME', 'BATCH_ID', 'REQUEST_USER additional_info.loc[0] = [client, batch_id, st.session_state.user_info['mail'], datetime.now().strftime("%Y-%m-%d %H:%M:%S")] st.write(additional_info) +st.session_state.contract_count = 0 contract_list = [] for index, row in edited_df.iterrows(): + allow_run_for_contract = False group_list = [] if row['Unique Key']: group_list.append('Unique Key') + allow_run_for_contract = True if row['Pricing Before Carveouts']: group_list.append('Pricing Before Carveouts') + allow_run_for_contract = True if row['Contract Related']: group_list.append('Contract Related') + allow_run_for_contract = True if row['Provider']: group_list.append('Provider') + allow_run_for_contract = True if row['Timeline']: group_list.append('Timeline') + allow_run_for_contract = True if row['Carveout Indicator']: group_list.append('Carveout Indicator') + allow_run_for_contract = True if row['Carveout Methodology']: group_list.append('Carveout Methodology') + allow_run_for_contract = True + if allow_run_for_contract: st.session_state.contract_count += 1 entry_dict = { "contract_name": row['Contract Name'], "groups": group_list, @@ -197,23 +247,27 @@ with buttons[0]: st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv') with buttons[1]: if st.button("Run Doczy.AI Pipeline"): - # csv_buf = StringIO() - # additional_info.to_csv(csv_buf, header=True, index=False) - # csv_buf.seek(0) - # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/request_submission.csv') - # csv_buf = StringIO() - # edited_df.to_csv(csv_buf, header=True, index=False) - # csv_buf.seek(0) - # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/contract_config.csv') - # try: - # save_to_sf('load_request_and_contract_submissions', request_submission_file_name = "request_submission.csv", contract_config_file_name = "contract_config.csv") - # except Exception as e: - # st.write(e) - response = requests.post(doczy_pipeline, json = myobj) - if response.status_code >= 200 and response.status_code < 300: - st.write("Success") - # st.write(myobj) + if not st.session_state.contract_count == len(edited_df): + st.error("Select at least one Group No. for every Contract") else: - st.write("Failed") - # st.write(response.text) + with st.spinner('Running...'): + # csv_buf = StringIO() + # additional_info.to_csv(csv_buf, header=True, index=False) + # csv_buf.seek(0) + # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/request_submission.csv') + # csv_buf = StringIO() + # edited_df.to_csv(csv_buf, header=True, index=False) + # csv_buf.seek(0) + # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/contract_config.csv') + # try: + # save_to_sf('load_request_and_contract_submissions', request_submission_file_name = "request_submission.csv", contract_config_file_name = "contract_config.csv") + # except Exception as e: + # st.write(e) + response = requests.post(doczy_pipeline, json = myobj) + if response.status_code >= 200 and response.status_code < 300: + st.write("Success") + # st.write(myobj) + else: + st.write("Failed") + # st.write(response.text) diff --git a/streamlit/interface_2.py b/streamlit/interface_2.py index e6b4ffb..f9bad41 100644 --- a/streamlit/interface_2.py +++ b/streamlit/interface_2.py @@ -10,6 +10,7 @@ from langchain.chains import RetrievalQA import streamlit as st from streamlit_extras.add_vertical_space import add_vertical_space +from streamlit_pdf_viewer import pdf_viewer # needs to be installed on the server import os import pandas as pd import numpy as np @@ -18,24 +19,26 @@ import anthropic from pydantic import BaseModel from typing import List import re +import base64 from sf_conn import get_snowflake_conn +import io REDIRECT_URI = 'https://doczydev.aarete.com:8502' user_list = USER_LIST st.set_page_config(layout = "wide") # Sidebar contents -with st.sidebar: - st.title("Doczy.AI ™") - st.markdown( - """ - ## About - This app extracts data from contracts +# with st.sidebar: +# st.title("Doczy.AI ™") +# st.markdown( +# """ +# ## About +# This app extracts data from contracts - """ - ) - add_vertical_space(15) - # st.write("Doczy") +# """ +# ) +# add_vertical_space(15) +# # st.write("Doczy") # AARETE LOGO x,y,z = st.columns([15,2,15]) @@ -104,13 +107,11 @@ for obj in objects['Contents']: contract_list = sorted(file_list) - - file_row = st.columns([0.2, 0.7, 0.1]) with file_row[0]: st.write("**Contract Name**") with file_row[1]: - file_name = st.selectbox('Select a file', contract_list + ['All'], label_visibility = "collapsed") + file_name = st.selectbox('Select a file', ['All'] + contract_list, label_visibility = "collapsed", index= None) # MODIFIED - Append 'All' in the front instead of at the end field_row = st.columns([0.2, 0.7, 0.1]) with field_row[0]: @@ -120,7 +121,7 @@ with field_row[1]: , 'Pricing Before Carveouts - II', 'Carveout Indicator, Code Type and Code #s - I' , 'Carveout Indicator, Code Type and Code #s - II', 'Carveout Indicator, Code Type and Code #s - III' , 'Optimize Carving Indic.', 'Carveout Method - I', 'Carveout Method - II', 'Provider' - , 'Timeline'), label_visibility = "collapsed") + , 'Timeline'), label_visibility = "collapsed", index = None) if field_group == 'Unique Key': fields = fields[fields['PRIORITY'] == 'A'] @@ -164,10 +165,51 @@ if st.button("Show Results"): df2 = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description]) # get this dataframe from snowflake table - # df2 = pd.DataFrame(columns=['Contract Name','Field Name', 'SF_DB_COL_NAME', 'Snippet','Page Number' - # , 'Field Extracted Value', 'Actual Value','Imputed Value']) + df2 = pd.DataFrame(columns=['Contract Name','Field Name', 'SF_DB_COL_NAME', 'Snippet','Page Number' + , 'Field Extracted Value', 'Actual Value','Imputed Value']) df2.to_csv('temp2.csv', index=False) +if st.button("Show PDF"): + if file_name == None or file_name == "All": + st.error("Choose one specific file.") + else: + with st.sidebar: + st.markdown( + """ + + """, + unsafe_allow_html=True, + ) + s3_client.Object(bucket,file_name) + data=obj.get()['Body'].read() + pdf_viewer(io.BytesIO(data), width=1500) + +# if st.button("Show PDF"): +# if file_name == None or file_name == "All": +# st.error("Choose one specific file.") +# else: +# with st.sidebar: +# with open(file_name, "rb") as f: +# base64_pdf = base64.b64encode(f.read()).decode('utf-8') +# # Embedding PDF in HTML +# pdf_display = F'' +# # Displaying File +# st.markdown( +# """ +# +# """, +# unsafe_allow_html=True, +# ) +# st.markdown(pdf_display, unsafe_allow_html=True) + df2 = pd.read_csv('temp2.csv') df2['Imputed Value'] = '' edited_df = st.data_editor(df2) diff --git a/streamlit/local/local_interface_1.py b/streamlit/local/local_interface_1.py new file mode 100644 index 0000000..fd03888 --- /dev/null +++ b/streamlit/local/local_interface_1.py @@ -0,0 +1,259 @@ +import streamlit as st +from streamlit_extras.add_vertical_space import add_vertical_space +import os +import streamlit as st +import pandas as pd +from io import StringIO +from datetime import datetime +import boto3 +# import util +import requests +import time +# from sf_conn import get_client_names, get_secret, save_to_sf +# from constants import USER_LIST, DOCZY_PIPELINE_URL_DEV + +# doczy_pipeline = DOCZY_PIPELINE_URL_DEV + +# REDIRECT_URI = 'https://doczydev.aarete.com:8501' +# user_list = USER_LIST +st.set_page_config(layout = "wide") +# # # Sidebar contents +# # with st.sidebar: +# # st.title("Doczy.AI ™") +# # st.markdown( +# # """ +# # ## About +# # This app extracts data from contracts + +# # """ +# # ) +# # add_vertical_space(15) +# # # st.write("Doczy") + +_,c1= st.columns([5,1]) +# try: +# util.setup_page(REDIRECT_URI) +# except: +# st.write("SSO Failed") +# st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} +st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} +try: + c1.write(f"User: **{st.session_state.user_info['displayName']}**") + user_mail = st.session_state.user_info['mail'] +except KeyError as e: + # Do we add a link to get to the login page here? + st.write("Session Expired.") + st.stop() + + + +s3_client = boto3.client('s3', + region_name="us-east-2", +) + +# # to be replaced with snowflake data +client_list = ['doczy-ai-client-1', 'Delaware First Health, Inc.', 'Community Health Choice, Inc','CareSource Network Partners LLC', + 'HealthNet of Cali', 'Oklahoma Complete Health, Inc', 'HealthFirst', 'Molina Healthcare of TX', 'AvMed', 'Arizona Care1st', + 'WellCare New Jersey'] + +# client_list, s3_paths = get_client_names() +# client_s3_paths = dict(zip(client_list, s3_paths)) + +client_row = st.columns([0.1, 0.8]) +with client_row[0]: + st.write("**Client Name**") +with client_row[1]: + client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344 + +# client_bucket = client_s3_paths.get(client) + +# # to be deleted when buckets for different clients are ready; below line is added only for testing the corresponding DAG +client_bucket = 'doczy-ai-client-1' + +# batch_objects = s3_client.list_objects_v2(Bucket=client_bucket +# , Prefix="contracts_landing_zone/", Delimiter='/') + +# batch_list = [] +# for prefix in batch_objects['CommonPrefixes']: +# batch_list.append(prefix['Prefix'][:-1].split('/')[-1]) + +batch_list = ['batch_020524103737', 'batch_090524131433', 'batch_090524131607', 'batch_100524123000', 'batch_130524064322', + 'batch_160524071331', 'batch_200524213550', 'batch_250424112237', 'batch_280524120530', 'batch_280524121721', 'batch_280524144222', + 'batch_290524123926', 'batch_290524164044', 'batch_310524102029', 'batch_310524124050', 'batch_310524162346', 'batch_310524162631'] + +if 'sorted_list' not in st.session_state: + st.session_state.sorted_list = batch_list + +def sort_list(ex_list, sort_by, order): + if sort_by == 'Alphabetical': + ex_list = sorted(ex_list, reverse=(order == 'Descending')) + elif sort_by == 'Create Date': + ex_list = ex_list if order == 'Ascending' else list(reversed(ex_list)) + + return ex_list + +col1, col2, col3, col4 = st.columns([0.5, 0.5, 0.5, 0.5]) + + +with col1: + sort_by = st.radio("**Sort Batch_IDs**", ('Alphabetical', 'Create Date')) + +with col2: + order = st.radio('', ('Ascending','Descending')) + +with col3: + add_vertical_space(2) + if st.button('Apply'): + st.session_state.sorted_list = sort_list(batch_list, sort_by, order) + +path_row = st.columns([0.1, 0.8]) +with path_row[0]: + st.write("**Batch ID**") +with path_row[1]: + batch_id = st.selectbox('**Batch ID**', st.session_state.sorted_list, label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344 + +if not batch_id: + batch_id = "None" + +checks = st.columns([0.1, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12]) +with checks[0]: + st.write("**Group No.**") + +with checks[1]: + a = st.checkbox('Unique Key', key = str(1), args="Unique") +with checks[2]: + b = st.checkbox('Pricing Before Carveouts', key = str(2)) +with checks[3]: + c = st.checkbox('Contract Related', key = str(3)) +with checks[4]: + d = st.checkbox('Provider', key = str(4)) +with checks[5]: + e = st.checkbox('Timeline', key = str(5)) +with checks[6]: + f = st.checkbox('Carveout Indicator', key = str(6)) +with checks[7]: + g = st.checkbox('Carveout Methodology', key = str(7)) + +add_vertical_space(1) + +df = pd.DataFrame(columns=['Contract Name', 'Unique Key','Pricing Before Carveouts' + , 'Contract Related', 'Provider', 'Timeline', 'Carveout Indicator', 'Carveout Methodology']) +# file_list = [] +# file_objects = s3_client.list_objects_v2(Bucket=client_bucket +# , Prefix="contracts_landing_zone/"+batch_id+"/", Delimiter='/') +file_list = ['Boilerplate_TX Amendment Mission Health Network effective_040114 MU.pdf', 'Custom_TX - MP AMENDMENT - MISSION HEALTH NETWORK - MU.pdf', + 'Delaware First Health_First State Homecare Agency_212260_7 MU.pdf', 'Molina Healthcare of Texas, Inc. Amendment 4 - HIX ACA__EFF 01012016_MU.pdf'] + +if st.button("Read the contracts from Path"): + # for obj in file_objects.get('Contents',[]): + # if not obj['Key'].endswith('/'): + # file_list.append(obj['Key'].split('/')[-1]) + df['Contract Name'] = file_list + # df['Request ID'] = range(len(file_list)) + # df['Contract ID'] = file_list + df['Unique Key'] = a + df['Pricing Before Carveouts'] = b + df['Contract Related'] = c + df['Provider'] = d + df['Timeline'] = e + df['Carveout Indicator'] = f + df['Carveout Methodology'] = g + dir_path = os.path.dirname(os.path.realpath(__file__)) + print(f'DEBUGGING: PWD= {dir_path}') + df.to_csv('temp1.csv', index=False) + +add_vertical_space(1) +df2 = pd.read_csv('temp1.csv') +edited_df = st.data_editor(df2) + +edited_df['REQUEST_USER'] = user_mail +edited_df['LATEST_FLAG BOOLEAN'] = True +edited_df['PIPELINE_KICKOFF_DATETIME'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") +edited_df['REQUEST_DATETIME'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + +@st.cache_data +def convert_df(df): + return df.to_csv(index=False).encode('utf-8') + +csv = convert_df(edited_df) +# edited_df = edited_df.reset_index() # make sure indexes pair with number of rows + +# additional_info = pd.DataFrame(columns=['REQUEST_ID','T_DRIVE_PATH','CLIENT_NAME' +# , 'GROUP_NAME', 'REQUEST_USERNAME', 'REQUEST_DATETIME']) +additional_info = pd.DataFrame(columns=['CLIENT_NAME', 'BATCH_ID', 'REQUEST_USERNAME', 'REQUEST_DATETIME']) +additional_info.loc[0] = [client, batch_id, st.session_state.user_info['mail'], datetime.now().strftime("%Y-%m-%d %H:%M:%S")] +st.write(additional_info) + +st.session_state.contract_count = 0 +contract_list = [] +for index, row in edited_df.iterrows(): + allow_run_for_contract = False + group_list = [] + if row['Unique Key']: + group_list.append('Unique Key') + allow_run_for_contract = True + if row['Pricing Before Carveouts']: + group_list.append('Pricing Before Carveouts') + allow_run_for_contract = True + if row['Contract Related']: + group_list.append('Contract Related') + allow_run_for_contract = True + if row['Provider']: + group_list.append('Provider') + allow_run_for_contract = True + if row['Timeline']: + group_list.append('Timeline') + allow_run_for_contract = True + if row['Carveout Indicator']: + group_list.append('Carveout Indicator') + allow_run_for_contract = True + if row['Carveout Methodology']: + group_list.append('Carveout Methodology') + allow_run_for_contract = True + if allow_run_for_contract: st.session_state.contract_count += 1 + entry_dict = { + "contract_name": row['Contract Name'], + "groups": group_list, + "contract_source_path": "contracts_landing_zone/"+batch_id+"/"+row['Contract Name'] + } + contract_list.append(entry_dict) + +myobj = { + "s3_bucket": client_bucket, + "batch_id": batch_id, + "client_name": client, + "username": user_mail, + "contract_list": contract_list +} + +buttons = st.columns([0.8, 0.2]) +with buttons[0]: + st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv') +with buttons[1]: + if st.button("Run Doczy.AI Pipeline"): + if not st.session_state.contract_count == len(edited_df): + st.error("Select at least one Group No. for every Contract") + else: + with st.spinner('Running...'): # Feedback to User while API endpoint sends response for DOC-342 + # csv_buf = StringIO() + # additional_info.to_csv(csv_buf, header=True, index=False) + # csv_buf.seek(0) + # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/request_submission.csv') + # csv_buf = StringIO() + # edited_df.to_csv(csv_buf, header=True, index=False) + # csv_buf.seek(0) + # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/contract_config.csv') + # try: + # save_to_sf('load_request_and_contract_submissions', request_submission_file_name = "request_submission.csv", contract_config_file_name = "contract_config.csv") + # except Exception as e: + # st.write(e) + # response = requests.post(doczy_pipeline, json = myobj) + # if response.status_code >= 200 and response.status_code < 300: + # st.write("Success") + # # st.write(myobj) + # else: + # st.write("Failed") + # st.write(response.text) + time.sleep(5) + st.write("Success!") + diff --git a/streamlit/local/local_interface_2.py b/streamlit/local/local_interface_2.py new file mode 100644 index 0000000..18702f3 --- /dev/null +++ b/streamlit/local/local_interface_2.py @@ -0,0 +1,228 @@ +import json + +import boto3 +from langchain.prompts import PromptTemplate +from langchain.embeddings.bedrock import BedrockEmbeddings +from langchain.llms.bedrock import Bedrock +from langchain_community.vectorstores import Chroma +# from constants import CHROMA_SETTINGS, EMBEDDING_MODEL_NAME, PERSIST_DIRECTORY, MODEL_ID, MODEL_BASENAME, SOURCE_DIRECTORY, USER_LIST +from langchain.chains import RetrievalQA + +import streamlit as st +from streamlit_extras.add_vertical_space import add_vertical_space +from streamlit_pdf_viewer import pdf_viewer # needs to be installed on the server +import os +import pandas as pd +import numpy as np +# import util +import anthropic +from pydantic import BaseModel +from typing import List +import re +import base64 +# from sf_conn import get_snowflake_conn + +REDIRECT_URI = 'https://doczydev.aarete.com:8502' +# user_list = USER_LIST + +st.set_page_config(layout = "wide") +# Sidebar contents +# with st.sidebar: +# st.title("Doczy.AI ™") +# st.markdown( +# """ +# ## About +# This app extracts data from contracts + +# """ +# ) +# add_vertical_space(15) +# # st.write("Doczy") + +_,c1= st.columns([5,1]) +# try: +# util.setup_page(REDIRECT_URI) +# except: +# st.write("SSO Failed") +# st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} +st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} +try: + c1.write(f"User: **{st.session_state.user_info['displayName']}**") + user_mail = st.session_state.user_info['mail'] +except KeyError as e: + st.write("Session Expired.") + st.stop() + +# remove below try except statement if comparison with actual vales is not required +# try: +# conn = get_snowflake_conn('STG') +# cur = conn.cursor() +# query = 'select * from "TRAINING_DATA_RAW"' +# cur.execute(query) +# field_values = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description]) +# field_values['Document_Name'] = field_values['DOCUMENT_NAME'] +# except: +# # field_values = pd.read_csv('contract_field_values.csv', encoding='unicode_escape', skipinitialspace=True) +# # field_values = field_values.loc[:, ~field_values.columns.str.contains('Unnamed:')] +# st.write("Conn failed, unable to fetch data from training data table in Snowflake") + +field_values = pd.read_csv('contract_field_values.csv', encoding='unicode_escape', skipinitialspace=True) +field_values = field_values.loc[:, ~field_values.columns.str.contains('Unnamed:')] + +# try: +# query = 'select * from "PROMPT_CONFIG"' +# cur.execute(query) +# fields = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description]) + +# fields.rename(columns={'FIELD_DESC': 'Field Name'}, inplace = True) +# fields.rename(columns={'PROMPT': 'Interrogation Question?'}, inplace = True) +# fields.rename(columns={'GROUP_ID': 'PRIORITY'}, inplace = True) +# fields.rename(columns={'FIELD_NAME': 'SF_DB_COL_NAME'}, inplace = True) +# fields.rename(columns={'FM_MODEL_ID': 'llm_selected'}, inplace = True) +# except Exception as e: +# st.write("Unable to fetch data from Snowflake: ",e) +# # fields = pd.read_csv('contract_fields.csv', encoding='unicode_escape', skipinitialspace=True) +# # fields = fields[~fields['SF_COL_NAME'].str.endswith('_PG', na=None)] + +fields = pd.read_csv('contract_fields.csv', encoding='unicode_escape', skipinitialspace=True) +# fields = fields[fields['SF_DB_COL_NAME'].str.endswith('_PG', na=None)] + +# change the code below if contract list is fetched from snowflake +s3_client = boto3.client('s3', + region_name="us-east-2" +) +bucket = 'doczy-dev-infra-textract' +# objects = s3_client.list_objects_v2(Bucket=bucket, Prefix="training-data/") +# file_list = [] +# for obj in objects['Contents']: +# if not obj['Key'].endswith('/'): +# file_list.append(obj['Key']) + +file_list = ['Contract_Training_Exercise_Pricing.pdf', 'Contract_Training_Exercise_SLA.pdf'] +contract_list = sorted(file_list) + +file_row = st.columns([0.2, 0.7, 0.1]) +with file_row[0]: + st.write("**Contract Name**") +with file_row[1]: + file_name = st.selectbox('Select a file', ['All'] + contract_list, label_visibility = "collapsed", index= None) # MODIFIED - Append 'All' in the front instead of at the end + +field_row = st.columns([0.2, 0.7, 0.1]) +with field_row[0]: + st.write("**Field Group**") +with field_row[1]: + field_group = st.selectbox('Field Group',('Unique Key', 'Contract Related', 'Pricing Before Carveouts - I' + , 'Pricing Before Carveouts - II', 'Carveout Indicator, Code Type and Code #s - I' + , 'Carveout Indicator, Code Type and Code #s - II', 'Carveout Indicator, Code Type and Code #s - III' + , 'Optimize Carving Indic.', 'Carveout Method - I', 'Carveout Method - II', 'Provider' + , 'Timeline'), label_visibility = "collapsed", index = None) + +if field_group == 'Unique Key': + fields = fields[fields['PRIORITY'] == 'A'] +elif field_group == 'Contract Related': + fields = fields[fields['PRIORITY'] == 'C'] +elif field_group == 'Pricing Before Carveouts - I': + fields = fields[fields['PRIORITY'] == 'B'] + fields = np.array_split(fields, 2)[0] +elif field_group == 'Pricing Before Carveouts - II': + fields = fields[fields['PRIORITY'] == 'B'] + fields = np.array_split(fields, 2)[1] +elif field_group == 'Carveout Indicator, Code Type and Code #s - I': + fields = fields[fields['PRIORITY'] == 'F'] + fields = np.array_split(fields, 3)[0] +elif field_group == 'Carveout Indicator, Code Type and Code #s - II': + fields = fields[fields['PRIORITY'] == 'F'] + fields = np.array_split(fields, 3)[1] +elif field_group == 'Carveout Indicator, Code Type and Code #s - III': + fields = fields[fields['PRIORITY'] == 'F'] + fields = np.array_split(fields, 3)[2] +elif field_group == 'Carveout Methodology - I': + fields = fields[fields['PRIORITY'] == 'G'] + fields = np.array_split(fields, 4)[0] +elif field_group == 'Carveout Methodology - II': + fields = fields[fields['PRIORITY'] == 'G'] + fields = np.array_split(fields, 4)[1] +elif field_group == 'Carveout Method - III': + fields = fields[fields['PRIORITY'] == 'G'] + fields = np.array_split(fields, 4)[2] +elif field_group == 'Carveout Method - IV': + fields = fields[fields['PRIORITY'] == 'G'] + fields = np.array_split(fields, 4)[3] +elif field_group == 'Provider': + fields = fields[fields['PRIORITY'] == 'D'] +elif field_group == 'Timeline': + fields = fields[fields['PRIORITY'] == 'E'] + +if st.button("Show Results"): + # query = 'select * from "DOCZY_PIPELINE_RAW_OUTPUT"' + # cur.execute(query) + # df2 = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description]) + + # get this dataframe from snowflake table + df2 = pd.DataFrame(columns=['Contract Name','Field Name', 'SF_DB_COL_NAME', 'Snippet','Page Number' + , 'Field Extracted Value', 'Actual Value','Imputed Value']) + df2.to_csv('temp2.csv', index=False) + +if st.button("Show PDF"): + if file_name == None or file_name == "All": + st.error("Choose one specific file.") + else: + with st.sidebar: + st.markdown( + """ + + """, + unsafe_allow_html=True, + ) + pdf_viewer(file_name, width=1500) + +# if st.button("Show PDF"): +# if file_name == None or file_name == "All": +# st.error("Choose one specific file.") +# else: +# with st.sidebar: +# with open(file_name, "rb") as f: +# base64_pdf = base64.b64encode(f.read()).decode('utf-8') +# # Embedding PDF in HTML +# pdf_display = F'' +# # Displaying File +# st.markdown( +# """ +# +# """, +# unsafe_allow_html=True, +# ) +# st.markdown(pdf_display, unsafe_allow_html=True) + +df2 = pd.read_csv('temp2.csv') +df2['Imputed Value'] = '' +edited_df = st.data_editor(df2) + +@st.cache_data +def convert_df(df): + return df.to_csv(index=False).encode('utf-8') + +csv = convert_df(edited_df) + +buttons = st.columns(3) +with buttons[0]: + # st.button("Save All Imputations") + st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv') +with buttons[1]: + # st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv') + st.write("") +with buttons[2]: + if st.button("Kickoff Database Integration"): + st.write("Stored in DB") + + + + diff --git a/terminal.py b/terminal.py new file mode 100644 index 0000000..0b0f224 --- /dev/null +++ b/terminal.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +import os +import re +import subprocess +import logging +import signal +from typing import Callable, Optional, Dict, Tuple, List + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.INFO) +handler = logging.StreamHandler() +handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')) +LOGGER.addHandler(handler) + + +class SensitiveFormatter(logging.Formatter): + """Formatter that removes sensitive information from logs.""" + + @staticmethod + def mask_password(log: str) -> str: + return re.sub(r'password=([^\s]+)', r'password=*****', log) + + @staticmethod + def mask_api_key(log: str) -> str: + return re.sub(r'api_key=([^\s]+)', r'api_key=*****', log) + + @staticmethod + def _mask(s: str) -> str: + filtered = SensitiveFormatter.mask_password(s) + filtered = SensitiveFormatter.mask_api_key(filtered) + return filtered + + def format(self, record) -> str: + original = super().format(record) + return self._mask(original) + + +def prepare_logger() -> logging.Logger: + global LOGGER + if LOGGER is not None: + return LOGGER + LOGGER = logging.getLogger(__name__) + LOGGER.setLevel(logging.INFO) + log_format = '%(asctime)s %(filename)s:%(lineno)-4s [%(levelname)s] %(message)s' + handler = logging.StreamHandler() + handler.setFormatter(SensitiveFormatter(log_format)) + LOGGER.addHandler(handler) + return LOGGER + + +def get_blue_shade(log_prefix: str) -> str: + """Returns an ANSI color code for a shade of blue based on the hash of the log_prefix.""" + blue_shades = [81, 87, 117, 153, 159, 195, 111, 45, 39] + hash_value = hash(log_prefix) + blue_index = hash_value % len(blue_shades) + return f"\033[38;5;{blue_shades[blue_index]}m" + + +def signal_handler(sig, frame, process): + print("Ctrl+C pressed! Sending SIGTERM to Terraform process...") + process.terminate() + + +def run_command(command: str, log_output: bool = False, decorate_logs: bool = True, log_cmd: bool = False, + log_prefix: str = "", envs: Optional[str] = None, secrets: Optional[Dict[str, str]] = None, + failure_callback: Optional[Callable[[str], None]] = None, + cwd: Optional[str] = None) -> Tuple[int, List[str]]: + new_env = os.environ.copy() + if secrets: + for key, value in secrets.items(): + new_env[key] = os.path.expandvars(value) + + full_command = f"{envs} {command}" if envs else command + process = subprocess.Popen(full_command, shell=True, cwd=cwd, env=new_env, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + + # Setup signal handler + original_sigint_handler = signal.getsignal(signal.SIGINT) + signal.signal(signal.SIGINT, lambda sig, frame: signal_handler(sig, frame, process)) + + log_prefix_env = os.getenv("LOG_PREFIX", "") + log_prefix_full = f"[{log_prefix_env}] {log_prefix}" if log_prefix_env else log_prefix + + if log_cmd: + LOGGER.info( + f"{get_blue_shade(log_prefix_full)}{log_prefix_full}\033[0m: Running command: \033[1;34m{command}\033[0m") + if envs: + LOGGER.info(f"Using additional envs: \033[1;34m{envs}\033[0m") + if secrets: + LOGGER.info(f"Using additional environment variables with secrets: \033[1;34m{list(secrets.keys())}\033[0m") + + cmd_output = [] + while True: + output = process.stdout.readline() if process.stdout else '' + if output: + output_line = output.strip() + if log_output: + log_statement = ( + f"{get_blue_shade(log_prefix_full)}{log_prefix_full}\033[0m: " + f"{output_line}") if log_prefix else output_line + LOGGER.info(log_statement) if decorate_logs else print(log_statement) + cmd_output.append(output_line) + elif process.poll() is not None: + break + + # Restore original signal handler + signal.signal(signal.SIGINT, original_sigint_handler) + + rc = process.poll() or 0 # Ensure rc is an int, default to 0 if None + if rc != 0 and failure_callback: + failure_callback(f"Command returned code: {rc}") + + return rc, cmd_output + + +def decorate_successful(text: str) -> str: + return f"\033[32;1m{text}\033[0m" + + +def decorate_white_bold(text: str) -> str: + return f"\033[37;1m{text}\033[0m" + + +def decorate_warn(text: str) -> str: + return f"\033[33;1m{text}\033[0m" diff --git a/terraform-approach/src/lambda-layer/libreoffice/lo.tar.zip b/terraform-approach/src/lambda-layer/libreoffice/lo.tar.zip deleted file mode 100644 index 6a2b9a6..0000000 Binary files a/terraform-approach/src/lambda-layer/libreoffice/lo.tar.zip and /dev/null differ diff --git a/terraform-approach/src/lambda/textract-receiver/index.py b/terraform-approach/src/lambda/textract-receiver/index.py deleted file mode 100644 index 1f11d80..0000000 --- a/terraform-approach/src/lambda/textract-receiver/index.py +++ /dev/null @@ -1,3 +0,0 @@ -# AWS Lambda handler function -def lambda_handler(event, context): - print("Receiver Lambda") \ No newline at end of file diff --git a/terraform-approach/src/lambda/textract-sender/index.py b/terraform-approach/src/lambda/textract-sender/index.py deleted file mode 100644 index 015fbee..0000000 --- a/terraform-approach/src/lambda/textract-sender/index.py +++ /dev/null @@ -1,3 +0,0 @@ -# AWS Lambda handler function -def lambda_handler(event, context): - print("Sender Lambda") \ No newline at end of file diff --git a/terraform-approach/terraform/main.tf b/terraform-approach/terraform/main.tf deleted file mode 100644 index 8e8b181..0000000 --- a/terraform-approach/terraform/main.tf +++ /dev/null @@ -1,42 +0,0 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - version = "5.39.0" - } - } - - backend "s3" { - bucket = "doczy-test-bucket" # Parameterize using -backend-config flag with "terraform init" - key = "terraform/terraform_new2.tfstate" # Parameterize - region = "us-east-2" # Parameterize - # profile = "doczyai" # Parameterize - dynamodb_table = "terraform-lock" # Parameterize - encrypt = true - } -} - -provider "aws" { - profile = var.aws_profile -} - -module "textract_pipeline" { - source = "./modules/aws-textract-pipeline" - project_name = var.project_name - aws_region = var.aws_region - environment = var.environment - - # for each for multiple resources - for_each = var.client_list - - client_name = each.value.client_name - s3_bucket_name = each.value.s3_bucket_name - - lambda_role = "arn:aws:iam::660131068782:role/doczy-dev-infra-lambda-universal-role" - -} - - -# List - multiple client, separate state files -# client abbrv -# stream-wise folders or repo \ No newline at end of file diff --git a/terraform-approach/terraform/modules/aws-textract-pipeline-part2/main.tf b/terraform-approach/terraform/modules/aws-textract-pipeline-part2/main.tf deleted file mode 100644 index 1da8a80..0000000 --- a/terraform-approach/terraform/modules/aws-textract-pipeline-part2/main.tf +++ /dev/null @@ -1,212 +0,0 @@ -locals { - region_map = { - us-east-1 = "use1" - us-east-2 = "use2" - us-west-1 = "usw1" - us-west-2 = "usw2" - } - environment_map = { - prod = "p" - uat = "u" - qa = "q" - dev = "d" - } - - common_tags = { - environment = var.environment - - } - - # region abbreviation - region_short_name = local.region_map[var.aws_region] - - # environment abbreviation - environment_short_name = local.environment_map[var.environment] - - # global prefix - global_prefix = "${var.project_name}-${local.region_short_name}-${local.environment_short_name}-${var.client_name}" - - # Resource prefix - lambda_prefix = "${local.global_prefix}-lmb" - sqs_prefix = "${local.global_prefix}-sqs" - sns_prefix = "${local.global_prefix}-sns" - - # Lambda names - sender_lambda_name = "${local.lambda_prefix}-${var.textract_sender_lambda.name}" - receiver_lambda_name = "${local.lambda_prefix}-${var.textract_receiver_lambda.name}" - - # SQS names - sender_queue_name = "${local.sqs_prefix}-${var.sender_sqs_queue.name}" - receiver_queue_name = "${local.sqs_prefix}-${var.receiver_sqs_queue.name}" - - # SNS names - textract_notification_sns_name = "${local.sns_prefix}-textract-notification" - - # property file path - property_file_s3_path = "${var.s3_bucket_name}/config.properties" -} - -# sender lambda function -module "textract_sender_lambda_function" { - source = "terraform-aws-modules/lambda/aws" - version = "7.2.2" - - function_name = local.sender_lambda_name - description = var.textract_sender_lambda.description - handler = var.textract_sender_lambda.handler - runtime = var.textract_sender_lambda.runtime - timeout = var.textract_sender_lambda.timeout - memory_size = var.textract_sender_lambda.memory_size - ephemeral_storage_size = var.textract_sender_lambda.ephemeral_storage_size - architectures = var.textract_sender_lambda.architectures - - environment_variables = { - PROPERTY_FILE_S3_PATH = local.property_file_s3_path - } - - source_path = var.textract_sender_lambda.source_path - - event_source_mapping = { - sqs = { - event_source_arn = module.textract_sender_sqs.queue_arn - function_response_types = ["ReportBatchItemFailures"] - scaling_config = { - maximum_concurrency = 20 - - } - batch_size = 1 - } - } - - lambda_role = var.lambda_role - create_role = false - - tags = local.common_tags -} - -# Sender SQS Queue -module "textract_sender_sqs" { - source = "terraform-aws-modules/sqs/aws" - version = "4.1.1" - name = local.sender_queue_name - delay_seconds = var.sender_sqs_queue.delay_seconds - max_message_size = var.sender_sqs_queue.max_message_size - message_retention_seconds = var.sender_sqs_queue.message_retention_seconds - visibility_timeout_seconds= var.sender_sqs_queue.visibility_timeout_seconds - tags = local.common_tags -} - -# receiver lambda function -module "textract_receiver_lambda_function" { - source = "terraform-aws-modules/lambda/aws" - version = "7.2.2" - - function_name = local.receiver_lambda_name - description = var.textract_receiver_lambda.description - handler = var.textract_receiver_lambda.handler - runtime = var.textract_receiver_lambda.runtime - timeout = var.textract_receiver_lambda.timeout - memory_size = var.textract_receiver_lambda.memory_size - ephemeral_storage_size = var.textract_receiver_lambda.ephemeral_storage_size - architectures = var.textract_receiver_lambda.architectures - - environment_variables = { - PROPERTY_FILE_S3_PATH = local.property_file_s3_path - } - - source_path = var.textract_receiver_lambda.source_path - - event_source_mapping = { - sqs = { - event_source_arn = module.textract_receiver_sqs.queue_arn - function_response_types = ["ReportBatchItemFailures"] - scaling_config = { - maximum_concurrency = 20 - } - batch_size = 1 - } - } - - lambda_role = var.lambda_role - create_role = false - - tags = local.common_tags -} - -# textract notification sns -module "textract_notification_sns" { - source = "terraform-aws-modules/sns/aws" - version = ">= 5.0" - - name = local.textract_notification_sns_name - - topic_policy_statements = { - sqs = { - sid = "SQSSubscribe" - actions = [ - "sns:Subscribe", - "sns:Receive", - ] - - principals = [{ - type = "AWS" - identifiers = ["*"] - }] - - conditions = [{ - test = "StringLike" - variable = "sns:Endpoint" - values = [module.textract_receiver_sqs.queue_arn] - }] - } - } - - subscriptions = { - sqs = { - protocol = "sqs" - endpoint = module.textract_receiver_sqs.queue_arn - } - } - - tags = { - Environment = "dev" - } -} - -# Receiver SQS Queue -module "textract_receiver_sqs" { - source = "terraform-aws-modules/sqs/aws" - version = "4.1.1" - name = local.receiver_queue_name - delay_seconds = var.receiver_sqs_queue.delay_seconds - max_message_size = var.receiver_sqs_queue.max_message_size - message_retention_seconds = var.receiver_sqs_queue.message_retention_seconds - visibility_timeout_seconds= var.receiver_sqs_queue.visibility_timeout_seconds - tags = local.common_tags - - create_queue_policy = true - queue_policy_statements = { - sns = { - sid = "SNSPublish" - actions = ["sqs:SendMessage"] - - principals = [ - { - type = "Service" - identifiers = ["sns.amazonaws.com"] - } - ] - - conditions = [{ - test = "ArnEquals" - variable = "aws:SourceArn" - values = [module.textract_notification_sns.topic_arn] - }] - } - } -} - - - - - diff --git a/terraform-approach/terraform/modules/aws-textract-pipeline-part2/outputs.tf b/terraform-approach/terraform/modules/aws-textract-pipeline-part2/outputs.tf deleted file mode 100644 index 52923fc..0000000 --- a/terraform-approach/terraform/modules/aws-textract-pipeline-part2/outputs.tf +++ /dev/null @@ -1,26 +0,0 @@ -# Output variable definitions - -output "sender_lambda_arn" { - description = "Sender lambda arn" - value = module.textract_sender_lambda_function.lambda_function_arn -} - -output "receiver_lambda_arn" { - description = "Receiver lambda arn" - value = module.textract_receiver_lambda_function.lambda_function_arn -} - -output "sender_queue_arn" { - description = "Sender queue arn" - value = module.textract_sender_sqs.queue_arn -} - -output "receiver_queue_arn" { - description = "Receiver queue arn" - value = module.textract_receiver_sqs.queue_arn -} - -output "textract_notification_sns_arn" { - description = "value" - value = module.textract_notification_sns.topic_arn -} \ No newline at end of file diff --git a/terraform-approach/terraform/modules/aws-textract-pipeline-part2/variables.tf b/terraform-approach/terraform/modules/aws-textract-pipeline-part2/variables.tf deleted file mode 100644 index a29e909..0000000 --- a/terraform-approach/terraform/modules/aws-textract-pipeline-part2/variables.tf +++ /dev/null @@ -1,130 +0,0 @@ -# Required -variable "project_name" { - type = string -} - -# Required -variable "client_name" { - type = string -} - - -# Required -variable "environment" { - type = string -} - -# Required -variable "aws_region" { - type = string -} - -# Required -variable "lambda_role" { - type = string -} - -# Required -variable "s3_bucket_name" { - type = string -} - -# textract_sender_lambda -variable "textract_sender_lambda" { - type = object({ - name = string - description = string - handler = string - runtime = string - timeout = number - memory_size = number - ephemeral_storage_size = number - architectures = list(string) - layers = list(string) - environment_variables = map(string) - source_path = string - }) - default = { - name = "textract-sender" - description = "This function sends async request to TEXTRACT service" - handler = "index.lambda_handler" - runtime = "python3.12" - timeout = 60 - memory_size = 256 - ephemeral_storage_size = 512 - architectures = ["x86_64"] - layers = [] - environment_variables = { - PROPERTY_FILE_S3_PATH = "doczy-dev-infra-textract/config.properties" - } - source_path = "../src/lambda/textract-sender" - } -} - -# textract sender queue -variable "sender_sqs_queue" { - type = object({ - name = string - delay_seconds = number - max_message_size = number - message_retention_seconds = number - visibility_timeout_seconds= number - }) - default = { - name = "textract-sender-queue" - delay_seconds = 90 - max_message_size = 2048 - message_retention_seconds = 86400 - visibility_timeout_seconds= 901 - } -} - -# textract_receiver_lambda -variable "textract_receiver_lambda" { - type = object({ - name = string - description = string - handler = string - runtime = string - timeout = number - memory_size = number - ephemeral_storage_size = number - architectures = list(string) - layers = list(string) - environment_variables = map(string) - source_path = string - }) - default = { - name = "textract-receiver" - description = "This function receive async request to TEXTRACT service" - handler = "index.lambda_handler" - runtime = "python3.12" - timeout = 60 - memory_size = 256 - ephemeral_storage_size = 512 - architectures = ["x86_64"] - layers = [] - environment_variables = { - PROPERTY_FILE_S3_PATH = "doczy-dev-infra-textract/config.properties" - } - source_path = "../src/lambda/textract-receiver" - } -} - -# textract receiver queue -variable "receiver_sqs_queue" { - type = object({ - name = string - delay_seconds = number - max_message_size = number - message_retention_seconds = number - visibility_timeout_seconds= number - }) - default = { - name = "textract-receiver-queue" - delay_seconds = 90 - max_message_size = 2048 - message_retention_seconds = 86400 - visibility_timeout_seconds= 901 - } -} diff --git a/terraform-approach/terraform/outputs.tf b/terraform-approach/terraform/outputs.tf deleted file mode 100644 index 7caebf7..0000000 --- a/terraform-approach/terraform/outputs.tf +++ /dev/null @@ -1,25 +0,0 @@ -# Output variable definitions - -output "sender_lambda_arn" { - description = "Sender lambda arn" - value = [ for output_list in module.textract_pipeline : output_list.sender_lambda_arn] -} - -output "receiver_lambda_arn" { - description = "Receiver lambda arn" - value = [ for output_list in module.textract_pipeline : output_list.receiver_lambda_arn] -} - -output "sender_queue_arn" { - description = "Sender queue arn" - value = [ for output_list in module.textract_pipeline : output_list.sender_queue_arn] -} - -output "receiver_queue_arn" { - description = "Receiver queue arn" - value = [ for output_list in module.textract_pipeline : output_list.receiver_queue_arn] -} - -output "textract_notification_sns_arn" { - value = [ for output_list in module.textract_pipeline : output_list.textract_notification_sns_arn] -} \ No newline at end of file diff --git a/terraform-approach/terraform/variables.tf b/terraform-approach/terraform/variables.tf deleted file mode 100644 index 440eeda..0000000 --- a/terraform-approach/terraform/variables.tf +++ /dev/null @@ -1,37 +0,0 @@ -variable "aws_profile" { - type = string - # default = "doczyai" -} - -variable "aws_region" { - type = string - default = "us-east-2" -} - -variable "project_name" { - type = string - default = "doczyai" -} - -variable "environment" { - type = string - default = "dev" -} - -variable "client_list" { - type = map(object({ - client_name = string - s3_bucket_name = string - })) - default = { - client1 = { - s3_bucket_name = "doczy-dev-infra-textract" - client_name = "cn1" - }, - client2 = { - s3_bucket_name = "doczy-dev-infra-textract" - client_name = "cn2" - } - - } -} \ No newline at end of file diff --git a/terraform.py b/terraform.py new file mode 100644 index 0000000..0d6faaf --- /dev/null +++ b/terraform.py @@ -0,0 +1,227 @@ +from terminal import run_command, prepare_logger + +import os +import argparse +import sys +import subprocess +import time +from concurrent.futures import ThreadPoolExecutor, as_completed + +script_dir = os.path.dirname(os.path.abspath(__file__)) + +LOGGER = prepare_logger() + +LOGGER.info(sys.argv) + +parser = argparse.ArgumentParser(description='Run Terraform init with dynamic backend configuration.') +parser.add_argument('terraform_command', type=str, choices=['plan', 'apply'], + help='The Terraform command to execute (plan, apply, init).') +parser.add_argument('--environment', type=str, required=True, help='The environment name i.e. dev, prod.') +parser.add_argument('--module', type=str, required=False, help='The module name.') +parser.add_argument('--init-extra-args', type=str, required=False, help='Terraform init extra arguments') +parser.add_argument('--plan-extra-args', type=str, required=False, help='Terraform plan extra arguments') +parser.add_argument('--apply-extra-args', type=str, required=False, help='Terraform plan extra arguments') +parser.add_argument('--threads', type=int, default=4, help='Number of threads to use (default: 4)') + +args = parser.parse_args() + +raw_terraform_command = args.terraform_command +environment = args.environment +init_extra_args = args.init_extra_args +plan_extra_args = args.plan_extra_args +apply_extra_args = args.apply_extra_args +terraform_threads = args.threads +module_name = args.module + +bitbucket_ci = os.getenv('BITBUCKET_CI') + + +def change_dir(path): + if path: + _, git_root = run_command('git rev-parse --show-toplevel', log_cmd=True, log_output=True) + LOGGER.info(f"git_root={git_root}") + absolute_path = os.path.join(git_root[0], path) + return absolute_path + # os.chdir(absolute_path) + else: + LOGGER.error("TF_ROOT_MODULE_PATH environment variable is not set.") + sys.exit(1) + + +def select_vars_file(): + tf_vars_file = os.getenv('TFVARS_FILE_PATH', f'vars-{environment}.tfvars') + if tf_vars_file: + LOGGER.info(f"Using {tf_vars_file} as vars-file...") + else: + LOGGER.error("TFVARS_FILE_PATH variable is not set.") + sys.exit(1) + return tf_vars_file + + +def prepare_tf_init_command(stack_path, init_extra_args, state_environment, tf_state_bucket_name): + command = (f'terraform init --reconfigure ' + f'-backend-config="dynamodb_table=doczyai-use2-{get_environment_value(environment)}-infra-dyd-terraform-lock" ' + f'-backend-config="bucket=doczyai-use2-{get_environment_value(environment)}-infra-s3-terraform-state" ' + f'-backend-config="key=terraform/{stack_path}/terraform.tfstate" -backend-config="region=us-east-2"') + if init_extra_args: + command = f"{command} {init_extra_args}" + return command + + +def get_environment_value(environment): + environment_map = { + 'prod': 'p', + 'uat': 'u', + 'qa': 'q', + 'dev': 'd' + } + + return environment_map.get(environment, None) # Returns None if the environment is not found + + +def prepare_tf_main_command(environment, stack_path, state_environment, tf_state_bucket_name, vars_file, + plan_extra_args, + apply_extra_args): + common_command = f'-var-file=vars-{environment}.tfvars -var "aws_region=us-east-2" -var "environment={environment}"' + if raw_terraform_command == "plan": + path_part = stack_path.split('/')[1] if '/' in stack_path else stack_path + command = f"terraform plan -out .{path_part}.{environment}.tfplan {common_command}" + if plan_extra_args: + command = f"{command} {plan_extra_args}" + elif raw_terraform_command == "apply": + command = f"terraform apply -auto-approve {common_command}" + if apply_extra_args: + command = f"{command} {apply_extra_args}" + return command + + +tf_root_module_path = os.getenv('TF_ROOT_MODULE_PATH') if not module_name else module_name + + +def do_terraform(stack_path): + LOGGER.info(f"stack_path={stack_path}") + absolute_path = change_dir(stack_path) + # TF INIT + state_environment = "dev-new" if environment == "dev" else environment + tf_state_bucket_name = f"pi2new-{state_environment}-terraform-state-file" + + terraform_init_command = prepare_tf_init_command(stack_path, init_extra_args, state_environment, + tf_state_bucket_name) + rc_init, init_result = run_command(terraform_init_command, log_output=True, log_cmd=True, log_prefix=stack_path, + cwd=absolute_path) + + + if rc_init > 0: + raise Exception(f"{stack_path}: Failed to run terraform init") + + vars_file = select_vars_file() + terraform_main_command = prepare_tf_main_command(environment, stack_path, state_environment, tf_state_bucket_name, + vars_file, plan_extra_args, apply_extra_args) + + rc_main, command_result = run_command(terraform_main_command, log_output=True, log_cmd=True, + log_prefix=stack_path, cwd=absolute_path) + if rc_init or rc_main > 0: + raise RuntimeError(f"Unexpected Terraform error for command: {terraform_main_command}") + else: + return True, "OK" + + # return True, "OK" + + +def find_paths_with_file(file_name): + try: + completed_process = subprocess.run(['git', 'ls-files', f'*{file_name}'], check=True, text=True, + stdout=subprocess.PIPE) + + paths = set() + for file_path in completed_process.stdout.splitlines(): + dir_path = '/'.join(file_path.split('/')[:-1]) + if dir_path.startswith("modules/"): + continue + if dir_path.startswith("infra/"): + dir_path = dir_path[len("infra/"):] + if dir_path.startswith("kinesis") or dir_path.startswith("aws-transfer-sftp"): + paths.add(dir_path) + + return paths + except subprocess.CalledProcessError as e: + LOGGER.error(f"Error executing git command: {e}") + return set() + + +def read_dependencies(path): + depends_on_file = os.path.join(path, ".depends-on.txt") + if os.path.exists(depends_on_file): + with open(depends_on_file, "r") as f: + dependencies = [line.strip() for line in f.readlines()] + return dependencies + return [] + + +def wait_for_dependencies(dependencies, completed, failed, path): + """ + Waits for all dependencies to be satisfied before returning. + Now also checks for failed dependencies and raises an exception if found. + """ + while dependencies: + pending_dependencies = [dep for dep in dependencies if dep not in completed] + if any(dep in failed for dep in pending_dependencies): + failed_deps = [dep for dep in pending_dependencies if dep in failed] + raise Exception(f"{path}: Cannot proceed. Dependencies failed: {failed_deps}") + if not pending_dependencies: + return + LOGGER.warning(f"{path}: Waiting for dependencies to complete: {pending_dependencies}") + time.sleep(5) + + +def execute_with_dependencies(paths): + completed = set() # Track completed paths + errored_modules = set() # Track paths of modules where an error occurred + failed = {} # Track failures + + with ThreadPoolExecutor(max_workers=terraform_threads) as executor: + future_to_path = {} + + for path in paths: + dependencies = read_dependencies(path) + if dependencies: + # Submit a future for waiting dependencies + dep_future = executor.submit(wait_for_dependencies, dependencies, completed, failed, path) + # Submit the main task to be executed after dependencies are resolved + future = executor.submit(lambda p=dep_future, x=path: do_terraform(x) if p.result() is None else None) + else: + future = executor.submit(do_terraform, path) + future_to_path[future] = path + + for future in as_completed(future_to_path): + path = future_to_path[future] + try: + result = future.result() + if result is None or not result[0]: # Assuming do_terraform returns (success: bool, message: str) + errored_modules.add(path) + failed[path] = "Failed due to internal error" # Mark as failed + else: + completed.add(path) # Mark as completed + LOGGER.info(f"Thread result for {path}: {result}") + except Exception as exc: + errored_modules.add(path) + failed[path] = str(exc) # Mark as failed + LOGGER.error(f"{path} generated an exception: {exc}") + + # Check if any errors occurred and print the errored module paths + if errored_modules: + LOGGER.error("One or more modules failed to process correctly.") + for module_path in errored_modules: + LOGGER.error(f"Module with error: {module_path}") + sys.exit(1) + + +# Your main execution logic here +if __name__ == "__main__": + if tf_root_module_path: + do_terraform(tf_root_module_path) + else: + paths = find_paths_with_file('provider.tf') + for path in paths: + LOGGER.warning(f"Discovered: {path}") + execute_with_dependencies(paths) diff --git a/terraform/evironments/dev/main.tf b/terraform/evironments/dev/main.tf deleted file mode 100644 index e69de29..0000000 diff --git a/terraform/evironments/dev/outputs.tf b/terraform/evironments/dev/outputs.tf deleted file mode 100644 index e69de29..0000000 diff --git a/terraform/evironments/dev/variables.tf b/terraform/evironments/dev/variables.tf deleted file mode 100644 index e69de29..0000000 diff --git a/terraform/evironments/prod/backend.tf b/terraform/evironments/prod/backend.tf deleted file mode 100644 index e69de29..0000000 diff --git a/terraform/evironments/prod/main.tf b/terraform/evironments/prod/main.tf deleted file mode 100644 index e69de29..0000000 diff --git a/terraform/evironments/prod/outputs.tf b/terraform/evironments/prod/outputs.tf deleted file mode 100644 index e69de29..0000000 diff --git a/terraform/evironments/prod/variables.tf b/terraform/evironments/prod/variables.tf deleted file mode 100644 index e69de29..0000000 diff --git a/terraform/evironments/uat/backend.tf b/terraform/evironments/uat/backend.tf deleted file mode 100644 index e69de29..0000000 diff --git a/terraform/evironments/uat/main.tf b/terraform/evironments/uat/main.tf deleted file mode 100644 index e69de29..0000000 diff --git a/terraform/evironments/uat/outputs.tf b/terraform/evironments/uat/outputs.tf deleted file mode 100644 index e69de29..0000000 diff --git a/terraform/evironments/uat/variables.tf b/terraform/evironments/uat/variables.tf deleted file mode 100644 index e69de29..0000000 diff --git a/terraform/global/state/backend.tf b/terraform/global/state/backend.tf deleted file mode 100644 index e69de29..0000000 diff --git a/terraform/global/state/main.tf b/terraform/global/state/main.tf deleted file mode 100644 index e69de29..0000000 diff --git a/terraform/modules/airflow/main.tf b/terraform/modules/airflow/main.tf deleted file mode 100644 index e69de29..0000000 diff --git a/terraform/modules/airflow/variables.tf b/terraform/modules/airflow/variables.tf deleted file mode 100644 index e69de29..0000000 diff --git a/terraform/modules/iam/main.tf b/terraform/modules/iam/main.tf deleted file mode 100644 index e69de29..0000000 diff --git a/terraform/modules/iam/variables.tf b/terraform/modules/iam/variables.tf deleted file mode 100644 index e69de29..0000000 diff --git a/terraform/modules/lambda/main.tf b/terraform/modules/lambda/main.tf deleted file mode 100644 index e69de29..0000000 diff --git a/terraform/modules/lambda/variables.tf b/terraform/modules/lambda/variables.tf deleted file mode 100644 index e69de29..0000000 diff --git a/terraform/modules/s3/main.tf b/terraform/modules/s3/main.tf deleted file mode 100644 index e69de29..0000000 diff --git a/terraform/modules/s3/variables.tf b/terraform/modules/s3/variables.tf deleted file mode 100644 index e69de29..0000000 diff --git a/terraform/modules/sns_sqs/main.tf b/terraform/modules/sns_sqs/main.tf deleted file mode 100644 index e69de29..0000000 diff --git a/terraform/modules/sns_sqs/variables.tf b/terraform/modules/sns_sqs/variables.tf deleted file mode 100644 index e69de29..0000000 diff --git a/terraform/modules/textract/main.tf b/terraform/modules/textract/main.tf deleted file mode 100644 index e69de29..0000000 diff --git a/terraform/modules/textract/variables.tf b/terraform/modules/textract/variables.tf deleted file mode 100644 index e69de29..0000000 diff --git a/terraform/textract-approach3/src/lambda-layer/libreoffice/lo.tar.zip b/terraform/textract-approach3/src/lambda-layer/libreoffice/lo.tar.zip deleted file mode 100644 index 6a2b9a6..0000000 Binary files a/terraform/textract-approach3/src/lambda-layer/libreoffice/lo.tar.zip and /dev/null differ diff --git a/terraform/textract-approach3/src/lambda/textract-receiver/index.py b/terraform/textract-approach3/src/lambda/textract-receiver/index.py deleted file mode 100644 index 1f11d80..0000000 --- a/terraform/textract-approach3/src/lambda/textract-receiver/index.py +++ /dev/null @@ -1,3 +0,0 @@ -# AWS Lambda handler function -def lambda_handler(event, context): - print("Receiver Lambda") \ No newline at end of file diff --git a/terraform/textract-approach3/src/lambda/textract-sender/index.py b/terraform/textract-approach3/src/lambda/textract-sender/index.py deleted file mode 100644 index 015fbee..0000000 --- a/terraform/textract-approach3/src/lambda/textract-sender/index.py +++ /dev/null @@ -1,3 +0,0 @@ -# AWS Lambda handler function -def lambda_handler(event, context): - print("Sender Lambda") \ No newline at end of file diff --git a/textract-pipeline/src/lambda/prompt-orchestrator/index.py b/textract-pipeline/src/lambda/prompt-orchestrator/index.py index cfacc59..77503e6 100644 --- a/textract-pipeline/src/lambda/prompt-orchestrator/index.py +++ b/textract-pipeline/src/lambda/prompt-orchestrator/index.py @@ -13,12 +13,12 @@ sqs = boto3.client('sqs') # Initialize S3 & Textract clients s3_client = boto3.client('s3') -def lambda_handler(event, context): +def lambda_handler(event, context): try: # Read environment variables property_file_path = os.environ.get('PROPERTY_FILE_S3_PATH', '') - + # Read config.properties file_path_array = property_file_path.split("/") @@ -36,14 +36,14 @@ def lambda_handler(event, context): for record in event['Records']: # Extract the message body from the record record_body = json.loads(record['body']) - #logger.info('Message Count: ', str(len(record_body['Records'])) ) + # logger.info('Message Count: ', str(len(record_body['Records'])) ) for sqs_record in record_body['Records']: # Construct the source paths source_path = unquote_plus(sqs_record['s3']['object']['key']) document_id = get_filename_from_path(source_path) document_id = os.path.splitext(document_id)[0] - - prompt_config_query = "SELECT DISTINCT PC.FIELD_NAME,LENGTH(PC.PROMPT) as PROMPT_LENGTH, PC.FM_MODEL_ID,PC.GROUP_ID FROM PROMPT_CONFIG PC JOIN DOCUMENT_LOGS DL ON DL.GROUP_ID LIKE '%'||PC.GROUP_ID ||'%' AND DL.DOCUMENT_ID = '"+document_id+"' ORDER BY PROMPT_LENGTH" + + prompt_config_query = "SELECT DISTINCT PC.FIELD_NAME,LENGTH(PC.PROMPT) as PROMPT_LENGTH, PC.FM_MODEL_ID,PC.GROUP_ID FROM PROMPT_CONFIG PC JOIN DOCUMENT_LOGS DL ON DL.GROUP_ID LIKE '%'||PC.GROUP_ID ||'%' AND DL.DOCUMENT_ID = '" + document_id + "' ORDER BY PROMPT_LENGTH" field_Prompt_group_response = call_snowflake_db_select_lambda(prompt_config_query) # Maximum prompt length per group @@ -56,7 +56,11 @@ def lambda_handler(event, context): for i, (key, batch_list) in enumerate(batches.items(), 1): group_id, fm_model_id = key for j, batch in enumerate(batch_list, 1): - send_batch_to_sqs(batch,"https://sqs.us-east-2.amazonaws.com/660131068782/doczy-prompt-processor-sqs-dev",S3_BUCKET_NAME+"/"+source_path,i) + AWS_REGION_PARAM = os.getenv('AWS_REGION_PARAM') + AWS_ACCOUNT_ID = os.getenv('AWS_ACCOUNT_ID') + ENVIRONMENT = os.getenv('ENVIRONMENT') + sqs_url = f"https://sqs.{AWS_REGION_PARAM}.amazonaws.com/{AWS_ACCOUNT_ID}/doczy-prompt-processor-sqs-{ENVIRONMENT}" + send_batch_to_sqs(batch, sqs_url, S3_BUCKET_NAME + "/" + source_path, i) return { 'statusCode': 200, 'body': json.dumps('Message sent to SQS queue') @@ -68,12 +72,12 @@ def lambda_handler(event, context): return { 'statusCode': 500, 'body': error_message - } - + } + + # Function to retrieve configuration values from S3 + -# Function to retrieve configuration values from S3 def load_config_from_s3(bucket_name, file_key): - # Download the config file from S3 response = s3_client.get_object(Bucket=bucket_name, Key=file_key) config_content = response['Body'].read().decode('utf-8') @@ -89,12 +93,13 @@ def load_config_from_s3(bucket_name, file_key): return config_dict + def get_filename_from_path(full_path): return os.path.basename(full_path) + # Function to get s3 object tags def get_s3_object_tags(bucket_name, object_key): - try: # Get object tags response = s3_client.get_object_tagging( @@ -110,15 +115,15 @@ def get_s3_object_tags(bucket_name, object_key): except Exception as e: logger.exception(f"Error: {e}") return None - + def call_snowflake_db_select_lambda(query): # Create a Lambda client lambda_client = boto3.client('lambda') - + # Specify your Lambda function name lambda_function_name = 'doczy-dev-get-snowflake-logs' - + # Prepare payload payload = { 'query': query @@ -130,14 +135,15 @@ def call_snowflake_db_select_lambda(query): InvocationType='RequestResponse', # Synchronous invocation Payload=json.dumps(payload) ) - + # Return response body response_body = response['Payload'].read().decode('utf-8') return response_body except Exception as e: logger.exception(f"Error: {e}") return None - + + def extract_group_id(json_string): try: logger.info(f"json_string: {json_string}") @@ -146,13 +152,14 @@ def extract_group_id(json_string): group_id = body[0]['GROUP_ID'] group_ids = group_id.split(',') group_ids_quoted = ["'" + group_id.strip() + "'" for group_id in group_ids] - + # Join the elements back together with commas return ','.join(group_ids_quoted) except (json.JSONDecodeError, KeyError, IndexError) as e: logger.error("Error parsing JSON or extracting GROUP_ID:", e) return None - + + def extract_body_tags(json_data): try: # Parse the JSON string @@ -167,14 +174,13 @@ def extract_body_tags(json_data): logger.info(f"tags_list: {tags_list}") return tags_list - + except (json.JSONDecodeError, KeyError) as e: logger.error("Error parsing JSON or extracting 'body' tag:", e) return None def divide_into_batches(data, max_prompt_length_per_group): - try: # Parse the input data data_dict = json.loads(data) @@ -189,7 +195,7 @@ def divide_into_batches(data, max_prompt_length_per_group): # Divide data into batches based on group_id and fm_model_id for item in body_dict: - + group_id = item["GROUP_ID"] fm_model_id = item["FM_MODEL_ID"] prompt_length = item["PROMPT_LENGTH"] @@ -212,8 +218,7 @@ def divide_into_batches(data, max_prompt_length_per_group): # Retrieve the current batch (last sub-batch) current_batch = batches[(group_id, fm_model_id)][-1] - - + # Append the current item to the current batch current_batch.append(item) # Update prompt length for the current group and model id @@ -227,16 +232,16 @@ def divide_into_batches(data, max_prompt_length_per_group): raise e -def send_batch_to_sqs(batch, queue_url,source_path,prompt_batch_id): +def send_batch_to_sqs(batch, queue_url, source_path, prompt_batch_id): # Initialize SQS client sqs = boto3.client('sqs') # Convert batch to JSON string - batch_json = json.dumps(batch) + batch_json = json.dumps(batch) try: result = {"field_list": batch_json, - "file_name": source_path, - "prompt_batch_id" : prompt_batch_id + "file_name": source_path, + "prompt_batch_id": prompt_batch_id } logger.info(f"Message sent to SQS queue: {json.dumps(result)}") # Send message to SQS queue @@ -250,4 +255,4 @@ def send_batch_to_sqs(batch, queue_url,source_path,prompt_batch_id): except Exception as e: logger.error('Error sending message to SQS queue:', e) - return False \ No newline at end of file + return False diff --git a/textract-pipeline/terraform/main.tf b/textract-pipeline/terraform/main.tf index 3ab3b0f..d3f8114 100644 --- a/textract-pipeline/terraform/main.tf +++ b/textract-pipeline/terraform/main.tf @@ -40,9 +40,9 @@ locals { uat = { vpc_id = "vpc-0392396d0e7bdd77f" - secret_manager_name = "" - devops_s3_bucket_name = "" - api_access_required_roles_arns = [] + secret_manager_name = "doczy-dev-db-svc-acc" + devops_s3_bucket_name = "doczyai-use2-u-infra-s3-devops-resources" + api_access_required_roles_arns = ["arn:aws:iam::975049960860:role/doczyai-use2-u-rol-streamlit-server"] } } @@ -54,7 +54,7 @@ locals { # global prefix global_prefix = "${var.project_name}-${local.region_short_name}-${local.environment_short_name}-infra" - + # Resource prefix iam_role_prefix = "${local.global_prefix}-rol" @@ -68,10 +68,6 @@ locals { } } provider "aws" { - profile = var.aws_profile - - # access_key = var.access_key - # secret_key = var.secret_key default_tags { tags = local.common_tags @@ -86,6 +82,8 @@ module "textract_pipeline" { client_name = each.value.client_name + aws_account_id = var.aws_account_id + project_name = var.project_name aws_region = var.aws_region environment = var.environment @@ -99,6 +97,8 @@ module "textract_pipeline" { textract_role_name = aws_iam_role.textract_role.name lambda_role_name = aws_iam_role.lambda_role.name + + text_creation_lambda_additional_layer_arn = var.text_creation_lambda_additional_layer_arn } # Create lambda_role diff --git a/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf b/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf index a30f011..b0e8a3c 100644 --- a/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf +++ b/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/main.tf @@ -25,12 +25,12 @@ locals { # global prefix global_prefix = "${var.project_name}-${local.region_short_name}-${local.environment_short_name}-${var.client_name}" - + # Resource prefix lambda_prefix = "${local.global_prefix}-lmb" api_gateway_prefix = "${local.global_prefix}-api" lambda_layer_prefix = "${local.global_prefix}-lyr" - + # Lambda names s3_folder_details_lambda_name = "${local.lambda_prefix}-${var.s3_folder_detail_lambda.name}" batch_creation_lambda_name = "${local.lambda_prefix}-${var.batch_creation_lambda.name}" @@ -42,7 +42,7 @@ locals { sender_lambda_name = "${local.lambda_prefix}-${var.textract_sender_lambda.name}" receiver_lambda_name = "${local.lambda_prefix}-${var.textract_receiver_lambda.name}" text_creation_lambda_name = "${local.lambda_prefix}-${var.text_creation_lambda.name}" - prompt_orchestrator_lambda_name = "${local.lambda_prefix}-${var.prompt_orchestrator_lambda.name}" + prompt_orchestrator_lambda_name = "${local.lambda_prefix}-${local.prompt_orchestrator_lambda.name}" prompt_processor_lambda_name = "${local.lambda_prefix}-${var.prompt_processor_lambda.name}" call_bedrock_lambda_name = "${local.lambda_prefix}-${var.call_bedrock_lambda.name}" database_interface_lambda_name = "${local.lambda_prefix}-${var.database_interface_lambda.name}" @@ -183,7 +183,7 @@ module "s3_folder_detail_lambda_function" { memory_size = var.s3_folder_detail_lambda.memory_size ephemeral_storage_size = var.s3_folder_detail_lambda.ephemeral_storage_size architectures = var.s3_folder_detail_lambda.architectures - + environment_variables = {} source_path = var.s3_folder_detail_lambda.source_path @@ -210,7 +210,7 @@ module "batch_creation_lambda_function" { memory_size = var.batch_creation_lambda.memory_size ephemeral_storage_size = var.batch_creation_lambda.ephemeral_storage_size architectures = var.batch_creation_lambda.architectures - + environment_variables = local.environment_variables source_path = var.batch_creation_lambda.source_path @@ -237,7 +237,7 @@ module "trigger_pipeline_lambda_function" { memory_size = var.trigger_pipeline_lambda.memory_size ephemeral_storage_size = var.trigger_pipeline_lambda.ephemeral_storage_size architectures = var.trigger_pipeline_lambda.architectures - + environment_variables = merge(local.environment_variables,{ INITIATE_DB_SQS_URL = var.insert_record_queue_url }) @@ -266,7 +266,7 @@ module "insert_record_lambda_function" { memory_size = var.insert_record_lambda.memory_size ephemeral_storage_size = var.insert_record_lambda.ephemeral_storage_size architectures = var.insert_record_lambda.architectures - + environment_variables = local.environment_variables source_path = var.insert_record_lambda.source_path @@ -277,7 +277,7 @@ module "insert_record_lambda_function" { function_response_types = ["ReportBatchItemFailures"] scaling_config = { maximum_concurrency = 20 - + } batch_size = 1 } @@ -310,7 +310,7 @@ module "docx_to_pdf_lambda_function" { module.lambda_layer_libre_office.lambda_layer_arn, module.lambda_layer_brotli.lambda_layer_arn, ] - + environment_variables = local.environment_variables source_path = var.docx_to_pdf_lambda.source_path @@ -321,7 +321,7 @@ module "docx_to_pdf_lambda_function" { function_response_types = ["ReportBatchItemFailures"] scaling_config = { maximum_concurrency = 20 - + } batch_size = 1 } @@ -349,7 +349,7 @@ module "tiff_to_pdf_lambda_function" { memory_size = var.tiff_to_pdf_lambda.memory_size ephemeral_storage_size = var.tiff_to_pdf_lambda.ephemeral_storage_size architectures = var.tiff_to_pdf_lambda.architectures - + environment_variables = local.environment_variables source_path = var.tiff_to_pdf_lambda.source_path @@ -360,7 +360,7 @@ module "tiff_to_pdf_lambda_function" { function_response_types = ["ReportBatchItemFailures"] scaling_config = { maximum_concurrency = 20 - + } batch_size = 1 } @@ -388,7 +388,7 @@ module "pdf_validation_lambda_function" { memory_size = var.pdf_validation_lambda.memory_size ephemeral_storage_size = var.pdf_validation_lambda.ephemeral_storage_size architectures = var.pdf_validation_lambda.architectures - + layers = [ module.lambda_layer_pypdf2.lambda_layer_arn ] @@ -403,7 +403,7 @@ module "pdf_validation_lambda_function" { function_response_types = ["ReportBatchItemFailures"] scaling_config = { maximum_concurrency = 20 - + } batch_size = 1 } @@ -431,7 +431,7 @@ module "textract_sender_lambda_function" { memory_size = var.textract_sender_lambda.memory_size ephemeral_storage_size = var.textract_sender_lambda.ephemeral_storage_size architectures = var.textract_sender_lambda.architectures - + environment_variables = merge(local.environment_variables,{ TEXTRACT_ROLE_ARN = var.textract_role_arn SNS_TOPIC_ARN = var.textract_sns_topic_arn @@ -445,7 +445,7 @@ module "textract_sender_lambda_function" { function_response_types = ["ReportBatchItemFailures"] scaling_config = { maximum_concurrency = 20 - + } batch_size = 1 } @@ -473,7 +473,7 @@ module "textract_receiver_lambda_function" { memory_size = var.textract_receiver_lambda.memory_size ephemeral_storage_size = var.textract_receiver_lambda.ephemeral_storage_size architectures = var.textract_receiver_lambda.architectures - + environment_variables = local.environment_variables source_path = var.textract_receiver_lambda.source_path @@ -484,7 +484,7 @@ module "textract_receiver_lambda_function" { function_response_types = ["ReportBatchItemFailures"] scaling_config = { maximum_concurrency = 20 - + } batch_size = 1 } @@ -514,9 +514,10 @@ module "text_creation_lambda_function" { architectures = var.text_creation_lambda.architectures layers = [ - module.lambda_layer_pypdf2.lambda_layer_arn + module.lambda_layer_pypdf2.lambda_layer_arn, + var.text_creation_lambda_additional_layer_arn ] - + environment_variables = local.environment_variables source_path = var.text_creation_lambda.source_path @@ -527,7 +528,7 @@ module "text_creation_lambda_function" { function_response_types = ["ReportBatchItemFailures"] scaling_config = { maximum_concurrency = 20 - + } batch_size = 1 } @@ -548,22 +549,22 @@ module "prompt_orchestrator_lambda_function" { version = "7.2.2" function_name = local.prompt_orchestrator_lambda_name - description = var.prompt_orchestrator_lambda.description - handler = var.prompt_orchestrator_lambda.handler - runtime = var.prompt_orchestrator_lambda.runtime - timeout = var.prompt_orchestrator_lambda.timeout - memory_size = var.prompt_orchestrator_lambda.memory_size - ephemeral_storage_size = var.prompt_orchestrator_lambda.ephemeral_storage_size - architectures = var.prompt_orchestrator_lambda.architectures + description = local.prompt_orchestrator_lambda.description + handler = local.prompt_orchestrator_lambda.handler + runtime = local.prompt_orchestrator_lambda.runtime + timeout = local.prompt_orchestrator_lambda.timeout + memory_size = local.prompt_orchestrator_lambda.memory_size + ephemeral_storage_size = local.prompt_orchestrator_lambda.ephemeral_storage_size + architectures = local.prompt_orchestrator_lambda.architectures layers = [] - - environment_variables = merge(local.environment_variables,{ + + environment_variables = merge(local.environment_variables, local.prompt_orchestrator_lambda.environment_variables, { PROMPT_PROCESSOR_QUEUE_URL = var.prompt_processor_queue_url MAX_PROMPT_LENGTH_PER_GROUP = "5000" }) - source_path = var.prompt_orchestrator_lambda.source_path + source_path = local.prompt_orchestrator_lambda.source_path event_source_mapping = { sqs = { @@ -571,7 +572,7 @@ module "prompt_orchestrator_lambda_function" { function_response_types = ["ReportBatchItemFailures"] scaling_config = { maximum_concurrency = 20 - + } batch_size = 1 enabled = false @@ -602,7 +603,7 @@ module "prompt_processor_lambda_function" { architectures = var.prompt_processor_lambda.architectures layers = [] - + environment_variables = merge(local.environment_variables,{ CALL_BEDROCK_LAMBDA_FUNCTION_NAME = local.call_bedrock_lambda_name }) @@ -615,7 +616,7 @@ module "prompt_processor_lambda_function" { function_response_types = ["ReportBatchItemFailures"] scaling_config = { maximum_concurrency = 20 - + } batch_size = 1 } @@ -645,7 +646,7 @@ module "call_bedrock_lambda_function" { architectures = var.call_bedrock_lambda.architectures layers = [] - + environment_variables = local.environment_variables source_path = var.call_bedrock_lambda.source_path @@ -672,7 +673,7 @@ module "database_interface_lambda_function" { memory_size = var.database_interface_lambda.memory_size ephemeral_storage_size = var.database_interface_lambda.ephemeral_storage_size architectures = var.database_interface_lambda.architectures - + environment_variables = merge(local.environment_variables,{ SECRET_MANAGER_NAME = var.secret_manager_name }) @@ -705,7 +706,7 @@ module "database_interface_get_lambda_function" { memory_size = var.database_interface_get_lambda.memory_size ephemeral_storage_size = var.database_interface_get_lambda.ephemeral_storage_size architectures = var.database_interface_get_lambda.architectures - + environment_variables = merge(local.environment_variables,{ SECRET_MANAGER_NAME = var.secret_manager_name }) @@ -775,7 +776,7 @@ resource "aws_s3_bucket_notification" "bucket_notification" { events = ["s3:ObjectCreated:*"] filter_prefix = "textract-output-json/" filter_suffix = ".json" - } + } queue { id = "text-file-upload-event" @@ -847,9 +848,9 @@ resource "aws_api_gateway_method" "create_batch_method" { http_method = "POST" authorization = "AWS_IAM" - depends_on = [ - aws_api_gateway_rest_api.project_api, - aws_api_gateway_resource.create_batch + depends_on = [ + aws_api_gateway_rest_api.project_api, + aws_api_gateway_resource.create_batch ] } @@ -863,8 +864,8 @@ resource "aws_api_gateway_method_response" "create_batch_method_response_200" { "application/json" = "Empty" } - depends_on = [ - aws_api_gateway_rest_api.project_api, + depends_on = [ + aws_api_gateway_rest_api.project_api, aws_api_gateway_resource.create_batch, aws_api_gateway_method.create_batch_method ] @@ -879,8 +880,8 @@ resource "aws_api_gateway_integration" "create_batch_integration" { type = "AWS" uri = module.batch_creation_lambda_function.lambda_function_invoke_arn - depends_on = [ - aws_api_gateway_rest_api.project_api, + depends_on = [ + aws_api_gateway_rest_api.project_api, aws_api_gateway_resource.create_batch, aws_api_gateway_method.create_batch_method, module.batch_creation_lambda_function @@ -894,8 +895,8 @@ resource "aws_api_gateway_integration_response" "create_batch_integration_respon http_method = aws_api_gateway_method.create_batch_method.http_method status_code = aws_api_gateway_method_response.create_batch_method_response_200.status_code - depends_on = [ - aws_api_gateway_rest_api.project_api, + depends_on = [ + aws_api_gateway_rest_api.project_api, aws_api_gateway_resource.create_batch, aws_api_gateway_method.create_batch_method, aws_api_gateway_method_response.create_batch_method_response_200, @@ -912,8 +913,8 @@ resource "aws_lambda_permission" "create_batch_lambda_permission" { principal = "apigateway.amazonaws.com" source_arn = "${aws_api_gateway_rest_api.project_api.execution_arn}/*/${aws_api_gateway_method.create_batch_method.http_method}${aws_api_gateway_resource.create_batch.path}" - depends_on = [ - aws_api_gateway_rest_api.project_api, + depends_on = [ + aws_api_gateway_rest_api.project_api, aws_api_gateway_resource.create_batch, aws_api_gateway_method.create_batch_method, module.batch_creation_lambda_function @@ -963,8 +964,8 @@ resource "aws_api_gateway_integration_response" "s3_folder_detail_integration_re http_method = aws_api_gateway_method.s3_folder_detail_method.http_method status_code = aws_api_gateway_method_response.s3_folder_detail_method_response_200.status_code - depends_on = [ - aws_api_gateway_rest_api.project_api, + depends_on = [ + aws_api_gateway_rest_api.project_api, aws_api_gateway_resource.s3_folder_detail, aws_api_gateway_method.s3_folder_detail_method, aws_api_gateway_method_response.s3_folder_detail_method_response_200, @@ -1023,8 +1024,8 @@ resource "aws_api_gateway_integration_response" "trigger_pipeline_integration_re http_method = aws_api_gateway_method.trigger_pipeline_method.http_method status_code = aws_api_gateway_method_response.trigger_pipeline_method_response_200.status_code - depends_on = [ - aws_api_gateway_rest_api.project_api, + depends_on = [ + aws_api_gateway_rest_api.project_api, aws_api_gateway_resource.trigger_pipeline, aws_api_gateway_method.trigger_pipeline_method, aws_api_gateway_method_response.trigger_pipeline_method_response_200, diff --git a/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/variables.tf b/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/variables.tf index 82718ad..4e7447b 100644 --- a/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/variables.tf +++ b/textract-pipeline/terraform/modules/aws-textract-pipeline-part3/variables.tf @@ -18,6 +18,11 @@ variable "aws_region" { type = string } +# Required +variable "aws_account_id" { + type = string +} + # Required variable "vpc_id" { type = string @@ -480,22 +485,15 @@ variable "text_creation_lambda" { } } +variable "text_creation_lambda_additional_layer_arn" { + type = string +} + # prompt-orchestrator lambda -variable "prompt_orchestrator_lambda" { - type = object({ - name = string - description = string - handler = string - runtime = string - timeout = number - memory_size = number - ephemeral_storage_size = number - architectures = list(string) - layers = list(string) - environment_variables = map(string) - source_path = string - }) - default = { + +# Use locals to merge default values with variable values +locals { + prompt_orchestrator_lambda_default = { name = "prompt-orchestrator" description = "This Lambda functions breaks prompts into chunks and send messages into SQS" handler = "index.lambda_handler" @@ -505,9 +503,15 @@ variable "prompt_orchestrator_lambda" { ephemeral_storage_size = 512 architectures = ["x86_64"] layers = [] - environment_variables = {} + environment_variables = { + AWS_REGION_PARAM = var.aws_region + AWS_ACCOUNT_ID = var.aws_account_id + ENVIRONMENT = var.environment + } source_path = "../src/lambda/prompt-orchestrator" } + + prompt_orchestrator_lambda = merge(local.prompt_orchestrator_lambda_default) } # prompt-processor lambda diff --git a/textract-pipeline/terraform/modules/aws-textract-pipeline/main.tf b/textract-pipeline/terraform/modules/aws-textract-pipeline/main.tf index 0925392..0489c33 100644 --- a/textract-pipeline/terraform/modules/aws-textract-pipeline/main.tf +++ b/textract-pipeline/terraform/modules/aws-textract-pipeline/main.tf @@ -52,6 +52,7 @@ module "textract_pipeline_part3" { project_name = var.project_name aws_region = var.aws_region + aws_account_id = var.aws_account_id environment = var.environment vpc_id = var.vpc_id secret_manager_name = var.secret_manager_name @@ -68,6 +69,7 @@ module "textract_pipeline_part3" { textract_sender_queue_arn = module.textract_pipeline_part1.sender_queue_arn textract_receiver_queue_arn = module.textract_pipeline_part1.receiver_queue_arn text_creation_queue_arn = module.textract_pipeline_part1.text_creation_queue_arn + text_creation_lambda_additional_layer_arn = var.text_creation_lambda_additional_layer_arn prompt_orchestrator_queue_arn = module.textract_pipeline_part1.prompt_orchestrator_queue_arn prompt_processor_queue_arn = module.textract_pipeline_part1.prompt_processor_queue_arn prompt_processor_queue_url = module.textract_pipeline_part1.prompt_processor_queue_url @@ -198,7 +200,7 @@ data "aws_iam_policy_document" "lambda_policy" { ] effect = "Allow" - resources = ["arn:aws:secretsmanager:*:660131068782:secret:*"] + resources = ["arn:aws:secretsmanager:*:${var.aws_account_id}:secret:*"] } } diff --git a/textract-pipeline/terraform/modules/aws-textract-pipeline/variables.tf b/textract-pipeline/terraform/modules/aws-textract-pipeline/variables.tf index ea91c51..9546ffe 100644 --- a/textract-pipeline/terraform/modules/aws-textract-pipeline/variables.tf +++ b/textract-pipeline/terraform/modules/aws-textract-pipeline/variables.tf @@ -56,4 +56,12 @@ variable "textract_role_name" { # Required variable "lambda_role_name" { type = string +} + +variable "aws_account_id" { + type = string +} + +variable "text_creation_lambda_additional_layer_arn" { + type = string } \ No newline at end of file diff --git a/textract-pipeline/terraform/variables.tf b/textract-pipeline/terraform/variables.tf index 276513c..73ee0fc 100644 --- a/textract-pipeline/terraform/variables.tf +++ b/textract-pipeline/terraform/variables.tf @@ -1,19 +1,3 @@ -# # required -# variable "secret_key" { -# type = string -# } - -# # required -# variable "access_key" { -# type = string -# } - - -# required -variable "aws_profile" { - type = string - # default = "doczyai" -} # required variable "aws_region" { type = string @@ -40,4 +24,12 @@ variable "client_list" { client_name = "cn1" } } +} + +variable "aws_account_id" { + type = string +} + +variable "text_creation_lambda_additional_layer_arn" { + type = string } \ No newline at end of file diff --git a/textract-pipeline/terraform/vars-dev.tfvars b/textract-pipeline/terraform/vars-dev.tfvars new file mode 100644 index 0000000..02453a3 --- /dev/null +++ b/textract-pipeline/terraform/vars-dev.tfvars @@ -0,0 +1,2 @@ +aws_account_id = 660131068782 +text_creation_lambda_additional_layer_arn = "arn:aws:lambda:us-east-2:336392948345:layer:AWSSDKPandas-Python312:8" \ No newline at end of file diff --git a/textract-pipeline/terraform/vars-uat.tfvars b/textract-pipeline/terraform/vars-uat.tfvars new file mode 100644 index 0000000..b6e909a --- /dev/null +++ b/textract-pipeline/terraform/vars-uat.tfvars @@ -0,0 +1,3 @@ +aws_account_id = 975049960860 +# TODO provide new layer arn for UAT? 336392948345 was used in DEV but not sure what account that is?! +text_creation_lambda_additional_layer_arn = "arn:aws:lambda:us-east-2:336392948345:layer:AWSSDKPandas-Python312:8"