diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 7b2856e..06cb48a 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -200,31 +200,49 @@ pipelines: name: "Plan & Apply DevOps Pipeline" image: hugree/bitbucket-aws-python38-tf154:latest oidc: true - trigger : automatic # Keeping it manual so that it doesnt conflict with existing resources + trigger : automatic script: - *aws-context-uat - cd devops-pipeline/terraform-backend-resources/ + - terraform init - terraform validate # Ensure the backend configuration for s3 obj key is correct - - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/devops-pipeline/terraform.tfstate" - - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" + - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat"- echo "S3_BUCKET=$(terraform output -raw terraform_s3_bucket_name)" >> backend.env + - echo "DYNAMODB_TABLE=$(terraform output -raw terraform_dynamodb_table_name)" >> backend.env + artifacts: + - backend.env condition: changesets: includePaths: - devops-pipeline/terraform-backend-resources/* - - devops-pipeline/terraform-backend-resources/**/* + - devops-pipeline/terraform-backend-resources/**/*- step: + name: "Plan & Apply DevOps Extra Resource Pipeline" + image: hugree/bitbucket-aws-python38-tf154:latest + oidc: true + trigger : automatic # Keeping it manual so that it doesnt conflict with existing resources + script: + - *aws-context-uat + - cd devops-pipeline/other-resources/ + - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/devops-pipeline-other-resources/terraform.tfstate" + - terraform validate + - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" + condition: + changesets: + includePaths: + - devops-pipeline/other-resources/* + - devops-pipeline/other-resources/**/* - step: name: "Plan & Apply Streamlit Pipeline" image: hugree/bitbucket-aws-python38-tf154:latest oidc: true - trigger : manual + trigger : automatic script: - *aws-context-uat - cd streamlit-server/ + - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/streamlit-server/terraform.tfstate" - terraform validate # Ensure the backend configuration for s3 obj key is correct - - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/streamlit-server/terraform.tfstate" - - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" + - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat"-var "vpc_id=vpc-0392396d0e7bdd77f" condition: changesets: includePaths: @@ -242,12 +260,12 @@ pipelines: image: python:3.8 name: "Deploy streamlit to EC2" oidc: true - trigger: manual # Keeping it manual as the server will be created first and then referenced in this + trigger: automatic # Keeping it manual as the server will be created first and then referenced in this script: - python -m pip install --upgrade pip - apt-get update && apt-get install -y jq git - pip install awscli - - *aws-context-dev + - *aws-context-uat - *ssm-send-command-uat condition: changesets: @@ -269,10 +287,10 @@ pipelines: script: - *aws-context-uat - cd textract-pipeline/terraform/ + - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/textract-pipeline/terraform.tfstate" - terraform validate # Ensure the backend configuration for s3 obj key is corresponding to the stream name - - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-u-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-u-infra-s3-terraform-state" -backend-config="key=terraform/textract-pipeline/terraform.tfstate" - - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" + - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=uat" condition: changesets: includePaths: diff --git a/devops-pipeline/other-resources/variables.tf b/devops-pipeline/other-resources/variables.tf index ac98867..97e452b 100644 --- a/devops-pipeline/other-resources/variables.tf +++ b/devops-pipeline/other-resources/variables.tf @@ -53,6 +53,7 @@ variable "mwaa_resources_s3_bucket" { } } + variable "aws_account_id" { type = string } \ No newline at end of file diff --git a/devops-pipeline/terraform-backend-resources/main.tf b/devops-pipeline/terraform-backend-resources/main.tf index 1dbd42c..56aec03 100644 --- a/devops-pipeline/terraform-backend-resources/main.tf +++ b/devops-pipeline/terraform-backend-resources/main.tf @@ -8,7 +8,7 @@ terraform { } provider "aws" { - region = var.aws_region + region = var.aws_region } module "aws_terraform_remote_backend" { diff --git a/devops-pipeline/terraform-backend-resources/modules/aws-terraform-remote-backend/variables.tf b/devops-pipeline/terraform-backend-resources/modules/aws-terraform-remote-backend/variables.tf index 48cfca4..3483195 100644 --- a/devops-pipeline/terraform-backend-resources/modules/aws-terraform-remote-backend/variables.tf +++ b/devops-pipeline/terraform-backend-resources/modules/aws-terraform-remote-backend/variables.tf @@ -10,6 +10,20 @@ variable "environment" { variable "client_name" { type = string } +# required +variable "secret_key" { + type = string +} + +# required +variable "access_key" { + type = string +} + +variable "token" { + type = string +} + # dynamoDB table variable "terraform_dynamodb_table" { type = object({ diff --git a/devops-pipeline/terraform-backend-resources/outputs.tf b/devops-pipeline/terraform-backend-resources/outputs.tf index 58a5ddb..2192baf 100644 --- a/devops-pipeline/terraform-backend-resources/outputs.tf +++ b/devops-pipeline/terraform-backend-resources/outputs.tf @@ -9,4 +9,4 @@ output "terraform_s3_bucket_arn" { } output "terraform_s3_bucket_name" { value = module.aws_terraform_remote_backend.terraform_s3_bucket_name -} \ No newline at end of file +} diff --git a/devops-pipeline/terraform-backend-resources/variables.tf b/devops-pipeline/terraform-backend-resources/variables.tf index b756d1b..3e52526 100644 --- a/devops-pipeline/terraform-backend-resources/variables.tf +++ b/devops-pipeline/terraform-backend-resources/variables.tf @@ -16,4 +16,19 @@ variable "environment" { variable "client_name" { type = string default = "infra" -} \ No newline at end of file +} + +#required +variable "secret_key" { + type = string +} + +#required +variable "access_key" { + type = string +} + +#required +variable "token" { + type = string +} diff --git a/snowflake/DEV/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql b/snowflake/DEV/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql index 4e5562c..bd1b22f 100644 --- a/snowflake/DEV/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql +++ b/snowflake/DEV/config_interface/R__004_LOAD_CONTRACT_CONFIG_SP.sql @@ -51,7 +51,7 @@ BEGIN batch_id := (SELECT NULLIF(TRIM($2),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1); request_datetime := (SELECT NULLIF(TRIM($13),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1); - UPDATE TABLE STG.CONTRACT_CONFIG + UPDATE STG.CONTRACT_CONFIG SET LATEST_FLAG = FALSE WHERE BATCH_ID = :batch_id AND CONTRACT_NAME = :contract_name AND REQUEST_DATETIME < :request_datetime AND LATEST_FLAG = TRUE; diff --git a/snowflake/UAT/R__1002_LOG_AUDIT_TABLE.sql b/snowflake/UAT/R__1002_LOG_AUDIT_TABLE.sql index 1cc0768..d3ce540 100644 --- a/snowflake/UAT/R__1002_LOG_AUDIT_TABLE.sql +++ b/snowflake/UAT/R__1002_LOG_AUDIT_TABLE.sql @@ -8,4 +8,4 @@ create or replace TABLE STG.LOG_AUDIT ( SUB_SECTION_NAME VARCHAR(255) COLLATE 'en-ci', AUDIT_SID NUMBER(38,0), TOTAL_MIN NUMBER(38,0) -); \ No newline at end of file +); diff --git a/snowflake/UAT/config_interface/R__1004_LOAD_CONTRACT_CONFIG_SP.sql b/snowflake/UAT/config_interface/R__1004_LOAD_CONTRACT_CONFIG_SP.sql index 4e5562c..e00a89c 100644 --- a/snowflake/UAT/config_interface/R__1004_LOAD_CONTRACT_CONFIG_SP.sql +++ b/snowflake/UAT/config_interface/R__1004_LOAD_CONTRACT_CONFIG_SP.sql @@ -50,8 +50,8 @@ BEGIN contract_name := (SELECT NULLIF(TRIM($1),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1); batch_id := (SELECT NULLIF(TRIM($2),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1); request_datetime := (SELECT NULLIF(TRIM($13),'') FROM @STG.CONTRACT_CONFIG_STAGE LIMIT 1); - - UPDATE TABLE STG.CONTRACT_CONFIG + + UPDATE STG.CONTRACT_CONFIG SET LATEST_FLAG = FALSE WHERE BATCH_ID = :batch_id AND CONTRACT_NAME = :contract_name AND REQUEST_DATETIME < :request_datetime AND LATEST_FLAG = TRUE; diff --git a/streamlit-server/alb.tf b/streamlit-server/alb.tf index a94a31c..48b6bdb 100644 --- a/streamlit-server/alb.tf +++ b/streamlit-server/alb.tf @@ -115,7 +115,8 @@ resource "aws_lb_listener" "https_8500" { protocol = "HTTPS" ssl_policy = "ELBSecurityPolicy-2016-08" # Using variable for ARN - certificate_arn = var.acm_arn_dev + certificate_arn = lookup(var.acm_arns, var.environment) + default_action { type = "forward" @@ -128,7 +129,7 @@ resource "aws_lb_listener" "https_8501" { port = "8501" protocol = "HTTPS" ssl_policy = "ELBSecurityPolicy-2016-08" - certificate_arn = var.acm_arn_dev + certificate_arn = lookup(var.acm_arns, var.environment) default_action { type = "forward" @@ -141,7 +142,7 @@ resource "aws_lb_listener" "https_8502" { port = "8502" protocol = "HTTPS" ssl_policy = "ELBSecurityPolicy-2016-08" - certificate_arn = var.acm_arn_dev + certificate_arn = lookup(var.acm_arns, var.environment) default_action { type = "forward" @@ -154,7 +155,7 @@ resource "aws_lb_listener" "https_8503" { port = "8503" protocol = "HTTPS" ssl_policy = "ELBSecurityPolicy-2016-08" - certificate_arn = var.acm_arn_dev + certificate_arn = lookup(var.acm_arns, var.environment) default_action { type = "forward" diff --git a/streamlit-server/main.tf b/streamlit-server/main.tf index 75d1352..0a82054 100644 --- a/streamlit-server/main.tf +++ b/streamlit-server/main.tf @@ -81,13 +81,13 @@ resource "aws_iam_instance_profile" "ec2_instance_profile" { } resource "aws_instance" "streamlit_server" { - ami = var.ubuntu_ami + ami = lookup(var.ubuntu_ami, var.environment) instance_type = var.ec2_instance_type subnet_id = data.aws_subnets.subnets.ids[0] vpc_security_group_ids = data.aws_security_groups.security_groups.ids iam_instance_profile = aws_iam_instance_profile.ec2_instance_profile.name associate_public_ip_address = false - key_name = "tf-test-key" # Created using TF and uploaded to S3. Prerequisite + key_name = "tf-uat-key" # Created using TF and uploaded to S3. Prerequisite root_block_device { volume_size = 30 # Variable @@ -101,7 +101,7 @@ resource "aws_instance" "streamlit_server" { apt-get update apt-get install -y python3 python3-pip sudo apt-get install libffi-dev - python3 -m pip3 install -r /tmp/requirements.txt + python3 -m pip install -r /tmp/requirements.txt sudo apt install openssh-client sudo apt install net-tools @@ -197,3 +197,4 @@ resource "aws_instance" "streamlit_server" { } + diff --git a/streamlit-server/variables.tf b/streamlit-server/variables.tf index c71c156..7e18a5f 100644 --- a/streamlit-server/variables.tf +++ b/streamlit-server/variables.tf @@ -30,10 +30,18 @@ variable "vpc_id" { type = string } +# variable "ubuntu_ami" { +# type = string +# default = "ami-0de7e97fedfbc6ef6" # Switch to this ami-0b8b44ec9a8f90422 + +# } + variable "ubuntu_ami" { - type = string - default = "ami-0de7e97fedfbc6ef6" # Switch to this ami-0b8b44ec9a8f90422 - + type = map(string) + default = { + dev = "ami-0de7e97fedfbc6ef6" + uat = "ami-0b8b44ec9a8f90422" + } } variable "ec2_instance_type" { @@ -52,6 +60,15 @@ variable "acm_arn_dev" { default = "arn:aws:acm:us-east-2:660131068782:certificate/947e04b3-be22-43bf-bea2-a0ad3b221d70" } +variable "acm_arns" { + type = map(string) + default = { + dev = "arn:aws:acm:us-east-2:660131068782:certificate/947e04b3-be22-43bf-bea2-a0ad3b221d70" + uat = "arn:aws:acm:us-east-2:975049960860:certificate/e036e76d-1f4f-496e-ac97-5f65ec4a34cc" + prod = "arn:aws:acm:region:account-id:certificate/prod-cert-id" + } +} + #variable "secret_key" { # type = string #} diff --git a/streamlit/interface_0.py b/streamlit/interface_0.py index f638209..656939d 100644 --- a/streamlit/interface_0.py +++ b/streamlit/interface_0.py @@ -18,6 +18,22 @@ create_batch_url = 'https://lfksus2t62.execute-api.us-east-2.amazonaws.com/dev/c REDIRECT_URI = 'https://doczydev.aarete.com:8500' user_list = USER_LIST +if 'uploading' not in st.session_state: + st.session_state.uploading = False + + + +def insert_upload_logs(batch_id, client_name, file_name, upload_datetime, upload_user): + """ + Input: batch_id, client_name, file_name, upload_datetime, upload_user + Output: status of the insert query + """ + try: + return 'Log inserted successfully' + except Exception as e: + return e + + st.set_page_config(layout = "wide") # # Sidebar contents # with st.sidebar: @@ -31,6 +47,7 @@ st.set_page_config(layout = "wide") # ) # add_vertical_space(15) # # st.write("Doczy") +# _,c1= st.columns([5,1]) try: @@ -45,7 +62,7 @@ except KeyError as e: st.write("Session Expired.") st.stop() - +print(st.session_state) s3_client = boto3.client('s3', region_name="us-east-2", @@ -65,7 +82,7 @@ client_row = st.columns([0.1, 0.8]) with client_row[0]: st.write("**Client Name**") with client_row[1]: - client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed") + client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344 client_bucket = client_s3_paths.get(client) @@ -76,42 +93,55 @@ file_row = st.columns([0.1, 0.8]) with file_row[0]: st.write("**Upload Files**") with file_row[1]: - file_list = st.file_uploader("Upload", type=None, accept_multiple_files=True, label_visibility = "collapsed") + file_list = st.file_uploader("Upload", type=['docx','tiff','pdf'], accept_multiple_files=True, label_visibility = "collapsed", help="Only PDF, TIFF and DOCX file formats are supported.", disabled=st.session_state.uploading) add_vertical_space(2) df = pd.DataFrame(columns=['Contract Name']) df['Contract Name'] = file_list file_names = [] buttons = st.columns([0.4, 0.4, 0.2]) -with buttons[1]: - if st.button("Create Batch"): - myobj = { "client-bucket-name": client_bucket } - response = requests.post(create_batch_url, json = myobj) - if response.status_code >= 200 and response.status_code < 300: - try: - batch_id = json.loads(json.loads(response.text)['body'])['batch_id'] - landing_zone = json.loads(json.loads(response.text)['body'])['landing_zone'] - except: - st.write(myobj) - st.write(response.text) - batch_id = 'failed_cases' - landing_zone = 'contracts_landing_zone' +def set_uploading_state(): + if not client == None and not len(file_list) == 0: + st.session_state.uploading = True + +with buttons[1]: + if st.button("Create Batch", on_click = set_uploading_state): + if client == None: + st.error("No Client Name Selected.") + elif len(file_list) == 0: + st.error("No Files Selected.") else: - st.write("Failed") - for uploaded_file in file_list: - stringio = BytesIO(uploaded_file.getvalue()) - stringio.seek(0) - s3_client.put_object(Bucket=client_bucket, Body=stringio.getvalue(), Key= - landing_zone+batch_id+'/'+str(uploaded_file.name)) - - # TODO: Test this insert function with snowflake - upload_log = insert_upload_logs(batch_id, client, str(uploaded_file.name), datetime.now().strftime("%Y-%m-%d %H:%M:%S"), user_mail) - st.write(upload_log) - - file_names.append(str(uploaded_file.name)) - st.write(f"{batch_id} created") - st.write(f"Files uploaded to s3://{client_bucket}/{landing_zone}{batch_id}") + myobj = { "client-bucket-name": client_bucket } + response = requests.post(create_batch_url, json = myobj) + if response.status_code >= 200 and response.status_code < 300: + try: + batch_id = json.loads(json.loads(response.text)['body'])['batch_id'] + landing_zone = json.loads(json.loads(response.text)['body'])['landing_zone'] + except: + st.write(myobj) + st.write(response.text) + batch_id = 'failed_cases' + landing_zone = 'contracts_landing_zone' + else: + st.write("Failed") + for uploaded_file in file_list: + stringio = BytesIO(uploaded_file.getvalue()) + stringio.seek(0) + s3_client.put_object(Bucket=client_bucket, Body=stringio.getvalue(), Key= + landing_zone+batch_id+'/'+str(uploaded_file.name)) + + # TODO: Test this insert function with snowflake + upload_log = insert_upload_logs(batch_id, client, str(uploaded_file.name), datetime.now().strftime("%Y-%m-%d %H:%M:%S"), user_mail) + st.write(upload_log) + + file_names.append(str(uploaded_file.name)) + st.session_state.uploading = False + st.write(f"{batch_id} created") + st.write(f"Files uploaded to s3://{client_bucket}/{landing_zone}{batch_id}") + + + # @st.cache_data diff --git a/streamlit/interface_1.py b/streamlit/interface_1.py index 3d3a695..5b3f82a 100644 --- a/streamlit/interface_1.py +++ b/streamlit/interface_1.py @@ -8,6 +8,7 @@ from datetime import datetime import boto3 import util import requests +import time from sf_conn import get_client_names, get_secret, save_to_sf from constants import USER_LIST, DOCZY_PIPELINE_URL_DEV @@ -61,10 +62,11 @@ client_row = st.columns([0.1, 0.8]) with client_row[0]: st.write("**Client Name**") with client_row[1]: - client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed") + client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344 client_bucket = client_s3_paths.get(client) -# # to be deleted when buckets for different clients are ready; below line is added only for testing the corresponding DAG + +# to be deleted when buckets for different clients are ready; below line is added only for testing the corresponding DAG client_bucket = 'doczy-ai-client-1' batch_objects = s3_client.list_objects_v2(Bucket=client_bucket @@ -74,17 +76,51 @@ batch_list = [] for prefix in batch_objects['CommonPrefixes']: batch_list.append(prefix['Prefix'][:-1].split('/')[-1]) +# Hardcoded batch_list for testing purposes +# batch_list = ['batch_020524103737', 'batch_090524131433', 'batch_090524131607', 'batch_100524123000', 'batch_130524064322', +# 'batch_160524071331', 'batch_200524213550', 'batch_250424112237', 'batch_280524120530', 'batch_280524121721', 'batch_280524144222', +# 'batch_290524123926', 'batch_290524164044', 'batch_310524102029', 'batch_310524124050', 'batch_310524162346', 'batch_310524162631'] + +if 'sorted_list' not in st.session_state: + st.session_state.sorted_list = batch_list + +def sort_list(ex_list, sort_by, order): + if sort_by == 'Alphabetical': + ex_list = sorted(ex_list, reverse=(order == 'Descending')) + elif sort_by == 'Create Date': + ex_list = ex_list if order == 'Ascending' else list(reversed(ex_list)) + + return ex_list + +col1, col2, col3, col4 = st.columns([0.5, 0.5, 0.5, 0.5]) + + +with col1: + sort_by = st.radio("**Sort Batch_IDs**", ('Alphabetical', 'Create Date')) + +with col2: + order = st.radio('', ('Ascending','Descending')) + +with col3: + add_vertical_space(2) + if st.button('Apply'): + st.session_state.sorted_list = sort_list(batch_list, sort_by, order) + path_row = st.columns([0.1, 0.8]) with path_row[0]: st.write("**Batch ID**") with path_row[1]: - batch_id = st.selectbox('**Batch ID**', batch_list, label_visibility = "collapsed") + batch_id = st.selectbox('**Batch ID**', st.session_state.sorted_list, label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344 + +if not batch_id: + batch_id = "None" checks = st.columns([0.1, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12]) with checks[0]: st.write("**Group No.**") + with checks[1]: - a = st.checkbox('Unique Key', key = str(1)) + a = st.checkbox('Unique Key', key = str(1), args="Unique") with checks[2]: b = st.checkbox('Pricing Before Carveouts', key = str(2)) with checks[3]: @@ -106,11 +142,15 @@ file_list = [] file_objects = s3_client.list_objects_v2(Bucket=client_bucket , Prefix="contracts_landing_zone/"+batch_id+"/", Delimiter='/') +# Hardcoded file_list for testing purposes +# file_list = ['Boilerplate_TX Amendment Mission Health Network effective_040114 MU.pdf', 'Custom_TX - MP AMENDMENT - MISSION HEALTH NETWORK - MU.pdf', +# 'Delaware First Health_First State Homecare Agency_212260_7 MU.pdf', 'Molina Healthcare of Texas, Inc. Amendment 4 - HIX ACA__EFF 01012016_MU.pdf'] + if st.button("Read the contracts from Path"): for obj in file_objects.get('Contents',[]): if not obj['Key'].endswith('/'): file_list.append(obj['Key'].split('/')[-1]) - + df['Contract Name'] = file_list # df['Request ID'] = range(len(file_list)) # df['Contract ID'] = file_list @@ -147,23 +187,33 @@ additional_info = pd.DataFrame(columns=['CLIENT_NAME', 'BATCH_ID', 'REQUEST_USER additional_info.loc[0] = [client, batch_id, st.session_state.user_info['mail'], datetime.now().strftime("%Y-%m-%d %H:%M:%S")] st.write(additional_info) +st.session_state.contract_count = 0 contract_list = [] for index, row in edited_df.iterrows(): + allow_run_for_contract = False group_list = [] if row['Unique Key']: group_list.append('Unique Key') + allow_run_for_contract = True if row['Pricing Before Carveouts']: group_list.append('Pricing Before Carveouts') + allow_run_for_contract = True if row['Contract Related']: group_list.append('Contract Related') + allow_run_for_contract = True if row['Provider']: group_list.append('Provider') + allow_run_for_contract = True if row['Timeline']: group_list.append('Timeline') + allow_run_for_contract = True if row['Carveout Indicator']: group_list.append('Carveout Indicator') + allow_run_for_contract = True if row['Carveout Methodology']: group_list.append('Carveout Methodology') + allow_run_for_contract = True + if allow_run_for_contract: st.session_state.contract_count += 1 entry_dict = { "contract_name": row['Contract Name'], "groups": group_list, @@ -184,23 +234,27 @@ with buttons[0]: st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv') with buttons[1]: if st.button("Run Doczy.AI Pipeline"): - # csv_buf = StringIO() - # additional_info.to_csv(csv_buf, header=True, index=False) - # csv_buf.seek(0) - # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/request_submission.csv') - # csv_buf = StringIO() - # edited_df.to_csv(csv_buf, header=True, index=False) - # csv_buf.seek(0) - # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/contract_config.csv') - # try: - # save_to_sf('load_request_and_contract_submissions', request_submission_file_name = "request_submission.csv", contract_config_file_name = "contract_config.csv") - # except Exception as e: - # st.write(e) - response = requests.post(doczy_pipeline, json = myobj) - if response.status_code >= 200 and response.status_code < 300: - st.write("Success") - # st.write(myobj) + if not st.session_state.contract_count == len(edited_df): + st.error("Select at least one Group No. for every Contract") else: - st.write("Failed") - # st.write(response.text) + with st.spinner('Running...'): + # csv_buf = StringIO() + # additional_info.to_csv(csv_buf, header=True, index=False) + # csv_buf.seek(0) + # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/request_submission.csv') + # csv_buf = StringIO() + # edited_df.to_csv(csv_buf, header=True, index=False) + # csv_buf.seek(0) + # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/contract_config.csv') + # try: + # save_to_sf('load_request_and_contract_submissions', request_submission_file_name = "request_submission.csv", contract_config_file_name = "contract_config.csv") + # except Exception as e: + # st.write(e) + response = requests.post(doczy_pipeline, json = myobj) + if response.status_code >= 200 and response.status_code < 300: + st.write("Success") + # st.write(myobj) + else: + st.write("Failed") + # st.write(response.text) diff --git a/streamlit/interface_2.py b/streamlit/interface_2.py index ed8c494..824509a 100644 --- a/streamlit/interface_2.py +++ b/streamlit/interface_2.py @@ -10,6 +10,7 @@ from langchain.chains import RetrievalQA import streamlit as st from streamlit_extras.add_vertical_space import add_vertical_space +from streamlit_pdf_viewer import pdf_viewer # needs to be installed on the server import os import pandas as pd import numpy as np @@ -18,24 +19,26 @@ import anthropic from pydantic import BaseModel from typing import List import re +import base64 from sf_conn import get_snowflake_conn +import io REDIRECT_URI = 'https://doczydev.aarete.com:8502' user_list = USER_LIST st.set_page_config(layout = "wide") # Sidebar contents -with st.sidebar: - st.title("Doczy.AI ™") - st.markdown( - """ - ## About - This app extracts data from contracts +# with st.sidebar: +# st.title("Doczy.AI ™") +# st.markdown( +# """ +# ## About +# This app extracts data from contracts - """ - ) - add_vertical_space(15) - # st.write("Doczy") +# """ +# ) +# add_vertical_space(15) +# # st.write("Doczy") _,c1= st.columns([5,1]) try: @@ -91,13 +94,11 @@ for obj in objects['Contents']: contract_list = sorted(file_list) - - file_row = st.columns([0.2, 0.7, 0.1]) with file_row[0]: st.write("**Contract Name**") with file_row[1]: - file_name = st.selectbox('Select a file', contract_list + ['All'], label_visibility = "collapsed") + file_name = st.selectbox('Select a file', ['All'] + contract_list, label_visibility = "collapsed", index= None) # MODIFIED - Append 'All' in the front instead of at the end field_row = st.columns([0.2, 0.7, 0.1]) with field_row[0]: @@ -107,7 +108,7 @@ with field_row[1]: , 'Pricing Before Carveouts - II', 'Carveout Indicator, Code Type and Code #s - I' , 'Carveout Indicator, Code Type and Code #s - II', 'Carveout Indicator, Code Type and Code #s - III' , 'Optimize Carving Indic.', 'Carveout Method - I', 'Carveout Method - II', 'Provider' - , 'Timeline'), label_visibility = "collapsed") + , 'Timeline'), label_visibility = "collapsed", index = None) if field_group == 'Unique Key': fields = fields[fields['PRIORITY'] == 'A'] @@ -151,10 +152,51 @@ if st.button("Show Results"): df2 = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description]) # get this dataframe from snowflake table - # df2 = pd.DataFrame(columns=['Contract Name','Field Name', 'SF_DB_COL_NAME', 'Snippet','Page Number' - # , 'Field Extracted Value', 'Actual Value','Imputed Value']) + df2 = pd.DataFrame(columns=['Contract Name','Field Name', 'SF_DB_COL_NAME', 'Snippet','Page Number' + , 'Field Extracted Value', 'Actual Value','Imputed Value']) df2.to_csv('temp2.csv', index=False) +if st.button("Show PDF"): + if file_name == None or file_name == "All": + st.error("Choose one specific file.") + else: + with st.sidebar: + st.markdown( + """ + + """, + unsafe_allow_html=True, + ) + s3_client.Object(bucket,file_name) + data=obj.get()['Body'].read() + pdf_viewer(io.BytesIO(data), width=1500) + +# if st.button("Show PDF"): +# if file_name == None or file_name == "All": +# st.error("Choose one specific file.") +# else: +# with st.sidebar: +# with open(file_name, "rb") as f: +# base64_pdf = base64.b64encode(f.read()).decode('utf-8') +# # Embedding PDF in HTML +# pdf_display = F'' +# # Displaying File +# st.markdown( +# """ +# +# """, +# unsafe_allow_html=True, +# ) +# st.markdown(pdf_display, unsafe_allow_html=True) + df2 = pd.read_csv('temp2.csv') df2['Imputed Value'] = '' edited_df = st.data_editor(df2) diff --git a/streamlit/local/local_interface_0.py b/streamlit/local/local_interface_0.py new file mode 100644 index 0000000..1c6a006 --- /dev/null +++ b/streamlit/local/local_interface_0.py @@ -0,0 +1,200 @@ +import json +import streamlit as st +from streamlit_extras.add_vertical_space import add_vertical_space +import os +import streamlit as st +import pandas as pd +from io import StringIO +from datetime import datetime +import boto3 +# import util +import requests +# from sf_conn import get_secret, save_to_sf +from io import StringIO, BytesIO +import time +# from sf_conn import get_client_names, insert_upload_logs +# from constants import USER_LIST + +create_batch_url = 'https://lfksus2t62.execute-api.us-east-2.amazonaws.com/dev/create-batch' + +# REDIRECT_URI = 'https://doczydev.aarete.com:8500' +# user_list = USER_LIST +if 'uploading' not in st.session_state: + st.session_state.uploading = False + + + +def insert_upload_logs(batch_id, client_name, file_name, upload_datetime, upload_user): + """ + Input: batch_id, client_name, file_name, upload_datetime, upload_user + Output: status of the insert query + """ + try: + return 'Log inserted successfully' + except Exception as e: + return e + + +st.set_page_config(layout = "wide") +# # # Sidebar contents +# # with st.sidebar: +# # st.title("Doczy.AI ™") +# # st.markdown( +# # """ +# # ## About +# # This app extracts data from contracts + +# # """ +# # ) +# # add_vertical_space(15) +# # # st.write("Doczy") +# # + +_,c1= st.columns([5,1]) +# try: +# util.setup_page(REDIRECT_URI) +# except Exception as e: +# st.write(f"SSO Failed = {e}") +# st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} +st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} +try: + c1.write(f"User: **{st.session_state.user_info['displayName']}**") + user_mail = st.session_state.user_info['mail'] +except KeyError as e: + st.write("Session Expired.") + st.stop() + +print(st.session_state) + +s3_client = boto3.client('s3', + region_name="us-east-2", +) + +client_list = ['doczy-ai-client-1', 'Delaware First Health, Inc.', 'Community Health Choice, Inc','CareSource Network Partners LLC', + 'HealthNet of Cali', 'Oklahoma Complete Health, Inc', 'HealthFirst', 'Molina Healthcare of TX', 'AvMed', 'Arizona Care1st', + 'WellCare New Jersey'] + +# This is the list of client fetched from Snowflake +# TODO: Need to update the streamlit code to use the client names from this list +# And use the s3 paths to save the objects for the respective client +# client_list, s3_paths = get_client_names() +# client_s3_paths = dict(zip(client_list, s3_paths)) + +client_row = st.columns([0.1, 0.8]) +with client_row[0]: + st.write("**Client Name**") +with client_row[1]: + client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344 + +# client_bucket = client_s3_paths.get(client) + +# to be deleted when client buckets are created +client_bucket = 'doczy-ai-client-1' + +file_row = st.columns([0.1, 0.8]) +with file_row[0]: + st.write("**Upload Files**") +with file_row[1]: + file_list = st.file_uploader("Upload", type=['docx','tiff','pdf'], accept_multiple_files=True, label_visibility = "collapsed", help="Only PDF, TIFF and DOCX file formats are supported.", disabled=st.session_state.uploading) + +add_vertical_space(2) +df = pd.DataFrame(columns=['Contract Name']) +df['Contract Name'] = file_list +file_names = [] +buttons = st.columns([0.4, 0.4, 0.2]) + +def set_uploading_state(): + if not client == None and not len(file_list) == 0: + st.session_state.uploading = True + +with buttons[1]: + if st.button("Create Batch", on_click = set_uploading_state): + if client == None: + st.error("No Client Name Selected.") + elif len(file_list) == 0: + st.error("No Files Selected.") + else: + time.sleep(5) + # myobj = { "client-bucket-name": client_bucket } + # response = requests.post(create_batch_url, json = myobj) + # if response.status_code >= 200 and response.status_code < 300: + # try: + # batch_id = json.loads(json.loads(response.text)['body'])['batch_id'] + # landing_zone = json.loads(json.loads(response.text)['body'])['landing_zone'] + # except: + # st.write(myobj) + # st.write(response.text) + # batch_id = 'failed_cases' + # landing_zone = 'contracts_landing_zone' + # else: + # st.write("Failed") + # for uploaded_file in file_list: + # stringio = BytesIO(uploaded_file.getvalue()) + # stringio.seek(0) + # s3_client.put_object(Bucket=client_bucket, Body=stringio.getvalue(), Key= + # landing_zone+batch_id+'/'+str(uploaded_file.name)) + + # # TODO: Test this insert function with snowflake + # upload_log = insert_upload_logs(batch_id, client, str(uploaded_file.name), datetime.now().strftime("%Y-%m-%d %H:%M:%S"), user_mail) + # st.write(upload_log) + + # file_names.append(str(uploaded_file.name)) + # st.write(f"{batch_id} created") + st.session_state.uploading = False + # st.write(f"Files uploaded to s3://{client_bucket}/{landing_zone}{batch_id}") + + + + + # @st.cache_data + # def convert_df(df): + # return df.to_csv(index=False).encode('utf-8') + + # csv = convert_df(df) + + # df = df.reset_index() # make sure indexes pair with number of rows + + # contract_list = [] + # for index, row in df.iterrows(): + # group_list = [] + # if row['Unique Key']: + # group_list.append('Unique Key') + # if row['Pricing Before Carveouts']: + # group_list.append('Pricing Before Carveouts') + # if row['Contract Related']: + # group_list.append('Contract Related') + # if row['Provider']: + # group_list.append('Provider') + # if row['Timeline']: + # group_list.append('Timeline') + # if row['Carveout Indicator']: + # group_list.append('Carveout Indicator') + # if row['Carveout Methodology']: + # group_list.append('Carveout Methodology') + # entry_dict = { + # "contract_name": row['Contract Name'], + # "groups": group_list, + # "contract_source_path": "batches/batch_1/"+client+"/"+row['Contract Name'] + # } + # contract_list.append(entry_dict) + # contract_list = list(df['Contract Name']) + + # myobj = { + # "s3_bucket": 'doczy-dev-infra-textract', + # "batch_id": "1", + # "client_name": client, + # "username": user_mail, + # "contract_list": contract_list + # } + + # buttons = st.columns([0.8, 0.2]) + # with buttons[0]: + # st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv') + # with buttons[1]: + # if st.button("Upload to DB"): + # response = requests.post(doczy_pipeline, json = myobj) + # if response.status_code >= 200 and response.status_code < 300: + # st.write("Success") + # else: + # st.write("Failed") + # # st.write(response.text) diff --git a/streamlit/local/local_interface_1.py b/streamlit/local/local_interface_1.py new file mode 100644 index 0000000..fd03888 --- /dev/null +++ b/streamlit/local/local_interface_1.py @@ -0,0 +1,259 @@ +import streamlit as st +from streamlit_extras.add_vertical_space import add_vertical_space +import os +import streamlit as st +import pandas as pd +from io import StringIO +from datetime import datetime +import boto3 +# import util +import requests +import time +# from sf_conn import get_client_names, get_secret, save_to_sf +# from constants import USER_LIST, DOCZY_PIPELINE_URL_DEV + +# doczy_pipeline = DOCZY_PIPELINE_URL_DEV + +# REDIRECT_URI = 'https://doczydev.aarete.com:8501' +# user_list = USER_LIST +st.set_page_config(layout = "wide") +# # # Sidebar contents +# # with st.sidebar: +# # st.title("Doczy.AI ™") +# # st.markdown( +# # """ +# # ## About +# # This app extracts data from contracts + +# # """ +# # ) +# # add_vertical_space(15) +# # # st.write("Doczy") + +_,c1= st.columns([5,1]) +# try: +# util.setup_page(REDIRECT_URI) +# except: +# st.write("SSO Failed") +# st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} +st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} +try: + c1.write(f"User: **{st.session_state.user_info['displayName']}**") + user_mail = st.session_state.user_info['mail'] +except KeyError as e: + # Do we add a link to get to the login page here? + st.write("Session Expired.") + st.stop() + + + +s3_client = boto3.client('s3', + region_name="us-east-2", +) + +# # to be replaced with snowflake data +client_list = ['doczy-ai-client-1', 'Delaware First Health, Inc.', 'Community Health Choice, Inc','CareSource Network Partners LLC', + 'HealthNet of Cali', 'Oklahoma Complete Health, Inc', 'HealthFirst', 'Molina Healthcare of TX', 'AvMed', 'Arizona Care1st', + 'WellCare New Jersey'] + +# client_list, s3_paths = get_client_names() +# client_s3_paths = dict(zip(client_list, s3_paths)) + +client_row = st.columns([0.1, 0.8]) +with client_row[0]: + st.write("**Client Name**") +with client_row[1]: + client = st.selectbox('Client Name',(client_list), label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344 + +# client_bucket = client_s3_paths.get(client) + +# # to be deleted when buckets for different clients are ready; below line is added only for testing the corresponding DAG +client_bucket = 'doczy-ai-client-1' + +# batch_objects = s3_client.list_objects_v2(Bucket=client_bucket +# , Prefix="contracts_landing_zone/", Delimiter='/') + +# batch_list = [] +# for prefix in batch_objects['CommonPrefixes']: +# batch_list.append(prefix['Prefix'][:-1].split('/')[-1]) + +batch_list = ['batch_020524103737', 'batch_090524131433', 'batch_090524131607', 'batch_100524123000', 'batch_130524064322', + 'batch_160524071331', 'batch_200524213550', 'batch_250424112237', 'batch_280524120530', 'batch_280524121721', 'batch_280524144222', + 'batch_290524123926', 'batch_290524164044', 'batch_310524102029', 'batch_310524124050', 'batch_310524162346', 'batch_310524162631'] + +if 'sorted_list' not in st.session_state: + st.session_state.sorted_list = batch_list + +def sort_list(ex_list, sort_by, order): + if sort_by == 'Alphabetical': + ex_list = sorted(ex_list, reverse=(order == 'Descending')) + elif sort_by == 'Create Date': + ex_list = ex_list if order == 'Ascending' else list(reversed(ex_list)) + + return ex_list + +col1, col2, col3, col4 = st.columns([0.5, 0.5, 0.5, 0.5]) + + +with col1: + sort_by = st.radio("**Sort Batch_IDs**", ('Alphabetical', 'Create Date')) + +with col2: + order = st.radio('', ('Ascending','Descending')) + +with col3: + add_vertical_space(2) + if st.button('Apply'): + st.session_state.sorted_list = sort_list(batch_list, sort_by, order) + +path_row = st.columns([0.1, 0.8]) +with path_row[0]: + st.write("**Batch ID**") +with path_row[1]: + batch_id = st.selectbox('**Batch ID**', st.session_state.sorted_list, label_visibility = "collapsed", index = None) # MODIFIED for Ticket DOC-344 + +if not batch_id: + batch_id = "None" + +checks = st.columns([0.1, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12, 0.12]) +with checks[0]: + st.write("**Group No.**") + +with checks[1]: + a = st.checkbox('Unique Key', key = str(1), args="Unique") +with checks[2]: + b = st.checkbox('Pricing Before Carveouts', key = str(2)) +with checks[3]: + c = st.checkbox('Contract Related', key = str(3)) +with checks[4]: + d = st.checkbox('Provider', key = str(4)) +with checks[5]: + e = st.checkbox('Timeline', key = str(5)) +with checks[6]: + f = st.checkbox('Carveout Indicator', key = str(6)) +with checks[7]: + g = st.checkbox('Carveout Methodology', key = str(7)) + +add_vertical_space(1) + +df = pd.DataFrame(columns=['Contract Name', 'Unique Key','Pricing Before Carveouts' + , 'Contract Related', 'Provider', 'Timeline', 'Carveout Indicator', 'Carveout Methodology']) +# file_list = [] +# file_objects = s3_client.list_objects_v2(Bucket=client_bucket +# , Prefix="contracts_landing_zone/"+batch_id+"/", Delimiter='/') +file_list = ['Boilerplate_TX Amendment Mission Health Network effective_040114 MU.pdf', 'Custom_TX - MP AMENDMENT - MISSION HEALTH NETWORK - MU.pdf', + 'Delaware First Health_First State Homecare Agency_212260_7 MU.pdf', 'Molina Healthcare of Texas, Inc. Amendment 4 - HIX ACA__EFF 01012016_MU.pdf'] + +if st.button("Read the contracts from Path"): + # for obj in file_objects.get('Contents',[]): + # if not obj['Key'].endswith('/'): + # file_list.append(obj['Key'].split('/')[-1]) + df['Contract Name'] = file_list + # df['Request ID'] = range(len(file_list)) + # df['Contract ID'] = file_list + df['Unique Key'] = a + df['Pricing Before Carveouts'] = b + df['Contract Related'] = c + df['Provider'] = d + df['Timeline'] = e + df['Carveout Indicator'] = f + df['Carveout Methodology'] = g + dir_path = os.path.dirname(os.path.realpath(__file__)) + print(f'DEBUGGING: PWD= {dir_path}') + df.to_csv('temp1.csv', index=False) + +add_vertical_space(1) +df2 = pd.read_csv('temp1.csv') +edited_df = st.data_editor(df2) + +edited_df['REQUEST_USER'] = user_mail +edited_df['LATEST_FLAG BOOLEAN'] = True +edited_df['PIPELINE_KICKOFF_DATETIME'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") +edited_df['REQUEST_DATETIME'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + +@st.cache_data +def convert_df(df): + return df.to_csv(index=False).encode('utf-8') + +csv = convert_df(edited_df) +# edited_df = edited_df.reset_index() # make sure indexes pair with number of rows + +# additional_info = pd.DataFrame(columns=['REQUEST_ID','T_DRIVE_PATH','CLIENT_NAME' +# , 'GROUP_NAME', 'REQUEST_USERNAME', 'REQUEST_DATETIME']) +additional_info = pd.DataFrame(columns=['CLIENT_NAME', 'BATCH_ID', 'REQUEST_USERNAME', 'REQUEST_DATETIME']) +additional_info.loc[0] = [client, batch_id, st.session_state.user_info['mail'], datetime.now().strftime("%Y-%m-%d %H:%M:%S")] +st.write(additional_info) + +st.session_state.contract_count = 0 +contract_list = [] +for index, row in edited_df.iterrows(): + allow_run_for_contract = False + group_list = [] + if row['Unique Key']: + group_list.append('Unique Key') + allow_run_for_contract = True + if row['Pricing Before Carveouts']: + group_list.append('Pricing Before Carveouts') + allow_run_for_contract = True + if row['Contract Related']: + group_list.append('Contract Related') + allow_run_for_contract = True + if row['Provider']: + group_list.append('Provider') + allow_run_for_contract = True + if row['Timeline']: + group_list.append('Timeline') + allow_run_for_contract = True + if row['Carveout Indicator']: + group_list.append('Carveout Indicator') + allow_run_for_contract = True + if row['Carveout Methodology']: + group_list.append('Carveout Methodology') + allow_run_for_contract = True + if allow_run_for_contract: st.session_state.contract_count += 1 + entry_dict = { + "contract_name": row['Contract Name'], + "groups": group_list, + "contract_source_path": "contracts_landing_zone/"+batch_id+"/"+row['Contract Name'] + } + contract_list.append(entry_dict) + +myobj = { + "s3_bucket": client_bucket, + "batch_id": batch_id, + "client_name": client, + "username": user_mail, + "contract_list": contract_list +} + +buttons = st.columns([0.8, 0.2]) +with buttons[0]: + st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv') +with buttons[1]: + if st.button("Run Doczy.AI Pipeline"): + if not st.session_state.contract_count == len(edited_df): + st.error("Select at least one Group No. for every Contract") + else: + with st.spinner('Running...'): # Feedback to User while API endpoint sends response for DOC-342 + # csv_buf = StringIO() + # additional_info.to_csv(csv_buf, header=True, index=False) + # csv_buf.seek(0) + # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/request_submission.csv') + # csv_buf = StringIO() + # edited_df.to_csv(csv_buf, header=True, index=False) + # csv_buf.seek(0) + # s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/contract_config.csv') + # try: + # save_to_sf('load_request_and_contract_submissions', request_submission_file_name = "request_submission.csv", contract_config_file_name = "contract_config.csv") + # except Exception as e: + # st.write(e) + # response = requests.post(doczy_pipeline, json = myobj) + # if response.status_code >= 200 and response.status_code < 300: + # st.write("Success") + # # st.write(myobj) + # else: + # st.write("Failed") + # st.write(response.text) + time.sleep(5) + st.write("Success!") + diff --git a/streamlit/local/local_interface_2.py b/streamlit/local/local_interface_2.py new file mode 100644 index 0000000..18702f3 --- /dev/null +++ b/streamlit/local/local_interface_2.py @@ -0,0 +1,228 @@ +import json + +import boto3 +from langchain.prompts import PromptTemplate +from langchain.embeddings.bedrock import BedrockEmbeddings +from langchain.llms.bedrock import Bedrock +from langchain_community.vectorstores import Chroma +# from constants import CHROMA_SETTINGS, EMBEDDING_MODEL_NAME, PERSIST_DIRECTORY, MODEL_ID, MODEL_BASENAME, SOURCE_DIRECTORY, USER_LIST +from langchain.chains import RetrievalQA + +import streamlit as st +from streamlit_extras.add_vertical_space import add_vertical_space +from streamlit_pdf_viewer import pdf_viewer # needs to be installed on the server +import os +import pandas as pd +import numpy as np +# import util +import anthropic +from pydantic import BaseModel +from typing import List +import re +import base64 +# from sf_conn import get_snowflake_conn + +REDIRECT_URI = 'https://doczydev.aarete.com:8502' +# user_list = USER_LIST + +st.set_page_config(layout = "wide") +# Sidebar contents +# with st.sidebar: +# st.title("Doczy.AI ™") +# st.markdown( +# """ +# ## About +# This app extracts data from contracts + +# """ +# ) +# add_vertical_space(15) +# # st.write("Doczy") + +_,c1= st.columns([5,1]) +# try: +# util.setup_page(REDIRECT_URI) +# except: +# st.write("SSO Failed") +# st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} +st.session_state['user_info'] = {'mail': 'maamseek@aarete.com', 'displayName': 'Mayank Aamseek'} +try: + c1.write(f"User: **{st.session_state.user_info['displayName']}**") + user_mail = st.session_state.user_info['mail'] +except KeyError as e: + st.write("Session Expired.") + st.stop() + +# remove below try except statement if comparison with actual vales is not required +# try: +# conn = get_snowflake_conn('STG') +# cur = conn.cursor() +# query = 'select * from "TRAINING_DATA_RAW"' +# cur.execute(query) +# field_values = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description]) +# field_values['Document_Name'] = field_values['DOCUMENT_NAME'] +# except: +# # field_values = pd.read_csv('contract_field_values.csv', encoding='unicode_escape', skipinitialspace=True) +# # field_values = field_values.loc[:, ~field_values.columns.str.contains('Unnamed:')] +# st.write("Conn failed, unable to fetch data from training data table in Snowflake") + +field_values = pd.read_csv('contract_field_values.csv', encoding='unicode_escape', skipinitialspace=True) +field_values = field_values.loc[:, ~field_values.columns.str.contains('Unnamed:')] + +# try: +# query = 'select * from "PROMPT_CONFIG"' +# cur.execute(query) +# fields = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description]) + +# fields.rename(columns={'FIELD_DESC': 'Field Name'}, inplace = True) +# fields.rename(columns={'PROMPT': 'Interrogation Question?'}, inplace = True) +# fields.rename(columns={'GROUP_ID': 'PRIORITY'}, inplace = True) +# fields.rename(columns={'FIELD_NAME': 'SF_DB_COL_NAME'}, inplace = True) +# fields.rename(columns={'FM_MODEL_ID': 'llm_selected'}, inplace = True) +# except Exception as e: +# st.write("Unable to fetch data from Snowflake: ",e) +# # fields = pd.read_csv('contract_fields.csv', encoding='unicode_escape', skipinitialspace=True) +# # fields = fields[~fields['SF_COL_NAME'].str.endswith('_PG', na=None)] + +fields = pd.read_csv('contract_fields.csv', encoding='unicode_escape', skipinitialspace=True) +# fields = fields[fields['SF_DB_COL_NAME'].str.endswith('_PG', na=None)] + +# change the code below if contract list is fetched from snowflake +s3_client = boto3.client('s3', + region_name="us-east-2" +) +bucket = 'doczy-dev-infra-textract' +# objects = s3_client.list_objects_v2(Bucket=bucket, Prefix="training-data/") +# file_list = [] +# for obj in objects['Contents']: +# if not obj['Key'].endswith('/'): +# file_list.append(obj['Key']) + +file_list = ['Contract_Training_Exercise_Pricing.pdf', 'Contract_Training_Exercise_SLA.pdf'] +contract_list = sorted(file_list) + +file_row = st.columns([0.2, 0.7, 0.1]) +with file_row[0]: + st.write("**Contract Name**") +with file_row[1]: + file_name = st.selectbox('Select a file', ['All'] + contract_list, label_visibility = "collapsed", index= None) # MODIFIED - Append 'All' in the front instead of at the end + +field_row = st.columns([0.2, 0.7, 0.1]) +with field_row[0]: + st.write("**Field Group**") +with field_row[1]: + field_group = st.selectbox('Field Group',('Unique Key', 'Contract Related', 'Pricing Before Carveouts - I' + , 'Pricing Before Carveouts - II', 'Carveout Indicator, Code Type and Code #s - I' + , 'Carveout Indicator, Code Type and Code #s - II', 'Carveout Indicator, Code Type and Code #s - III' + , 'Optimize Carving Indic.', 'Carveout Method - I', 'Carveout Method - II', 'Provider' + , 'Timeline'), label_visibility = "collapsed", index = None) + +if field_group == 'Unique Key': + fields = fields[fields['PRIORITY'] == 'A'] +elif field_group == 'Contract Related': + fields = fields[fields['PRIORITY'] == 'C'] +elif field_group == 'Pricing Before Carveouts - I': + fields = fields[fields['PRIORITY'] == 'B'] + fields = np.array_split(fields, 2)[0] +elif field_group == 'Pricing Before Carveouts - II': + fields = fields[fields['PRIORITY'] == 'B'] + fields = np.array_split(fields, 2)[1] +elif field_group == 'Carveout Indicator, Code Type and Code #s - I': + fields = fields[fields['PRIORITY'] == 'F'] + fields = np.array_split(fields, 3)[0] +elif field_group == 'Carveout Indicator, Code Type and Code #s - II': + fields = fields[fields['PRIORITY'] == 'F'] + fields = np.array_split(fields, 3)[1] +elif field_group == 'Carveout Indicator, Code Type and Code #s - III': + fields = fields[fields['PRIORITY'] == 'F'] + fields = np.array_split(fields, 3)[2] +elif field_group == 'Carveout Methodology - I': + fields = fields[fields['PRIORITY'] == 'G'] + fields = np.array_split(fields, 4)[0] +elif field_group == 'Carveout Methodology - II': + fields = fields[fields['PRIORITY'] == 'G'] + fields = np.array_split(fields, 4)[1] +elif field_group == 'Carveout Method - III': + fields = fields[fields['PRIORITY'] == 'G'] + fields = np.array_split(fields, 4)[2] +elif field_group == 'Carveout Method - IV': + fields = fields[fields['PRIORITY'] == 'G'] + fields = np.array_split(fields, 4)[3] +elif field_group == 'Provider': + fields = fields[fields['PRIORITY'] == 'D'] +elif field_group == 'Timeline': + fields = fields[fields['PRIORITY'] == 'E'] + +if st.button("Show Results"): + # query = 'select * from "DOCZY_PIPELINE_RAW_OUTPUT"' + # cur.execute(query) + # df2 = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description]) + + # get this dataframe from snowflake table + df2 = pd.DataFrame(columns=['Contract Name','Field Name', 'SF_DB_COL_NAME', 'Snippet','Page Number' + , 'Field Extracted Value', 'Actual Value','Imputed Value']) + df2.to_csv('temp2.csv', index=False) + +if st.button("Show PDF"): + if file_name == None or file_name == "All": + st.error("Choose one specific file.") + else: + with st.sidebar: + st.markdown( + """ + + """, + unsafe_allow_html=True, + ) + pdf_viewer(file_name, width=1500) + +# if st.button("Show PDF"): +# if file_name == None or file_name == "All": +# st.error("Choose one specific file.") +# else: +# with st.sidebar: +# with open(file_name, "rb") as f: +# base64_pdf = base64.b64encode(f.read()).decode('utf-8') +# # Embedding PDF in HTML +# pdf_display = F'' +# # Displaying File +# st.markdown( +# """ +# +# """, +# unsafe_allow_html=True, +# ) +# st.markdown(pdf_display, unsafe_allow_html=True) + +df2 = pd.read_csv('temp2.csv') +df2['Imputed Value'] = '' +edited_df = st.data_editor(df2) + +@st.cache_data +def convert_df(df): + return df.to_csv(index=False).encode('utf-8') + +csv = convert_df(edited_df) + +buttons = st.columns(3) +with buttons[0]: + # st.button("Save All Imputations") + st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv') +with buttons[1]: + # st.download_button("Download Table", csv, "file.csv", "text/csv", key='download-csv') + st.write("") +with buttons[2]: + if st.button("Kickoff Database Integration"): + st.write("Stored in DB") + + + + diff --git a/textract-pipeline/terraform/main.tf b/textract-pipeline/terraform/main.tf index 9c7f38b..0284a95 100644 --- a/textract-pipeline/terraform/main.tf +++ b/textract-pipeline/terraform/main.tf @@ -70,9 +70,6 @@ locals { provider "aws" { # profile = var.aws_profile - # access_key = var.access_key - # secret_key = var.secret_key - default_tags { tags = local.common_tags }