diff --git a/devops-pipeline/terraform-backend-resources/main.tf b/devops-pipeline/terraform-backend-resources/main.tf index 2b4a9ff..885ad4e 100644 --- a/devops-pipeline/terraform-backend-resources/main.tf +++ b/devops-pipeline/terraform-backend-resources/main.tf @@ -9,7 +9,7 @@ terraform { provider "aws" { profile = var.aws_profile - + region = var.aws_region # default_tags { # tags = { # Terraform = "true" diff --git a/devops-pipeline/terraform-backend-resources/variables.tf b/devops-pipeline/terraform-backend-resources/variables.tf index cea794b..25c303f 100644 --- a/devops-pipeline/terraform-backend-resources/variables.tf +++ b/devops-pipeline/terraform-backend-resources/variables.tf @@ -1,7 +1,7 @@ # required variable "aws_profile" { type = string - default = "doczyai" + default = "doczy_uat" } # required variable "aws_region" { @@ -16,7 +16,7 @@ variable "project_name" { # required variable "environment" { type = string - default = "dev" + default = "uat" } # required variable "client_name" { diff --git a/streamlit-server/main.tf b/streamlit-server/main.tf index e9e4dcf..9eac30b 100644 --- a/streamlit-server/main.tf +++ b/streamlit-server/main.tf @@ -3,10 +3,24 @@ provider "aws" { profile = "temp_cred" } -# data "aws_acm_certificate" "cert_global" { -# domain = "doczy.aarete.com" -# statuses = ["ISSUED"] -# } +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "5.39.0" + } + } + + backend "s3" { + bucket = "doczyai-use2-d-infra-s3-terraform-state" # Parameterize using -backend-config flag with "terraform init" + key = "terraform/streamlit-server/terraform.tfstate" # Parameterize + region = "us-east-2" # Parameterize + profile = "temp_cred" # Parameterize + dynamodb_table = "doczyai-use2-d-infra-dyd-terraform-lock" # Parameterize + encrypt = true + } +} + locals { region_map = { @@ -92,8 +106,10 @@ resource "aws_instance" "streamlit_server" { echo '${file("${path.module}/requirements.txt")}' > /tmp/requirements.txt apt-get update apt-get install -y python3 python3-pip - pip3 install $(cat /tmp/requirements.txt) + sudo apt-get install libffi-dev + python3 -m pip3 install -r /tmp/requirements.txt sudo apt install openssh-client + sudo apt install net-tools # Create UI0 systemd service cat < /etc/systemd/system/streamlit-ui0.service diff --git a/streamlit-server/requirements.txt b/streamlit-server/requirements.txt index c8ec80f..e717334 100644 --- a/streamlit-server/requirements.txt +++ b/streamlit-server/requirements.txt @@ -1,38 +1,212 @@ -# Natural Language Processing -langchain==0.0.267 +aiohttp==3.9.5 +aiosignal==1.3.1 +altair==5.3.0 +anthropic==0.25.7 +anyio==4.3.0 +asn1crypto==1.5.1 +async-timeout==4.0.3 +atpublic==4.1.0 +attrs==21.2.0 +Automat==20.2.0 +awscli==1.32.99 +Babel==2.8.0 +backoff==2.2.1 +bcrypt==3.2.0 +beautifulsoup4==4.12.3 +blinker==1.4 +boto3==1.34.99 +botocore==1.34.99 +cachetools==5.3.3 +certifi==2020.6.20 +cffi==1.16.0 +chardet==4.0.0 +charset-normalizer==3.3.2 +chroma-hnswlib==0.7.2 chromadb==0.4.6 +click==8.0.3 +cloud-init==23.4.4 +cloudpickle==2.2.1 +colorama==0.4.4 +coloredlogs==15.0.1 +command-not-found==0.3 +configobj==5.0.6 +constantly==15.1.0 +contourpy==1.2.1 +cryptography==42.0.7 +cycler==0.12.1 +dataclasses-json==0.5.14 +dbus-python==1.2.18 +distro==1.7.0 +distro-info==1.1+ubuntu0.2 +docutils==0.16 +docx2txt==0.8 +ec2-hibinit-agent==1.0.0 +emoji==2.11.1 +entrypoints==0.4 +et-xmlfile==1.1.0 +exceptiongroup==1.2.1 +Faker==25.0.1 +fastapi==0.99.1 +favicon==0.7.0 +filelock==3.14.0 +filetype==1.2.0 +flatbuffers==24.3.25 +fonttools==4.51.0 +frozenlist==1.4.1 +fsspec==2024.3.1 +gitdb==4.0.11 +GitPython==3.1.43 +greenlet==3.0.3 +h11==0.14.0 +hibagent==1.0.1 +htbuilder==0.6.2 +httpcore==1.0.5 +httplib2==0.20.2 +httptools==0.6.1 +httpx==0.27.0 +huggingface-hub==0.23.0 +humanfriendly==10.0 +hyperlink==21.0.0 +idna==3.3 +importlib-metadata==4.6.4 +importlib_resources==6.4.0 +incremental==21.3.0 +jeepney==0.7.1 +Jinja2==3.0.3 +jmespath==1.0.1 +joblib==1.4.2 +jsonpatch==1.33 +jsonpointer==2.0 +jsonschema==3.2.0 +keyring==23.5.0 +kiwisolver==1.4.5 +langchain==0.1.17 +langchain-community==0.0.37 +langchain-core==0.1.52 +langchain-text-splitters==0.0.1 +langdetect==1.0.9 +langsmith==0.1.54 +launchpadlib==1.10.16 +lazr.restfulclient==0.14.4 +lazr.uri==1.0.6 +lxml==5.2.1 +Markdown==3.6 +markdown-it-py==3.0.0 +markdownlit==0.0.7 +MarkupSafe==2.0.1 +marshmallow==3.21.2 +matplotlib==3.8.4 +mdurl==0.1.2 +monotonic==1.6 +more-itertools==8.10.0 +mpmath==1.3.0 +msal==1.27.0 +multidict==6.0.5 +mypy-extensions==1.0.0 +netifaces==0.11.0 +nltk==3.8.1 +numexpr==2.10.0 +numpy==1.26.4 +oauthlib==3.2.0 +onnxruntime==1.17.3 +openapi-schema-pydantic==1.2.4 +openpyxl==3.1.2 +orjson==3.10.3 +overrides==7.7.0 +packaging==23.2 +pandas==2.2.2 pdfminer.six==20221105 -InstructorEmbedding -sentence-transformers==2.2.2 -faiss-cpu -huggingface_hub -transformers -autoawq -protobuf==3.20.2; sys_platform != 'darwin' -protobuf==3.20.2; sys_platform == 'darwin' and platform_machine != 'arm64' -protobuf==3.20.3; sys_platform == 'darwin' and platform_machine == 'arm64' -auto-gptq==0.2.2 -docx2txt -unstructured -unstructured[pdf] - -# Utilities +pexpect==4.8.0 +pillow==10.3.0 +platformdirs==4.2.1 +posthog==3.5.0 +prometheus_client==0.20.0 +protobuf==4.25.3 +ptyprocess==0.7.0 +pulsar-client==3.5.0 +pyarrow==16.0.0 +pyasn1==0.4.8 +pyasn1-modules==0.2.1 +pycparser==2.22 +pydantic==1.10.15 +pydeck==0.9.0 +Pygments==2.18.0 +PyGObject==3.42.1 +PyHamcrest==2.0.2 +PyJWT==2.3.0 +pymdown-extensions==10.8.1 +pyOpenSSL==24.1.0 +pyparsing==2.4.7 +PyPika==0.48.9 +pyrsistent==0.18.1 +pyserial==3.5 +python-apt==2.4.0+ubuntu3 +python-dateutil==2.9.0.post0 +python-debian==0.1.43+ubuntu1.1 +python-dotenv==1.0.1 +python-iso639==2024.4.27 +python-magic==0.4.24 +pytz==2022.1 +PyYAML==5.4.1 +rapidfuzz==3.9.0 +regex==2024.4.28 +requests==2.31.0 +rich==13.7.1 +rsa==4.7.2 +s3transfer==0.10.1 +SecretStorage==3.3.1 +service-identity==18.1.0 +six==1.16.0 +smmap==5.0.1 +sniffio==1.3.1 +snowflake-connector-python==3.10.0 +snowflake-snowpark-python==1.15.0 +snowflake._legacy==0.8.0 +snowflake.core==0.8.0 +sortedcontainers==2.4.0 +sos==4.5.6 +soupsieve==2.5 +SQLAlchemy==2.0.30 +ssh-import-id==5.11 +st-annotated-text==4.0.1 +starlette==0.27.0 +streamlit==1.34.0 +streamlit-camera-input-live==0.2.0 +streamlit-card==1.0.0 +streamlit-embedcode==0.1.2 +streamlit-extras==0.4.2 +streamlit-faker==0.0.3 +streamlit-image-coordinates==0.1.6 +streamlit-keyup==0.2.4 +streamlit-toggle-switch==1.0.2 +streamlit-vertical-slider==2.5.5 +sympy==1.12 +systemd-python==234 +tabulate==0.9.0 +tenacity==8.2.3 +tokenizers==0.19.1 +toml==0.10.2 +tomlkit==0.12.4 +toolz==0.12.1 +tornado==6.4 +tqdm==4.66.4 +Twisted==22.1.0 +typing-inspect==0.9.0 +typing_extensions==4.11.0 +tzdata==2024.1 +ubuntu-advantage-tools==8001 +ufw==0.36.1 +unattended-upgrades==0.1 +unstructured==0.11.2 urllib3==1.26.6 -accelerate -bitsandbytes ; sys_platform != 'win32' -bitsandbytes-windows ; sys_platform == 'win32' -click -flask -requests - -# Streamlit related -streamlit -Streamlit-extras - -# Excel File Manipulation -openpyxl -numpy>=1.22.2 - -# AWS related -boto3 -awscli \ No newline at end of file +uvicorn==0.29.0 +uvloop==0.19.0 +validators==0.28.1 +wadllib==1.3.6 +watchdog==4.0.0 +watchfiles==0.21.0 +websockets==12.0 +wrapt==1.16.0 +yarl==1.9.4 +zipp==1.0.0 +zope.interface==5.4.0 diff --git a/streamlit/ingest.py b/streamlit/ingest.py index c5e5833..6046c77 100644 --- a/streamlit/ingest.py +++ b/streamlit/ingest.py @@ -180,9 +180,6 @@ def main(device_type): bedrock_runtime = boto3.client( service_name="bedrock-runtime", region_name="us-east-1", - aws_access_key_id="ASIAZTMXAXNXD3TOUOAJ", - aws_secret_access_key="pk7k69CqXZPB/bf2hdFsW+47D5WYkoWXFdxQ633X", - aws_session_token="IQoJb3JpZ2luX2VjELj//////////wEaCXVzLWVhc3QtMiJIMEYCIQCqcyLSwLeN9RX6tz+TgB4VMabHYwPcA3bTzx7xZ5b75QIhAMiOKbDlmPZ+eHkeskzelK4M9gWtS7GqlzJq5qmWbbrOKo4DCKH//////////wEQABoMNjYwMTMxMDY4NzgyIgzPuv0vReC65Mo/bQcq4gLXtII+VTaxES+JrjHHWpWywdmpsJneN6bcB3U37z7J8BFA5aaUqkATKmwJI0brmk5ZJKL8SpgBEC2TNdA/V9nzbMlf2HPunhEv6OPLzSWp5iJqaNL945MP764CbkYvfN9QWd6durUv1WgGZRNcbMzXg2UFsxKcRql795vtOmL207+R7uIouWl73So7NaCkEgaj4FdEJ9lbnfvWFeNcBlHbjwUx8e9EJjwm8D60OkTdS4w7Q3EacoEKLO94/kp2RtsaggAUV33OcvO/32VwYJzhRJYuveQUZnIzfkmybGtrXkkWLqMO9pls1bkTmIjaeMwcL8Uo7oowR9sTFCT87rY711yIYBGVOjkN9mfavPH4FSCNeeI6ta5aXqa7iVDU7rPerpFtle2i1VGfTW5bKoaWxjO13IdqI4yI9Pgibl+FuVeRB7md2tDuS6SMAuX4qpWnMVDLKCItlAxXlZdmVhuDFf8wr+7rrgY6pQH02EIrvCkYYXbJd8u7t78hOam3lSNPz+nghQHA5ppl7TnyRBL5/9Rfqp5Ib8y54HdC4hbzz/7w6lvfS0QbHE3Q6r2GC6XWk9L4FAxD8pyZkIYbtwf/WueX1g0r+0x7uLCxKxbWsYum/bigyvNxsDyFIcQcUOm+OJsXGPH3Z7z+qWKni5N9RRA4SEEpdSTRkAWipDtgVNII6kd/eWvyE74d+LYUrmA=" ) embeddings = BedrockEmbeddings( client=bedrock_runtime,