From 98cb644c2d802ba157fb61d6d7315c8dfe14df9a Mon Sep 17 00:00:00 2001 From: Umang Mistry Date: Fri, 17 May 2024 13:32:34 -0500 Subject: [PATCH] Creating new server, resuing same AMI from previous TF streamlit instance --- bitbucket-pipelines.yml | 2 +- on_demand_scripts/setup_ssh_auth.sh | 93 ++++++++++++++++ streamlit-server/alb.tf | 159 ++++++++++++++++++++++++++++ streamlit-server/alb_sg.tf | 55 ++++++++++ streamlit-server/main.tf | 17 +-- streamlit-server/user_data.sh | 25 +++++ streamlit-server/variables.tf | 9 +- 7 files changed, 350 insertions(+), 10 deletions(-) create mode 100644 on_demand_scripts/setup_ssh_auth.sh create mode 100644 streamlit-server/alb.tf create mode 100644 streamlit-server/alb_sg.tf create mode 100644 streamlit-server/user_data.sh diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 2b2d76b..d11b5d9 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -159,7 +159,7 @@ pipelines: - cd streamlit-server/ - terraform validate # Ensure the backend configuration for s3 obj key is correct - - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=streamlit-pipeline/terraform.tfstate" + - terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=streamlit-pipeline/terraform.tfstate" -backend-config="region=us-east-2" - terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=dev" condition: changesets: diff --git a/on_demand_scripts/setup_ssh_auth.sh b/on_demand_scripts/setup_ssh_auth.sh new file mode 100644 index 0000000..be7f11f --- /dev/null +++ b/on_demand_scripts/setup_ssh_auth.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +# Define variables +EMAIL="your_email@example.com" # Replace with your email +SSH_KEY_NAME="id_ed25519_bitbucket" # Replace with your desired key name + +# Function to display messages +function info { + echo -e "\033[1;34m[INFO] $1\033[0m" +} + +function success { + echo -e "\033[1;32m[SUCCESS] $1\033[0m" +} + +function error { + echo -e "\033[1;31m[ERROR] $1\033[0m" >&2 +} + +# Update package list and install OpenSSH client +info "Updating package list and installing OpenSSH client..." +if sudo apt update && sudo apt install -y openssh-client; then + success "OpenSSH client installed successfully." +else + error "Failed to install OpenSSH client." + exit 1 +fi + +# Start the SSH agent +info "Starting the SSH agent..." +if eval "$(ssh-agent)"; then + success "SSH agent started." +else + error "Failed to start SSH agent." + exit 1 +fi + +# Navigate to the home directory +cd ~ + +# Generate the SSH key +info "Generating SSH key..." +if ssh-keygen -t ed25519 -b 4096 -C "$EMAIL" -f ~/.ssh/"$SSH_KEY_NAME"; then + success "SSH key generated at ~/.ssh/$SSH_KEY_NAME." +else + error "Failed to generate SSH key." + exit 1 +fi + +# Add the SSH key to the agent +info "Adding SSH key to the agent..." +if ssh-add ~/.ssh/"$SSH_KEY_NAME"; then + success "SSH key added to the agent." +else + error "Failed to add SSH key to the agent." + exit 1 +fi + +# Create .ssh directory if it doesn't exist +if [ ! -d ~/.ssh ]; then + mkdir -p ~/.ssh + chmod 700 ~/.ssh +fi + +# Create or update the SSH config file +info "Configuring SSH for Bitbucket..." +SSH_CONFIG_PATH=~/.ssh/config +{ + echo "Host bitbucket.org" + echo " AddKeysToAgent yes" + echo " IdentityFile ~/.ssh/$SSH_KEY_NAME" +} | sudo tee -a $SSH_CONFIG_PATH > /dev/null + +if [ $? -eq 0 ]; then + success "SSH configuration updated." +else + error "Failed to update SSH configuration." + exit 1 +fi + +# Set correct permissions for the config file +sudo chmod 600 $SSH_CONFIG_PATH + +# Test the SSH connection to Bitbucket +info "Testing SSH connection to Bitbucket..." +if ssh -T git@bitbucket.org; then + success "SSH authentication with Bitbucket succeeded." +else + error "SSH authentication with Bitbucket failed." + exit 1 +fi + +success "SSH setup for Bitbucket completed successfully." diff --git a/streamlit-server/alb.tf b/streamlit-server/alb.tf new file mode 100644 index 0000000..579a5fe --- /dev/null +++ b/streamlit-server/alb.tf @@ -0,0 +1,159 @@ +resource "aws_lb" "app_lb" { +# name = "" + # Using global prefix from main.tf for name of the load balancer + name = "${local.global_prefix}-load-balancer" + internal = false + load_balancer_type = "application" + security_groups = [aws_security_group.lb_sg.id] + subnets = data.aws_subnets.subnets.ids # Referencing the subnet ID from the data source + + enable_deletion_protection = false +} + + +# TARGET GROUP FOR ALL PORTS +resource "aws_lb_target_group" "app_tg_8500" { + name = "app-target-group-8500" + port = 8500 + protocol = "HTTP" + vpc_id = var.vpc_id + + health_check { + path = "/" + interval = 30 + timeout = 5 + healthy_threshold = 5 + unhealthy_threshold = 2 + matcher = "200" + } +} + +resource "aws_lb_target_group" "app_tg_8501" { + name = "app-target-group-8501" + port = 8501 + protocol = "HTTP" + vpc_id = var.vpc_id + + health_check { + path = "/" + interval = 30 + timeout = 5 + healthy_threshold = 5 + unhealthy_threshold = 2 + matcher = "200" + } +} + +resource "aws_lb_target_group" "app_tg_8502" { + name = "app-target-group-8502" + port = 8502 + protocol = "HTTP" + vpc_id = var.vpc_id + + health_check { + path = "/" + interval = 30 + timeout = 5 + healthy_threshold = 5 + unhealthy_threshold = 2 + matcher = "200" + } +} + +resource "aws_lb_target_group" "app_tg_8503" { + name = "app-target-group-8503" + port = 8503 + protocol = "HTTP" + vpc_id = var.vpc_id + + health_check { + path = "/" + interval = 30 + timeout = 5 + healthy_threshold = 5 + unhealthy_threshold = 2 + matcher = "200" + } +} + + + +# Attach the target group to the listener +resource "aws_lb_target_group_attachment" "app_instance_8500" { + target_group_arn = aws_lb_target_group.app_tg_8500.arn + target_id = aws_instance.streamlit_server.id + port = 8500 +} + +resource "aws_lb_target_group_attachment" "app_instance_8501" { + target_group_arn = aws_lb_target_group.app_tg_8501.arn + target_id = aws_instance.streamlit_server.id + port = 8501 +} + +resource "aws_lb_target_group_attachment" "app_instance_8502" { + target_group_arn = aws_lb_target_group.app_tg_8502.arn + target_id = aws_instance.streamlit_server.id + port = 8502 +} + +resource "aws_lb_target_group_attachment" "app_instance_8503" { + target_group_arn = aws_lb_target_group.app_tg_8503.arn + target_id = aws_instance.streamlit_server.id + port = 8503 +} + + +# Update listener rules +resource "aws_lb_listener" "https_8500" { + load_balancer_arn = aws_lb.app_lb.arn + port = "443" + protocol = "HTTPS" + ssl_policy = "ELBSecurityPolicy-2016-08" + # Using variable for ARN + certificate_arn = var.acm_arn_dev + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.app_tg_8500.arn + } +} + +resource "aws_lb_listener" "https_8501" { + load_balancer_arn = aws_lb.app_lb.arn + port = "443" + protocol = "HTTPS" + ssl_policy = "ELBSecurityPolicy-2016-08" + certificate_arn = var.acm_arn_dev + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.app_tg_8501.arn + } +} + +resource "aws_lb_listener" "https_8502" { + load_balancer_arn = aws_lb.app_lb.arn + port = "443" + protocol = "HTTPS" + ssl_policy = "ELBSecurityPolicy-2016-08" + certificate_arn = var.acm_arn_dev + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.app_tg_8502.arn + } +} + +resource "aws_lb_listener" "https_8503" { + load_balancer_arn = aws_lb.app_lb.arn + port = "443" + protocol = "HTTPS" + ssl_policy = "ELBSecurityPolicy-2016-08" + certificate_arn = var.acm_arn_dev + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.app_tg_8503.arn + } +} diff --git a/streamlit-server/alb_sg.tf b/streamlit-server/alb_sg.tf new file mode 100644 index 0000000..6087d54 --- /dev/null +++ b/streamlit-server/alb_sg.tf @@ -0,0 +1,55 @@ +resource "aws_security_group" "lb_sg" { + name = "${local.global_prefix}-load_balancer_sg" + description = "Security group for the load balancer" + vpc_id = var.vpc_id + + ingress { + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = ["172.19.0.0/16"] + } + + ingress { + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = ["172.19.0.0/16"] + } + + ingress { + from_port = 8500 + to_port = 8500 + protocol = "tcp" + cidr_blocks = ["172.19.0.0/16"] + } + + ingress { + from_port = 8501 + to_port = 8501 + protocol = "tcp" + cidr_blocks = ["172.19.0.0/16"] + } + + ingress { + from_port = 8502 + to_port = 8502 + protocol = "tcp" + cidr_blocks = ["172.19.0.0/16"] + } + + ingress { + from_port = 8503 + to_port = 8503 + protocol = "tcp" + cidr_blocks = ["172.19.0.0/16"] + } + + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } +} diff --git a/streamlit-server/main.tf b/streamlit-server/main.tf index 9eac30b..d305ac8 100644 --- a/streamlit-server/main.tf +++ b/streamlit-server/main.tf @@ -1,6 +1,5 @@ provider "aws" { region = "us-east-2" - profile = "temp_cred" } terraform { @@ -12,15 +11,19 @@ terraform { } backend "s3" { - bucket = "doczyai-use2-d-infra-s3-terraform-state" # Parameterize using -backend-config flag with "terraform init" - key = "terraform/streamlit-server/terraform.tfstate" # Parameterize - region = "us-east-2" # Parameterize - profile = "temp_cred" # Parameterize - dynamodb_table = "doczyai-use2-d-infra-dyd-terraform-lock" # Parameterize + # bucket = "doczyai-use2-d-infra-s3-terraform-state" # Parameterize using -backend-config flag with "terraform init" + # key = "terraform/streamlit-server/terraform.tfstate" # Parameterize + # region = "us-east-2" # Parameterize + # profile = "temp_cred" # Parameterize + # dynamodb_table = "doczyai-use2-d-infra-dyd-terraform-lock" # Parameterize encrypt = true } } +# data "aws_acm_certificate" "cert_global" { +# domain = "doczy.aarete.com" +# statuses = ["ISSUED"] +# } locals { region_map = { @@ -198,7 +201,7 @@ resource "aws_instance" "streamlit_server" { tags = { Terraform = "true" - Environment = "dev" + Environment = var.environment } } diff --git a/streamlit-server/user_data.sh b/streamlit-server/user_data.sh new file mode 100644 index 0000000..782aa40 --- /dev/null +++ b/streamlit-server/user_data.sh @@ -0,0 +1,25 @@ +#!/bin/bash +echo '${file("${path.module}/requirements.txt")}' > /tmp/requirements.txt +apt-get update +apt-get install -y python3 python3-pip +pip3 install $(cat /tmp/requirements.txt) + +# Create a basic systemd service +cat < /etc/systemd/system/streamlit-ui0.service +[Unit] +Description=Streamlit Server Service + +[Service] +User=ubuntu +Type=simple +Restart=always +WorkingDirectory=/home/ubuntu/streamlit/doczy.ai/streamlit +ExecStart=/home/ubuntu/.local/bin/streamlit run /home/ubuntu/streamlit/doczy.ai/streamlit/interface_0.py --server.port 8500 + +[Install] +WantedBy=multi-user.target +EOT + +# Enable and start the service +systemctl enable streamlit-ui0.service +systemctl start streamlit-ui0.service diff --git a/streamlit-server/variables.tf b/streamlit-server/variables.tf index 08071d0..59c7f11 100644 --- a/streamlit-server/variables.tf +++ b/streamlit-server/variables.tf @@ -33,13 +33,13 @@ variable "vpc_id" { variable "ubuntu_ami" { type = string - default = "ami-0b8b44ec9a8f90422" + default = "ami-0de7e97fedfbc6ef6" # Switch to this ami-0b8b44ec9a8f90422 } variable "ec2_instance_type" { type = string - default = "t2.small" + default = "t2.large" } # EC2 iam role name @@ -52,4 +52,9 @@ variable "ec2_iam_role_name" { variable "SecretsNames"{ type = list(string) default = [] +} + +variable "acm_arn_dev" { + type = string + default = "arn:aws:acm:us-east-2:660131068782:certificate/3a146592-2b00-4b33-90a6-0d9ab0040d8b" } \ No newline at end of file