jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Creating new server, resuing same AMI from previous TF streamlit instance

Browse Source
This commit is contained in:
Umang Mistry
2024-05-17 13:32:34 -05:00
parent 851b6f6a47
commit 98cb644c2d
7 changed files with 350 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bitbucket-pipelines.yml
+1 -1
View File
@@ -159,7 +159,7 @@ pipelines:
- cd streamlit-server/
- terraform validate
# Ensure the backend configuration for s3 obj key is correct
- terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=streamlit-pipeline/terraform.tfstate"
- terraform init -migrate-state -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY" -backend-config="token=$AWS_SESSION_TOKEN" -backend-config="dynamodb_table=doczyai-use2-d-infra-dyd-terraform-lock" -backend-config="bucket=doczyai-use2-d-infra-s3-terraform-state" -backend-config="key=streamlit-pipeline/terraform.tfstate" -backend-config="region=us-east-2"
- terraform apply --auto-approve -no-color -var "access_key=$AWS_ACCESS_KEY_ID" -var "secret_key=$AWS_SECRET_ACCESS_KEY" -var "token=$AWS_SESSION_TOKEN" -var "aws_region=us-east-2" -var "environment=dev"
condition:
changesets:
on_demand_scripts/setup_ssh_auth.sh
+93
View File
@@ -0,0 +1,93 @@
#!/bin/bash
# Define variables
EMAIL="your_email@example.com" # Replace with your email
SSH_KEY_NAME="id_ed25519_bitbucket" # Replace with your desired key name
# Function to display messages
function info {
echo -e "\033[1;34m[INFO] $1\033[0m"
}
function success {
echo -e "\033[1;32m[SUCCESS] $1\033[0m"
}
function error {
echo -e "\033[1;31m[ERROR] $1\033[0m" >&2
}
# Update package list and install OpenSSH client
info "Updating package list and installing OpenSSH client..."
if sudo apt update && sudo apt install -y openssh-client; then
success "OpenSSH client installed successfully."
else
error "Failed to install OpenSSH client."
exit 1
fi
# Start the SSH agent
info "Starting the SSH agent..."
if eval "$(ssh-agent)"; then
success "SSH agent started."
else
error "Failed to start SSH agent."
exit 1
fi
# Navigate to the home directory
cd ~
# Generate the SSH key
info "Generating SSH key..."
if ssh-keygen -t ed25519 -b 4096 -C "$EMAIL" -f ~/.ssh/"$SSH_KEY_NAME"; then
success "SSH key generated at ~/.ssh/$SSH_KEY_NAME."
else
error "Failed to generate SSH key."
exit 1
fi
# Add the SSH key to the agent
info "Adding SSH key to the agent..."
if ssh-add ~/.ssh/"$SSH_KEY_NAME"; then
success "SSH key added to the agent."
else
error "Failed to add SSH key to the agent."
exit 1
fi
# Create .ssh directory if it doesn't exist
if [ ! -d ~/.ssh ]; then
mkdir -p ~/.ssh
chmod 700 ~/.ssh
fi
# Create or update the SSH config file
info "Configuring SSH for Bitbucket..."
SSH_CONFIG_PATH=~/.ssh/config
{
echo "Host bitbucket.org"
echo " AddKeysToAgent yes"
echo " IdentityFile ~/.ssh/$SSH_KEY_NAME"
} | sudo tee -a $SSH_CONFIG_PATH > /dev/null
if [ $? -eq 0 ]; then
success "SSH configuration updated."
else
error "Failed to update SSH configuration."
exit 1
fi
# Set correct permissions for the config file
sudo chmod 600 $SSH_CONFIG_PATH
# Test the SSH connection to Bitbucket
info "Testing SSH connection to Bitbucket..."
if ssh -T git@bitbucket.org; then
success "SSH authentication with Bitbucket succeeded."
else
error "SSH authentication with Bitbucket failed."
exit 1
fi
success "SSH setup for Bitbucket completed successfully."
streamlit-server/alb.tf
+159
View File
@@ -0,0 +1,159 @@
resource "aws_lb" "app_lb" {
# name = ""
# Using global prefix from main.tf for name of the load balancer
name = "${local.global_prefix}-load-balancer"
internal = false
load_balancer_type = "application"
security_groups = [aws_security_group.lb_sg.id]
subnets = data.aws_subnets.subnets.ids # Referencing the subnet ID from the data source
enable_deletion_protection = false
}
# TARGET GROUP FOR ALL PORTS
resource "aws_lb_target_group" "app_tg_8500" {
name = "app-target-group-8500"
port = 8500
protocol = "HTTP"
vpc_id = var.vpc_id
health_check {
path = "/"
interval = 30
timeout = 5
healthy_threshold = 5
unhealthy_threshold = 2
matcher = "200"
}
}
resource "aws_lb_target_group" "app_tg_8501" {
name = "app-target-group-8501"
port = 8501
protocol = "HTTP"
vpc_id = var.vpc_id
health_check {
path = "/"
interval = 30
timeout = 5
healthy_threshold = 5
unhealthy_threshold = 2
matcher = "200"
}
}
resource "aws_lb_target_group" "app_tg_8502" {
name = "app-target-group-8502"
port = 8502
protocol = "HTTP"
vpc_id = var.vpc_id
health_check {
path = "/"
interval = 30
timeout = 5
healthy_threshold = 5
unhealthy_threshold = 2
matcher = "200"
}
}
resource "aws_lb_target_group" "app_tg_8503" {
name = "app-target-group-8503"
port = 8503
protocol = "HTTP"
vpc_id = var.vpc_id
health_check {
path = "/"
interval = 30
timeout = 5
healthy_threshold = 5
unhealthy_threshold = 2
matcher = "200"
}
}
# Attach the target group to the listener
resource "aws_lb_target_group_attachment" "app_instance_8500" {
target_group_arn = aws_lb_target_group.app_tg_8500.arn
target_id = aws_instance.streamlit_server.id
port = 8500
}
resource "aws_lb_target_group_attachment" "app_instance_8501" {
target_group_arn = aws_lb_target_group.app_tg_8501.arn
target_id = aws_instance.streamlit_server.id
port = 8501
}
resource "aws_lb_target_group_attachment" "app_instance_8502" {
target_group_arn = aws_lb_target_group.app_tg_8502.arn
target_id = aws_instance.streamlit_server.id
port = 8502
}
resource "aws_lb_target_group_attachment" "app_instance_8503" {
target_group_arn = aws_lb_target_group.app_tg_8503.arn
target_id = aws_instance.streamlit_server.id
port = 8503
}
# Update listener rules
resource "aws_lb_listener" "https_8500" {
load_balancer_arn = aws_lb.app_lb.arn
port = "443"
protocol = "HTTPS"
ssl_policy = "ELBSecurityPolicy-2016-08"
# Using variable for ARN
certificate_arn = var.acm_arn_dev
default_action {
type = "forward"
target_group_arn = aws_lb_target_group.app_tg_8500.arn
}
}
resource "aws_lb_listener" "https_8501" {
load_balancer_arn = aws_lb.app_lb.arn
port = "443"
protocol = "HTTPS"
ssl_policy = "ELBSecurityPolicy-2016-08"
certificate_arn = var.acm_arn_dev
default_action {
type = "forward"
target_group_arn = aws_lb_target_group.app_tg_8501.arn
}
}
resource "aws_lb_listener" "https_8502" {
load_balancer_arn = aws_lb.app_lb.arn
port = "443"
protocol = "HTTPS"
ssl_policy = "ELBSecurityPolicy-2016-08"
certificate_arn = var.acm_arn_dev
default_action {
type = "forward"
target_group_arn = aws_lb_target_group.app_tg_8502.arn
}
}
resource "aws_lb_listener" "https_8503" {
load_balancer_arn = aws_lb.app_lb.arn
port = "443"
protocol = "HTTPS"
ssl_policy = "ELBSecurityPolicy-2016-08"
certificate_arn = var.acm_arn_dev
default_action {
type = "forward"
target_group_arn = aws_lb_target_group.app_tg_8503.arn
}
}
streamlit-server/alb_sg.tf
+55
View File
@@ -0,0 +1,55 @@
resource "aws_security_group" "lb_sg" {
name = "${local.global_prefix}-load_balancer_sg"
description = "Security group for the load balancer"
vpc_id = var.vpc_id
ingress {
from_port = 80
to_port = 80
protocol = "tcp"
cidr_blocks = ["172.19.0.0/16"]
}
ingress {
from_port = 443
to_port = 443
protocol = "tcp"
cidr_blocks = ["172.19.0.0/16"]
}
ingress {
from_port = 8500
to_port = 8500
protocol = "tcp"
cidr_blocks = ["172.19.0.0/16"]
}
ingress {
from_port = 8501
to_port = 8501
protocol = "tcp"
cidr_blocks = ["172.19.0.0/16"]
}
ingress {
from_port = 8502
to_port = 8502
protocol = "tcp"
cidr_blocks = ["172.19.0.0/16"]
}
ingress {
from_port = 8503
to_port = 8503
protocol = "tcp"
cidr_blocks = ["172.19.0.0/16"]
}
egress {
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}
}
streamlit-server/main.tf
+10 -7
View File
@@ -1,6 +1,5 @@
provider "aws" {
region = "us-east-2"
profile = "temp_cred"
}
terraform {
@@ -12,15 +11,19 @@ terraform {
}
backend "s3" {
bucket = "doczyai-use2-d-infra-s3-terraform-state" # Parameterize using -backend-config flag with "terraform init"
key = "terraform/streamlit-server/terraform.tfstate" # Parameterize
region = "us-east-2" # Parameterize
profile = "temp_cred" # Parameterize
dynamodb_table = "doczyai-use2-d-infra-dyd-terraform-lock" # Parameterize
# bucket = "doczyai-use2-d-infra-s3-terraform-state" # Parameterize using -backend-config flag with "terraform init"
# key = "terraform/streamlit-server/terraform.tfstate" # Parameterize
# region = "us-east-2" # Parameterize
# profile = "temp_cred" # Parameterize
# dynamodb_table = "doczyai-use2-d-infra-dyd-terraform-lock" # Parameterize
encrypt = true
}
}
# data "aws_acm_certificate" "cert_global" {
# domain = "doczy.aarete.com"
# statuses = ["ISSUED"]
# }
locals {
region_map = {
@@ -198,7 +201,7 @@ resource "aws_instance" "streamlit_server" {
tags = {
Terraform = "true"
Environment = "dev"
Environment = var.environment
}
}
streamlit-server/user_data.sh
+25
View File
@@ -0,0 +1,25 @@
#!/bin/bash
echo '${file("${path.module}/requirements.txt")}' > /tmp/requirements.txt
apt-get update
apt-get install -y python3 python3-pip
pip3 install $(cat /tmp/requirements.txt)
# Create a basic systemd service
cat <<EOT > /etc/systemd/system/streamlit-ui0.service
[Unit]
Description=Streamlit Server Service
[Service]
User=ubuntu
Type=simple
Restart=always
WorkingDirectory=/home/ubuntu/streamlit/doczy.ai/streamlit
ExecStart=/home/ubuntu/.local/bin/streamlit run /home/ubuntu/streamlit/doczy.ai/streamlit/interface_0.py --server.port 8500
[Install]
WantedBy=multi-user.target
EOT
# Enable and start the service
systemctl enable streamlit-ui0.service
systemctl start streamlit-ui0.service
streamlit-server/variables.tf
+7 -2
View File
@@ -33,13 +33,13 @@ variable "vpc_id" {
variable "ubuntu_ami" {
type = string
default = "ami-0b8b44ec9a8f90422"
default = "ami-0de7e97fedfbc6ef6" # Switch to this ami-0b8b44ec9a8f90422
}
variable "ec2_instance_type" {
type = string
default = "t2.small"
default = "t2.large"
}
# EC2 iam role name
@@ -52,4 +52,9 @@ variable "ec2_iam_role_name" {
variable "SecretsNames"{
type = list(string)
default = []
}
variable "acm_arn_dev" {
type = string
default = "arn:aws:acm:us-east-2:660131068782:certificate/3a146592-2b00-4b33-90a6-0d9ab0040d8b"
}