Merged in bugfix/deployConfigProperties (pull request #224)
Bugfix/deployConfigProperties * syncconfig * copydir * specbucket * pipplusdevdeploy * struct * script * image * cleanup * testplan * addallplans * stepnames
This commit is contained in:
Michael McGuinness
+52
-53
@@ -1,6 +1,9 @@
|
||||
definitions:
|
||||
scripts:
|
||||
- script: &aws-context
|
||||
apt-get update;
|
||||
apt-get install -y jq;
|
||||
pip install awscli;
|
||||
export AWS_ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_NO:role/$OIDC_ROLE;
|
||||
export AWS_ROLE_SESSION_NAME=$OIDC_ROLE;
|
||||
export AWS_WEB_IDENTITY_TOKEN_FILE=$(pwd)/web-identity-token;
|
||||
@@ -10,6 +13,7 @@ definitions:
|
||||
export AWS_SECURITY_TOKEN=$(echo $STS_OUTPUT | jq -r '.Credentials.SessionToken');
|
||||
export AWS_REGION='us-east-2';
|
||||
aws sts get-caller-identity;
|
||||
export IS_DEPLOY_ALL=true;
|
||||
steps: &deployment-plan-steps
|
||||
- step:
|
||||
name: Plan devops-pipeline
|
||||
@@ -19,6 +23,8 @@ definitions:
|
||||
- *aws-context
|
||||
- pip install -r requirements.txt
|
||||
- python terraform.py plan --environment=$ENVIRONMENT --module=devops-pipeline/other-resources
|
||||
caches:
|
||||
- pip
|
||||
- step:
|
||||
name: Plan textract-pipeline
|
||||
image: hugree/terraform-with-snowsql:latest
|
||||
@@ -27,6 +33,8 @@ definitions:
|
||||
- *aws-context
|
||||
- pip install -r requirements.txt
|
||||
- python terraform.py plan --environment=$ENVIRONMENT --module=textract-pipeline/terraform
|
||||
caches:
|
||||
- pip
|
||||
- step:
|
||||
name: Plan streamlit-server
|
||||
image: hugree/bitbucket-aws-python38-tf154:latest
|
||||
@@ -35,17 +43,8 @@ definitions:
|
||||
- *aws-context
|
||||
- pip install -r requirements.txt
|
||||
- python terraform.py plan --environment=$ENVIRONMENT --module=streamlit-server
|
||||
# - step:
|
||||
# name: Plan Snowflake Changes
|
||||
# image: python:3.7
|
||||
# size: 2x
|
||||
# script:
|
||||
# - python -m pip install --upgrade pip
|
||||
# - pip install snowflake-connector-python
|
||||
# - pip install schemachange --upgrade
|
||||
# - schemachange -a $SNOWFLAKE_ACCOUNT -u $SNOWFLAKE_USER -r $SNOWFLAKE_ROLE -w $SNOWFLAKE_WAREHOUSE -d $SNOWFLAKE_DATABASE -c $SNOWFLAKE_DATABASE.SCHEMACHANGE.CHANGE_HISTORY --create-change-history-table -v
|
||||
# caches:
|
||||
# - pip
|
||||
caches:
|
||||
- pip
|
||||
- step:
|
||||
image: hugree/terraform-with-snowsql:latest
|
||||
name: Plan Snowsql
|
||||
@@ -53,24 +52,9 @@ definitions:
|
||||
script:
|
||||
- *aws-context
|
||||
- pip install -r requirements.txt
|
||||
- IS_DEPLOY_ALL=true python terraform.py plan --environment=$ENVIRONMENT --module=textract-pipeline/terraform --apply-extra-args="-target=null_resource.snowsql_client_create"
|
||||
# - step:
|
||||
# image: python:3.8
|
||||
# name: "Deploy streamlit to EC2"
|
||||
# oidc: true
|
||||
# script:
|
||||
# - python -m pip install --upgrade pip
|
||||
# - apt-get update && apt-get install -y jq git
|
||||
# - pip install awscli
|
||||
# - *aws-context
|
||||
# - >-
|
||||
# aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID_TF --region $AWS_DEFAULT_REGION --parameters commands='[
|
||||
# "sudo -u ubuntu -i <<EOF",
|
||||
# "export HOME=/home/ubuntu",
|
||||
# "git config --global --add safe.directory /home/ubuntu/doczy.ai",
|
||||
# "if [ -d /home/ubuntu/doczy.ai ]; then cd /home/ubuntu/doczy.ai && git fetch && git pull; else cd /home/ubuntu && git clone git@bitbucket.org:aarete/doczy.ai.git; fi",
|
||||
# "EOF"
|
||||
# ]'
|
||||
- python terraform.py plan --environment=$ENVIRONMENT --module=textract-pipeline/terraform --apply-extra-args="-target=null_resource.snowsql_client_create"
|
||||
caches:
|
||||
- pip
|
||||
steps: &deployment-steps
|
||||
- step:
|
||||
name: Apply devops-pipeline
|
||||
@@ -80,6 +64,17 @@ definitions:
|
||||
- *aws-context
|
||||
- pip install -r requirements.txt
|
||||
- python terraform.py apply --environment=$ENVIRONMENT --module=devops-pipeline/other-resources
|
||||
caches:
|
||||
- pip
|
||||
- step:
|
||||
name: Sync S3 Config
|
||||
image: python:3.8
|
||||
oidc: true
|
||||
script:
|
||||
- *aws-context
|
||||
- aws s3 sync ./textract-pipeline/src/s3-files s3://${TEXTRACT_BUCKET}
|
||||
caches:
|
||||
- pip
|
||||
- step:
|
||||
name: Apply textract-pipeline
|
||||
image: hugree/terraform-with-snowsql:latest
|
||||
@@ -88,6 +83,8 @@ definitions:
|
||||
- *aws-context
|
||||
- pip install -r requirements.txt
|
||||
- python terraform.py apply --environment=$ENVIRONMENT --module=textract-pipeline/terraform
|
||||
caches:
|
||||
- pip
|
||||
- step:
|
||||
name: Apply streamlit-server
|
||||
image: hugree/bitbucket-aws-python38-tf154:latest
|
||||
@@ -96,40 +93,35 @@ definitions:
|
||||
- *aws-context
|
||||
- pip install -r requirements.txt
|
||||
- python terraform.py apply --environment=$ENVIRONMENT --module=streamlit-server
|
||||
caches:
|
||||
- pip
|
||||
- step:
|
||||
name: "Deploy Snowflake Changes"
|
||||
image: python:3.7
|
||||
size: 2x
|
||||
script:
|
||||
- python -m pip install --upgrade pip
|
||||
- pip install snowflake-connector-python
|
||||
- pip install schemachange --upgrade
|
||||
- schemachange -a $SNOWFLAKE_ACCOUNT -u $SNOWFLAKE_USER -r $SNOWFLAKE_ROLE -w $SNOWFLAKE_WAREHOUSE -d $SNOWFLAKE_DATABASE -c $SNOWFLAKE_DATABASE.SCHEMACHANGE.CHANGE_HISTORY --create-change-history-table -v
|
||||
caches:
|
||||
- pip
|
||||
#condition:
|
||||
# changesets:
|
||||
# includePaths:
|
||||
# - snowflake/**/*
|
||||
# - snowflake/**/**/*
|
||||
# - snowflake/**/**/**/*
|
||||
- step:
|
||||
image: hugree/terraform-with-snowsql:latest
|
||||
name: Run Snowsql
|
||||
name: Apply Snowsql
|
||||
oidc: true
|
||||
script:
|
||||
- *aws-context
|
||||
- pip install -r requirements.txt
|
||||
- IS_DEPLOY_ALL=true python terraform.py apply --environment=$ENVIRONMENT --module=textract-pipeline/terraform --apply-extra-args="-target=null_resource.snowsql_client_create"
|
||||
- python terraform.py apply --environment=$ENVIRONMENT --module=textract-pipeline/terraform --apply-extra-args="-target=null_resource.snowsql_client_create"
|
||||
caches:
|
||||
- pip
|
||||
- step:
|
||||
image: python:3.8
|
||||
name: "Deploy streamlit to EC2"
|
||||
oidc: true
|
||||
script:
|
||||
- python -m pip install --upgrade pip
|
||||
- apt-get update && apt-get install -y jq git
|
||||
- pip install awscli
|
||||
- *aws-context
|
||||
- apt-get install -y git
|
||||
- >-
|
||||
aws ssm send-command --document-name "AWS-RunShellScript" --instance-ids $INSTANCE_ID_TF --region $AWS_DEFAULT_REGION --parameters commands='[
|
||||
"sudo -u ubuntu -i <<EOF",
|
||||
@@ -138,35 +130,43 @@ definitions:
|
||||
"if [ -d /home/ubuntu/doczy.ai ]; then cd /home/ubuntu/doczy.ai && git fetch && git pull; else cd /home/ubuntu && git clone git@bitbucket.org:aarete/doczy.ai.git; fi",
|
||||
"EOF"
|
||||
]'
|
||||
#condition:
|
||||
# changesets:
|
||||
# includePaths:
|
||||
# - streamlit/*
|
||||
# - streamlit/**/*
|
||||
caches:
|
||||
- pip
|
||||
|
||||
pipelines:
|
||||
default:
|
||||
- step:
|
||||
image: python:3.8
|
||||
name: Install Pip
|
||||
script:
|
||||
- python -m pip install --upgrade pip
|
||||
caches:
|
||||
- pip
|
||||
- step:
|
||||
name: Lint the code
|
||||
image: hugree/bitbucket-aws-python38-tf154:latest
|
||||
script:
|
||||
- pip install -r dev-requirements.txt
|
||||
- python -m black --check .
|
||||
caches:
|
||||
- pip
|
||||
branches:
|
||||
main:
|
||||
- step:
|
||||
name: Build and Test
|
||||
script:
|
||||
- echo Build and test pipeline will go here
|
||||
- stage:
|
||||
name: deployment-dev-plan
|
||||
deployment: development-plan
|
||||
steps: *deployment-plan-steps
|
||||
- stage:
|
||||
name: deployment-dev
|
||||
deployment: development
|
||||
trigger: automatic
|
||||
steps: *deployment-steps
|
||||
- stage:
|
||||
name: deployment-plan-uat
|
||||
deployment: uat
|
||||
trigger: manual
|
||||
name: deployment-uat-plan
|
||||
deployment: uat-plan
|
||||
steps: *deployment-plan-steps
|
||||
- stage:
|
||||
name: deployment-uat
|
||||
@@ -174,9 +174,8 @@ pipelines:
|
||||
trigger: manual
|
||||
steps: *deployment-steps
|
||||
- stage:
|
||||
name: deployment-plan-prod
|
||||
deployment: production
|
||||
trigger: manual
|
||||
name: deployment-prod-plan
|
||||
deployment: production-plan
|
||||
steps: *deployment-plan-steps
|
||||
- stage:
|
||||
name: deployment-prod
|
||||
|
||||
Reference in New Issue
Block a user