diff --git a/.github/workflows/destroy_aws_deployment.yml b/.github/workflows/destroy_aws_deployment.yml new file mode 100644 index 0000000..7002b3c --- /dev/null +++ b/.github/workflows/destroy_aws_deployment.yml @@ -0,0 +1,50 @@ +name: Destroy HydroServer AWS Cloud Deployment + +on: + workflow_dispatch: + inputs: + environment: + description: 'Enter a deployment environment name.' + required: true + +permissions: + id-token: write + contents: read + +jobs: + destroy-deployment: + runs-on: ubuntu-20.04 + environment: ${{ github.event.inputs.environment }} + defaults: + run: + working-directory: ./terraform/aws + steps: + - name: configureawscredentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.AWS_IAM_ROLE }} + role-session-name: create-hydroserver-resources + aws-region: ${{ vars.AWS_REGION }} + + - name: Checkout Repo + uses: actions/checkout@v3 + with: + ref: main + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v1 + + - name: Terraform Init + run: terraform init -backend-config="bucket=${{ vars.TERRAFORM_BUCKET }}" -backend-config="region=${{ vars.AWS_REGION }}" -backend-config="key=state/aws_application_${{ github.event.inputs.environment }}" + + - name: Terraform Plan + id: plan + run: terraform plan -destroy -no-color -var instance="${{ github.event.inputs.environment }}" -var region="${{ vars.AWS_REGION }}" + continue-on-error: true + + - name: Terraform Plan Status + if: steps.plan.outcome == 'failure' + run: exit 1 + + - name: Terraform Apply + run: terraform apply -destroy -auto-approve -var instance="${{ github.event.inputs.environment }}" -var region="${{ vars.AWS_REGION }}" diff --git a/.github/workflows/destroy_self_hosted_timescale.yml b/.github/workflows/destroy_self_hosted_timescale.yml new file mode 100644 index 0000000..61aaed9 --- /dev/null +++ b/.github/workflows/destroy_self_hosted_timescale.yml @@ -0,0 +1,70 @@ +name: Destroy Self Hosted TimescaleScale Database Cluster with Backup + +on: + workflow_dispatch: + inputs: + environment: + description: 'Enter a deployment environment name.' + required: true + db-user: + description: 'Enter a username for the timescale db' + required: true + db-password: + description: 'Enter a password for the timescale db' + required: true + aws-instance-type: + description: 'Enter aws ec2 isntance type, please use the same instance type as the one used to create the database' + required: false + +permissions: + id-token: write + contents: read + +jobs: + destroy-deployment: + runs-on: ubuntu-20.04 + environment: ${{ github.event.inputs.environment }} + steps: + - name: configureawscredentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.AWS_IAM_ROLE }} + role-session-name: create-hydroserver-resources + aws-region: ${{ vars.AWS_REGION }} + + - name: Checkout Repo + uses: actions/checkout@v3 + with: + ref: main + path: ops + + - name: Get EC2 Instance Type + id: get_instance_type + run: echo "instance_type=${{ github.event.inputs.aws-instance-type || 't2.micro' }}" >> $GITHUB_OUTPUT + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + + - name: Terraform Init + working-directory: ./ops/terraform/self_hosted_timescale + run: terraform init -backend-config="bucket=${{ vars.TERRAFORM_BUCKET }}" -backend-config="region=${{ vars.AWS_REGION }}" -backend-config="key=state/timescale_self_hosted_database_${{ github.event.inputs.environment }}" + + - name: Terraform Plan + id: plan + working-directory: ./ops/terraform/self_hosted_timescale + run: terraform plan -destroy -no-color -var instance="${{ github.event.inputs.environment }}" -var region="${{ vars.AWS_REGION }}" -var db_user="${{ github.event.inputs.db-user }}" -var db_password="${{ github.event.inputs.db-password }}" -var access_key="${{ secrets.USER_ACCESS_KEY_ID }}" -var secret_key="${{ secrets.USER_SECRET_ACCESS_KEY }}" -var private_key="${{ secrets.SELF_HOSTED_TIMESCALE_PRIVATE_KEY }}" -var public_key="${{ secrets.SELF_HOSTED_TIMESCALE_PUBLIC_KEY }}" -var aws_type="${{ github.event.inputs.aws-instance-type || steps.get_instance_type.outputs.instance_type }}" + continue-on-error: true + + - name: Terraform Plan Status + if: steps.plan.outcome == 'failure' + run: exit 1 + + - name: Terraform Apply + working-directory: ./ops/terraform/self_hosted_timescale + run: | + terraform apply -destroy -auto-approve -var instance="${{ github.event.inputs.environment }}" -var region="${{ vars.AWS_REGION }}" -var db_user="${{ github.event.inputs.db-user }}" -var db_password="${{ github.event.inputs.db-password }}" -var access_key="${{ secrets.TIMESCALE_ACCESS_KEY }}" -var secret_key="${{ secrets.TIMESCALE_SECRET_KEY }}" -var private_key="${{ secrets.SELF_HOSTED_TIMESCALE_PRIVATE_KEY }}" -var public_key="${{ secrets.SELF_HOSTED_TIMESCALE_PUBLIC_KEY }}" -var aws_type="${{ github.event.inputs.aws-instance-type || steps.get_instance_type.outputs.instance_type }}" + + - name: Delete the Connection Details in S3 + working-directory: ./ops/terraform/self_hosted_timescale + run: | + aws s3 rm s3://${{ vars.TERRAFORM_BUCKET }}/output/timescale_${{ github.event.inputs.environment }}_connection.txt diff --git a/.github/workflows/self_hosted_timescale_ create_database.yml b/.github/workflows/self_hosted_timescale_ create_database.yml new file mode 100644 index 0000000..f666cf2 --- /dev/null +++ b/.github/workflows/self_hosted_timescale_ create_database.yml @@ -0,0 +1,123 @@ +name: Create Self Hosted TimescaleScale Database Cluster with Backup For HydroServer + +on: + workflow_dispatch: + inputs: + environment: + description: 'Enter a deployment environment name.' + required: true + superuser-email: + description: 'Enter the email for the Django superuser.' + required: true + superuser-password: + description: 'Enter the password for the Django superuser.' + required: true + partition-interval: + description: 'Enter a partition interval in days.' + default: '365' + required: true + db-user: + description: 'Enter a username for the timescale db' + required: true + db-password: + description: 'Enter a password for the timescale db' + required: true + aws-instance-type: + description: 'Enter aws ec2 isntance type (default is t2.micro)' + required: false + hydroserver-version: + description: 'Enter a version of HydroServer to use. Leave blank to use the latest version.' + required: false + +permissions: + id-token: write + contents: read + +jobs: + setup-deployment: + runs-on: ubuntu-20.04 + environment: ${{ github.event.inputs.environment }} + steps: + - name: configureawscredentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.AWS_IAM_ROLE }} + role-session-name: create-hydroserver-resources + aws-region: ${{ vars.AWS_REGION }} + + - name: Checkout Repo + uses: actions/checkout@v3 + with: + ref: main + path: ops + + - name: Get Latest HydroServer Version + id: get_latest_tag + run: echo "tag=$(curl -sL https://api.github.com/repos/hydroserver2/hydroserver-api-services/releases/latest | jq -r '.tag_name')" >> $GITHUB_OUTPUT + + - name: Get EC2 Instance Type + id: get_instance_type + run: echo "instance_type=${{ github.event.inputs.aws-instance-type || 't2.micro' }}" >> $GITHUB_OUTPUT + + - name: Checkout Backend Repo + uses: actions/checkout@v4 + with: + repository: hydroserver2/hydroserver-api-services + ref: refs/tags/${{ github.event.inputs.hydroserver-version || steps.get_latest_tag.outputs.tag }} + path: backend + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + + - name: Terraform Init + working-directory: ./ops/terraform/self_hosted_timescale + run: terraform init -backend-config="bucket=${{ vars.TERRAFORM_BUCKET }}" -backend-config="region=${{ vars.AWS_REGION }}" -backend-config="key=state/timescale_self_hosted_database_${{ github.event.inputs.environment }}" + + - name: Terraform Plan + id: plan + working-directory: ./ops/terraform/self_hosted_timescale + run: terraform plan -no-color -var instance="${{ github.event.inputs.environment }}" -var region="${{ vars.AWS_REGION }}" -var db_user="${{ github.event.inputs.db-user }}" -var db_password="${{ github.event.inputs.db-password }}" -var access_key="${{ secrets.USER_ACCESS_KEY_ID }}" -var secret_key="${{ secrets.USER_SECRET_ACCESS_KEY }}" -var private_key="${{ secrets.SELF_HOSTED_TIMESCALE_PRIVATE_KEY }}" -var public_key="${{ secrets.SELF_HOSTED_TIMESCALE_PUBLIC_KEY }}" -var aws_type="${{ github.event.inputs.aws-instance-type || steps.get_instance_type.outputs.instance_type }}" + continue-on-error: true + + - name: Terraform Plan Status + if: steps.plan.outcome == 'failure' + run: exit 1 + + - name: Terraform Apply + working-directory: ./ops/terraform/self_hosted_timescale + run: | + terraform apply -auto-approve -var instance="${{ github.event.inputs.environment }}" -var region="${{ vars.AWS_REGION }}" -var db_user="${{ github.event.inputs.db-user }}" -var db_password="${{ github.event.inputs.db-password }}" -var access_key="${{ secrets.USER_ACCESS_KEY_ID }}" -var secret_key="${{ secrets.USER_SECRET_ACCESS_KEY }}" -var private_key="${{ secrets.SELF_HOSTED_TIMESCALE_PRIVATE_KEY }}" -var public_key="${{ secrets.SELF_HOSTED_TIMESCALE_PUBLIC_KEY }}" -var aws_type="${{ github.event.inputs.aws-instance-type || steps.get_instance_type.outputs.instance_type }}" + echo "HOSTNAME=$(terraform output -json | jq -r '.self_hosted_tsdb_hostname.value')" > timescale_${{ github.event.inputs.environment }}_connection.txt + echo "PORT=5432" >> timescale_${{ github.event.inputs.environment }}_connection.txt + echo "PASSWORD=${{ github.event.inputs.db-password }}" >> timescale_${{ github.event.inputs.environment }}_connection.txt + echo "CONNECTION_STRING=postgresql://${{ github.event.inputs.db-user }}:${{ github.event.inputs.db-password }}@$(terraform output -json | jq -r '.self_hosted_tsdb_hostname.value'):5432/tsdb" >> timescale_${{ github.event.inputs.environment }}_connection.txt + cat << EOF > ../../../backend/.env + PROXY_BASE_URL=http://127.0.0.1:8000 + DATABASE_URL=postgresql://${{ github.event.inputs.db-user }}:${{ github.event.inputs.db-password }}@$(terraform output -json | jq -r '.self_hosted_tsdb_hostname.value'):5432/tsdb + DEPLOYED=True + EOF + + - name: Upload Connection Details to S3 + working-directory: ./ops/terraform/self_hosted_timescale + run: | + aws s3 cp timescale_${{ github.event.inputs.environment }}_connection.txt s3://${{ vars.TERRAFORM_BUCKET }}/output/timescale_${{ github.event.inputs.environment }}_connection.txt + + - name: Install Django Dependencies + working-directory: ./backend + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install pyopenssl --upgrade + + - name: Run Database Setup Commands + working-directory: ./backend + env: + DJANGO_SETTINGS_MODULE: hydroserver.settings + DJANGO_SUPERUSER_EMAIL: ${{ github.event.inputs.superuser-email }} + DJANGO_SUPERUSER_PASSWORD: ${{ github.event.inputs.superuser-password }} + DJANGO_SUPERUSER_FIRST_NAME: ADMIN + DJANGO_SUPERUSER_LAST_NAME: ADMIN + run: | + python manage.py migrate + python manage.py configure_timescaledb --partition-interval-days ${{ github.event.inputs.partition-interval }} + python manage.py createsuperuser --noinput diff --git a/.github/workflows/timescale_create_database.yml b/.github/workflows/timescale_create_database.yml index 1d20ece..d5d098b 100644 --- a/.github/workflows/timescale_create_database.yml +++ b/.github/workflows/timescale_create_database.yml @@ -16,7 +16,7 @@ on: description: 'Enter a partition interval in days.' default: '365' required: true - hydroserver-version: + hydroserver-version: description: 'Enter a version of HydroServer to use. Leave blank to use the latest version.' required: false @@ -35,7 +35,7 @@ jobs: role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/${{ vars.AWS_IAM_ROLE }} role-session-name: create-hydroserver-resources aws-region: ${{ vars.AWS_REGION }} - + - name: Checkout Repo uses: actions/checkout@v3 with: @@ -45,7 +45,7 @@ jobs: - name: Get Latest HydroServer Version id: get_latest_tag run: echo "tag=$(curl -sL https://api.github.com/repos/hydroserver2/hydroserver-api-services/releases/latest | jq -r '.tag_name')" >> $GITHUB_OUTPUT - + - name: Checkout Backend Repo uses: actions/checkout@v4 with: @@ -88,14 +88,14 @@ jobs: working-directory: ./ops/terraform/timescale run: | aws s3 cp timescale_${{ github.event.inputs.environment }}_connection.txt s3://${{ vars.TERRAFORM_BUCKET }}/output/timescale_${{ github.event.inputs.environment }}_connection.txt - + - name: Install Django Dependencies working-directory: ./backend run: | python -m pip install --upgrade pip pip install -r requirements.txt pip install pyopenssl --upgrade - + - name: Run Database Setup Commands working-directory: ./backend env: @@ -107,4 +107,4 @@ jobs: run: | python manage.py migrate python manage.py configure_timescaledb --partition-interval-days ${{ github.event.inputs.partition-interval }} - python manage.py createsuperuser --noinput + python manage.py createsuperuser --noinput diff --git a/terraform/aws/elasticbeanstalk.tf b/terraform/aws/elasticbeanstalk.tf index f7dc007..480ab8b 100644 --- a/terraform/aws/elasticbeanstalk.tf +++ b/terraform/aws/elasticbeanstalk.tf @@ -14,7 +14,7 @@ resource "aws_elastic_beanstalk_application" "hydroserver_django_app" { resource "aws_elastic_beanstalk_environment" "hydroserver_django_env" { name = "hydroserver-${var.instance}-env" application = aws_elastic_beanstalk_application.hydroserver_django_app.name - solution_stack_name = "64bit Amazon Linux 2 v3.5.12 running Python 3.8" + solution_stack_name = "64bit Amazon Linux 2 v3.6.0 running Python 3.8" setting { namespace = "aws:elasticbeanstalk:environment" diff --git a/terraform/self_hosted_timescale/app-instances.tf b/terraform/self_hosted_timescale/app-instances.tf new file mode 100644 index 0000000..16c1602 --- /dev/null +++ b/terraform/self_hosted_timescale/app-instances.tf @@ -0,0 +1,62 @@ +resource "aws_instance" "primary_1" { + ami = var.aws_ami + instance_type = var.aws_type + security_groups = ["${aws_security_group.swarm.name}"] + key_name = aws_key_pair.deployer.key_name + connection { + host = self.public_ip + user = "ec2-user" + private_key = var.private_key + } + provisioner "remote-exec" { + inline = [ + "sudo yum update -y", + "sudo yum install git -y", + "sudo yum install -y docker", + "sudo service docker start", + "sudo curl -L https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose", + "sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose", + "sudo chmod +x /usr/local/bin/docker-compose;", + "sudo docker network create tsdb", + "sudo docker run --restart=unless-stopped --name=tsdb_db -d -p 5432:5432 --network tsdb -e POSTGRES_DB=tsdb -e POSTGRES_USER=${var.db_user} -e POSTGRES_PASSWORD=${var.db_password} -v $(pwd)/data:/var/lib/postgresql/data timescale/timescaledb:latest-pg13", + "sudo docker run -d --restart=unless-stopped --name=postgres_backup --network tsdb -e SCHEDULE='@daily' -e S3_REGION=${var.region} -e S3_ACCESS_KEY_ID=${var.access_key} -e S3_SECRET_ACCESS_KEY=${var.secret_key} -e S3_BUCKET=timescale-backup-${var.instance}-${data.aws_caller_identity.current.account_id} -e POSTGRES_DATABASE=tsdb -e POSTGRES_USER=${var.db_user} -e POSTGRES_HOST=tsdb_db -e POSTGRES_PASSWORD=${var.db_password} -e S3_PREFIX=backup -e POSTGRES_EXTRA_OPTS='--format=plain --quote-all-identifiers --no-tablespaces --no-owner --no-privileges' schickling/postgres-backup-s3" + ] + } + tags = { + Name = "tsdb-primary-${var.instance}-${data.aws_caller_identity.current.account_id}" + } + depends_on = [ + aws_s3_bucket.timescale_backup_bucket + ] +} +resource "aws_instance" "replica_1" { + ami = var.aws_ami + instance_type = var.aws_type + security_groups = ["${aws_security_group.swarm.name}"] + key_name = aws_key_pair.deployer.key_name + connection { + host = self.public_ip + user = "ec2-user" + private_key = var.private_key + } + provisioner "remote-exec" { + inline = [ + "sudo yum update -y", + "sudo yum install git -y", + "sudo yum install -y docker", + "sudo service docker start", + "sudo curl -L https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose", + "sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose", + "sudo chmod +x /usr/local/bin/docker-compose;", + "sudo docker network create tsdb", + "sudo docker run --restart=unless-stopped --name=tsdb_db -d -p 5432:5432 --network tsdb -e POSTGRES_DB=tsdb -e POSTGRES_USER=${var.db_user} -e POSTGRES_PASSWORD=${var.db_password} -v $(pwd)/data:/var/lib/postgresql/data timescale/timescaledb:latest-pg13", + "sudo docker run -d --restart=unless-stopped --name=postgres_backup --network tsdb -e SCHEDULE='@daily' -e S3_REGION=${var.region} -e S3_ACCESS_KEY_ID=${var.access_key} -e S3_SECRET_ACCESS_KEY=${var.secret_key} -e S3_BUCKET=timescale-backup-${var.instance}-${data.aws_caller_identity.current.account_id} -e POSTGRES_DATABASE=tsdb -e POSTGRES_USER=${var.db_user} -e POSTGRES_HOST=tsdb_db -e POSTGRES_PASSWORD=${var.db_password} -e S3_PREFIX=backup -e POSTGRES_EXTRA_OPTS='--format=plain --quote-all-identifiers --no-tablespaces --no-owner --no-privileges' schickling/postgres-backup-s3" + ] + } + tags = { + Name = "tsdb-replica-${var.instance}-${data.aws_caller_identity.current.account_id}" + } + depends_on = [ + aws_s3_bucket.timescale_backup_bucket + ] +} diff --git a/terraform/self_hosted_timescale/key-pairs.tf b/terraform/self_hosted_timescale/key-pairs.tf new file mode 100644 index 0000000..27d6819 --- /dev/null +++ b/terraform/self_hosted_timescale/key-pairs.tf @@ -0,0 +1,4 @@ +resource "aws_key_pair" "deployer" { + key_name = "deploy-tsdb-${var.instance}-${data.aws_caller_identity.current.account_id}" + public_key = var.public_key +} diff --git a/terraform/self_hosted_timescale/main.tf b/terraform/self_hosted_timescale/main.tf new file mode 100644 index 0000000..611bcf5 --- /dev/null +++ b/terraform/self_hosted_timescale/main.tf @@ -0,0 +1,16 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.16" + } + } + backend "s3" {} + required_version = ">= 1.2.0" +} + +provider "aws" { + region = var.region +} + +data "aws_caller_identity" "current" {} diff --git a/terraform/self_hosted_timescale/outputs.tf b/terraform/self_hosted_timescale/outputs.tf new file mode 100644 index 0000000..3477703 --- /dev/null +++ b/terraform/self_hosted_timescale/outputs.tf @@ -0,0 +1,3 @@ +output "self_hosted_tsdb_hostname" { + value = aws_instance.primary_1.public_ip +} diff --git a/terraform/self_hosted_timescale/s3.tf b/terraform/self_hosted_timescale/s3.tf new file mode 100644 index 0000000..9c6ac70 --- /dev/null +++ b/terraform/self_hosted_timescale/s3.tf @@ -0,0 +1,30 @@ +# Creation of S3 bucket for TimescaleDB backup +resource "aws_s3_bucket" "timescale_backup_bucket" { + bucket = "timescale-backup-${var.instance}-${data.aws_caller_identity.current.account_id}" +} + + +# ------------------------------------------------ # +# S3 Restrict Public Access # +# ------------------------------------------------ # +resource "aws_s3_bucket_public_access_block" "timescale_backup_bucket" { + bucket = aws_s3_bucket.timescale_backup_bucket.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +# ------------------------------------------------ # +# S3 Ownership Controls # +# ------------------------------------------------ # +resource "aws_s3_bucket_ownership_controls" "timescale_backup_bucket" { + bucket = aws_s3_bucket.timescale_backup_bucket.id + rule { + object_ownership = "BucketOwnerEnforced" + } + depends_on = [aws_s3_bucket_public_access_block.timescale_backup_bucket] +} + + diff --git a/terraform/self_hosted_timescale/security-group.tf b/terraform/self_hosted_timescale/security-group.tf new file mode 100644 index 0000000..eda70ea --- /dev/null +++ b/terraform/self_hosted_timescale/security-group.tf @@ -0,0 +1,59 @@ +/* Default security group */ +resource "aws_security_group" "swarm" { + name = "swarm-group-tdsb-${var.instance}-${data.aws_caller_identity.current.account_id}" + description = "Default security group that allows inbound and outbound traffic from all instances in the VPC" + + ingress { + from_port = "0" + to_port = "0" + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + self = true + } + + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + ingress { + from_port = 5432 + to_port = 5432 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + ingress { + from_port = 3000 + to_port = 3000 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + from_port = "0" + to_port = "0" + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + self = true + } + egress { + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + egress { + from_port = 5432 + to_port = 5432 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + egress { + from_port = 3000 + to_port = 3000 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } +} diff --git a/terraform/self_hosted_timescale/variables.tf b/terraform/self_hosted_timescale/variables.tf new file mode 100644 index 0000000..0f7f24c --- /dev/null +++ b/terraform/self_hosted_timescale/variables.tf @@ -0,0 +1,15 @@ +// Default AWS Access Credentials +variable "access_key" {} +variable "secret_key" {} +variable "instance" {} +variable "region" {} +variable "db_user" {} +variable "db_password" {} +variable "aws_ami" { + default = "ami-00beae93a2d981137" +} +variable "private_key" {} +variable "public_key" {} +variable "aws_type" {} + +