From 0a6397254f80845e44acb3538a9b95b57454876f Mon Sep 17 00:00:00 2001 From: Tom Juntunen Date: Tue, 10 May 2022 23:01:36 -0700 Subject: [PATCH 1/8] adding psql and mssql services, updated readme --- README.md | 29 +++++++++++++++++++++++++++ db/mssql/Dockerfile | 36 ++++++++++++++++++++++++++++++++++ db/mssql/entrypoint.sh | 2 ++ db/mssql/run-initialization.sh | 6 ++++++ db/postgres/my-postgres.conf | 30 ++++++++++++++++++++++++++++ docker-compose.yaml | 6 ++++++ mssql.yml | 23 ++++++++++++++++++++++ psql.yml | 26 ++++++++++++++++++++++++ 8 files changed, 158 insertions(+) create mode 100644 db/mssql/Dockerfile create mode 100644 db/mssql/entrypoint.sh create mode 100644 db/mssql/run-initialization.sh create mode 100644 db/postgres/my-postgres.conf create mode 100644 mssql.yml create mode 100644 psql.yml diff --git a/README.md b/README.md index 8a1cd08..a3d85fa 100644 --- a/README.md +++ b/README.md @@ -21,3 +21,32 @@ Add your unit tests to files inside the `tests` folder ... name your files `test ## Data Flow High level description of data source(s) and sink(s), as well as the general pattern and data flow through the pipeline. Discuss any assumptions made. + +## Environment +Compose supports declaring default environment variables in an environment file named .env placed in the project directory. + +The following syntax rules apply to the .env file: + +- Compose expects each line in an env file to be in VAR=VAL format. +- Lines beginning with # are processed as comments and ignored. +- Blank lines are ignored. +- There is no special handling of quotation marks. This means that they are part of the VAL. + +## Database +For persisting data into SQL, a sql container using an external volume allows +for the database container to restart without losing data. + +The database is added to a bridge network where other hosts on the network can +reach the database. + +`docker network create test-network` + +Start a PostgreSQL database + +`docker volume create postgres-volume` +`docker-compose -f psql.yml up --build -d` + +Start a SQL Server database + +`docker volume create mssql-volume` +`docker-compose -f mssql.yml up --build -d` \ No newline at end of file diff --git a/db/mssql/Dockerfile b/db/mssql/Dockerfile new file mode 100644 index 0000000..122adbf --- /dev/null +++ b/db/mssql/Dockerfile @@ -0,0 +1,36 @@ +FROM mcr.microsoft.com/mssql/server:2019-latest + +USER root + +# Install dependencies for installing mssql-tools +RUN apt-get update \ + && apt-get install -y gnupg \ + && apt-get -y install curl + +# adding custom MS repository +RUN curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - +RUN curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | tee /etc/apt/sources.list.d/msprod.list + +# Install SQL Server Tools +RUN apt-get update +RUN apt-get -y install mssql-tools +RUN echo 'export PATH="$PATH:/opt/mssql-tools/bin"' >> ~/.bashrc +RUN #!/bin/bash source ~/.bashrc + +# copy over initialization scripts that create our sql databases +RUN mkdir -p /usr/src/app +WORKDIR /usr/src/app +COPY ./entrypoint.sh /usr/src/app +COPY ./run-initialization.sh /usr/src/app + +# set permissions allowing the shell script to be executed +RUN chmod +x /usr/src/app/entrypoint.sh +RUN chmod +x /usr/src/app/run-initialization.sh + +# expose the default mssql port from the container +EXPOSE 1433 + +USER mssql + +# override Dockerfile CMD entry from base image (which runs SQL server), with our own CMD command, which runs our custom script: +CMD ["/bin/bash", "./entrypoint.sh"] \ No newline at end of file diff --git a/db/mssql/entrypoint.sh b/db/mssql/entrypoint.sh new file mode 100644 index 0000000..7d18e85 --- /dev/null +++ b/db/mssql/entrypoint.sh @@ -0,0 +1,2 @@ +# Run initialization script and start ms sql server +/usr/src/app/run-initialization.sh & /opt/mssql/bin/sqlservr \ No newline at end of file diff --git a/db/mssql/run-initialization.sh b/db/mssql/run-initialization.sh new file mode 100644 index 0000000..2e770b3 --- /dev/null +++ b/db/mssql/run-initialization.sh @@ -0,0 +1,6 @@ +# Wait to be sure that SQL Server came up +sleep 60s + +# Run the setup script to create the DB and the schema in the DB +# Note: make sure that your password matches what is in the Dockerfile +/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P $MSSQL_SA_PASSWORD -d master -i sql/create-database.sql diff --git a/db/postgres/my-postgres.conf b/db/postgres/my-postgres.conf new file mode 100644 index 0000000..0c86555 --- /dev/null +++ b/db/postgres/my-postgres.conf @@ -0,0 +1,30 @@ +listen_addresses = '*' + +# DB Version: 14.0 +# OS Type: linux +# DB Type: test + +# Performance tweaks (sample values provided) +max_connections = 1000 +shared_buffers = 15872MB +effective_cache_size = 47616MB +maintenance_work_mem = 2GB +checkpoint_completion_target = 0.7 +wal_buffers = 16MB +default_statistics_target = 100 +random_page_cost = 1.1 +effective_io_concurrency = 200 +work_mem = 4063kB +min_wal_size = 1GB +max_wal_size = 4GB +max_worker_processes = 12 +max_parallel_workers_per_gather = 4 + +wal_level = logical # options are: replica, minimal, logical +hot_standby = on +max_wal_senders = 10 +max_replication_slots = 10 +hot_standby_feedback = on + +# For more information on WAL: +# https://www.postgresql.org/docs/current/runtime-config-wal.html \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 0619289..4264fea 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -7,3 +7,9 @@ services: volumes: - .:/app command: python3 -m pytest + networks: + - test-network + +networks: + test-network: + external: True diff --git a/mssql.yml b/mssql.yml new file mode 100644 index 0000000..dfd39a6 --- /dev/null +++ b/mssql.yml @@ -0,0 +1,23 @@ +services: + mssql: + build: ./db/mssql + hostname: mssql + environment: + - MSSQL_SA_PASSWORD=${MSSQL_SA_PASSWORD} + - MSSQL_SQL_CONNECTION=${MSSQL_SQL_CONNECTION} + - ACCEPT_EULA="Y" + ports: + - "1440:1433" + volumes: + - mssql-volume:/var/opt/mssql + restart: always + networks: + - test-network + +volumes: + mssql-volume: + external: True + +networks: + test-network: + external: True \ No newline at end of file diff --git a/psql.yml b/psql.yml new file mode 100644 index 0000000..6afb752 --- /dev/null +++ b/psql.yml @@ -0,0 +1,26 @@ +services: + psql: + image: postgres:14.1-alpine + hostname: postgres + environment: + - POSTGRES_DB=${POSTGRES_DB} + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - SQL_CONNECTION=${SQL_CONNECTION} + ports: + - "5440:5432" + volumes: + - postgres-volume:/var/lib/postgresql/data + - ./my-postgres.conf:/etc/postgresql/postgres.conf + - ./db/postgres/build/create_database.sql:/docker-entrypoint-initdb.d/create_database.sql + restart: always + networks: + - test-network + +volumes: + postgres-volume: + external: True + +networks: + test-network: + external: True From 6c74b5841f11c0c577f661ed5deb392c1212d221 Mon Sep 17 00:00:00 2001 From: Tom Juntunen Date: Tue, 10 May 2022 23:02:51 -0700 Subject: [PATCH 2/8] don't add .env to gitignore for test purposes; this file can be used to set env vars on the container easily --- .env | 9 +++++++++ .gitignore | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 0000000..13923b2 --- /dev/null +++ b/.env @@ -0,0 +1,9 @@ +POSTGRES_DB=postgres +POSTGRES_USER=postgres +POSTGRES_PASSWORD=p0stgr3s +PSQL_CONNECTION=postgresql://postgres:p0stgr3s@psql:5432 +HOST_PSQL_CONNECTION=postgresql://postgres:p0stgr3s@localhost:5440 + +MSSQL_SA_PASSWORD=mssql4docker# +MSSQL_SQL_CONNECTION=mssql+pyodbc://sa:mssql4docker#@mssql:1433?driver=ODBC+Driver+17+for+SQL+Server +HOST_MSSQL_CONNECTION=mssql+pyodbc://sa:mssql4docker#@localhost:1440?driver=ODBC+Driver+17+for+SQL+Server diff --git a/.gitignore b/.gitignore index b6e4761..e240f98 100644 --- a/.gitignore +++ b/.gitignore @@ -102,7 +102,7 @@ celerybeat.pid *.sage.py # Environments -.env +# .env using this to pass env variables to docker compose .venv env/ venv/ From e18e0ad54f38b5611f4399e2a5d111f4bc490ed3 Mon Sep 17 00:00:00 2001 From: Tom Juntunen <43662466+tom-juntunen@users.noreply.github.com> Date: Sun, 15 May 2022 15:11:01 -0700 Subject: [PATCH 3/8] Create create-database.sql --- db/postgres/create-database.sql | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 db/postgres/create-database.sql diff --git a/db/postgres/create-database.sql b/db/postgres/create-database.sql new file mode 100644 index 0000000..4321630 --- /dev/null +++ b/db/postgres/create-database.sql @@ -0,0 +1,6 @@ + CREATE DATABASE test_database; + + CREATE ROLE testadmin WITH PASSWORD 'm4k3rb0t#' LOGIN; + + GRANT ALL PRIVILEGES ON DATABASE test_database TO testadmin; + From 81d349e3ea3c32ec3b03679c17495bf534413b57 Mon Sep 17 00:00:00 2001 From: Tom Juntunen <43662466+tom-juntunen@users.noreply.github.com> Date: Sun, 15 May 2022 15:11:53 -0700 Subject: [PATCH 4/8] Create ignore --- db/postgres/build/ignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 db/postgres/build/ignore diff --git a/db/postgres/build/ignore b/db/postgres/build/ignore new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/db/postgres/build/ignore @@ -0,0 +1 @@ + From cf696c7660e0244fc9d1eff9661ad8e752ad58df Mon Sep 17 00:00:00 2001 From: Tom Juntunen <43662466+tom-juntunen@users.noreply.github.com> Date: Sun, 15 May 2022 15:12:09 -0700 Subject: [PATCH 5/8] Rename db/postgres/create-database.sql to db/postgres/build/create-database.sql --- db/postgres/{ => build}/create-database.sql | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename db/postgres/{ => build}/create-database.sql (100%) diff --git a/db/postgres/create-database.sql b/db/postgres/build/create-database.sql similarity index 100% rename from db/postgres/create-database.sql rename to db/postgres/build/create-database.sql From 6490945ed35d9c0cbc3af45a96affe789e67ecae Mon Sep 17 00:00:00 2001 From: Tom Juntunen <43662466+tom-juntunen@users.noreply.github.com> Date: Sun, 15 May 2022 15:12:20 -0700 Subject: [PATCH 6/8] Delete ignore --- db/postgres/build/ignore | 1 - 1 file changed, 1 deletion(-) delete mode 100644 db/postgres/build/ignore diff --git a/db/postgres/build/ignore b/db/postgres/build/ignore deleted file mode 100644 index 8b13789..0000000 --- a/db/postgres/build/ignore +++ /dev/null @@ -1 +0,0 @@ - From c0c9821bb8945643f302f44e17613572f458a11a Mon Sep 17 00:00:00 2001 From: Tom Juntunen <43662466+tom-juntunen@users.noreply.github.com> Date: Sun, 15 May 2022 15:13:27 -0700 Subject: [PATCH 7/8] Create create-database.sql --- db/mssql/build/create-database.sql | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 db/mssql/build/create-database.sql diff --git a/db/mssql/build/create-database.sql b/db/mssql/build/create-database.sql new file mode 100644 index 0000000..05aa453 --- /dev/null +++ b/db/mssql/build/create-database.sql @@ -0,0 +1,6 @@ +IF NOT EXISTS (SELECT name FROM master.sys.databases WHERE name = 'TEST_DATABASE') +create database TEST_DATABASE; +GO + +use TEST_DATABASE; +GO From 5028a315dde38f8a8dd0352471b57590e955a143 Mon Sep 17 00:00:00 2001 From: Tom Juntunen <43662466+tom-juntunen@users.noreply.github.com> Date: Sun, 15 May 2022 15:13:40 -0700 Subject: [PATCH 8/8] Update run-initialization.sh --- db/mssql/run-initialization.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/mssql/run-initialization.sh b/db/mssql/run-initialization.sh index 2e770b3..746b62b 100644 --- a/db/mssql/run-initialization.sh +++ b/db/mssql/run-initialization.sh @@ -3,4 +3,4 @@ sleep 60s # Run the setup script to create the DB and the schema in the DB # Note: make sure that your password matches what is in the Dockerfile -/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P $MSSQL_SA_PASSWORD -d master -i sql/create-database.sql +/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P $MSSQL_SA_PASSWORD -d master -i build/create-database.sql