diff --git a/.env b/.env new file mode 100644 index 0000000..13923b2 --- /dev/null +++ b/.env @@ -0,0 +1,9 @@ +POSTGRES_DB=postgres +POSTGRES_USER=postgres +POSTGRES_PASSWORD=p0stgr3s +PSQL_CONNECTION=postgresql://postgres:p0stgr3s@psql:5432 +HOST_PSQL_CONNECTION=postgresql://postgres:p0stgr3s@localhost:5440 + +MSSQL_SA_PASSWORD=mssql4docker# +MSSQL_SQL_CONNECTION=mssql+pyodbc://sa:mssql4docker#@mssql:1433?driver=ODBC+Driver+17+for+SQL+Server +HOST_MSSQL_CONNECTION=mssql+pyodbc://sa:mssql4docker#@localhost:1440?driver=ODBC+Driver+17+for+SQL+Server diff --git a/.gitignore b/.gitignore index b6e4761..e240f98 100644 --- a/.gitignore +++ b/.gitignore @@ -102,7 +102,7 @@ celerybeat.pid *.sage.py # Environments -.env +# .env using this to pass env variables to docker compose .venv env/ venv/ diff --git a/README.md b/README.md index 8a1cd08..a3d85fa 100644 --- a/README.md +++ b/README.md @@ -21,3 +21,32 @@ Add your unit tests to files inside the `tests` folder ... name your files `test ## Data Flow High level description of data source(s) and sink(s), as well as the general pattern and data flow through the pipeline. Discuss any assumptions made. + +## Environment +Compose supports declaring default environment variables in an environment file named .env placed in the project directory. + +The following syntax rules apply to the .env file: + +- Compose expects each line in an env file to be in VAR=VAL format. +- Lines beginning with # are processed as comments and ignored. +- Blank lines are ignored. +- There is no special handling of quotation marks. This means that they are part of the VAL. + +## Database +For persisting data into SQL, a sql container using an external volume allows +for the database container to restart without losing data. + +The database is added to a bridge network where other hosts on the network can +reach the database. + +`docker network create test-network` + +Start a PostgreSQL database + +`docker volume create postgres-volume` +`docker-compose -f psql.yml up --build -d` + +Start a SQL Server database + +`docker volume create mssql-volume` +`docker-compose -f mssql.yml up --build -d` \ No newline at end of file diff --git a/db/mssql/Dockerfile b/db/mssql/Dockerfile new file mode 100644 index 0000000..122adbf --- /dev/null +++ b/db/mssql/Dockerfile @@ -0,0 +1,36 @@ +FROM mcr.microsoft.com/mssql/server:2019-latest + +USER root + +# Install dependencies for installing mssql-tools +RUN apt-get update \ + && apt-get install -y gnupg \ + && apt-get -y install curl + +# adding custom MS repository +RUN curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - +RUN curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | tee /etc/apt/sources.list.d/msprod.list + +# Install SQL Server Tools +RUN apt-get update +RUN apt-get -y install mssql-tools +RUN echo 'export PATH="$PATH:/opt/mssql-tools/bin"' >> ~/.bashrc +RUN #!/bin/bash source ~/.bashrc + +# copy over initialization scripts that create our sql databases +RUN mkdir -p /usr/src/app +WORKDIR /usr/src/app +COPY ./entrypoint.sh /usr/src/app +COPY ./run-initialization.sh /usr/src/app + +# set permissions allowing the shell script to be executed +RUN chmod +x /usr/src/app/entrypoint.sh +RUN chmod +x /usr/src/app/run-initialization.sh + +# expose the default mssql port from the container +EXPOSE 1433 + +USER mssql + +# override Dockerfile CMD entry from base image (which runs SQL server), with our own CMD command, which runs our custom script: +CMD ["/bin/bash", "./entrypoint.sh"] \ No newline at end of file diff --git a/db/mssql/build/create-database.sql b/db/mssql/build/create-database.sql new file mode 100644 index 0000000..05aa453 --- /dev/null +++ b/db/mssql/build/create-database.sql @@ -0,0 +1,6 @@ +IF NOT EXISTS (SELECT name FROM master.sys.databases WHERE name = 'TEST_DATABASE') +create database TEST_DATABASE; +GO + +use TEST_DATABASE; +GO diff --git a/db/mssql/entrypoint.sh b/db/mssql/entrypoint.sh new file mode 100644 index 0000000..7d18e85 --- /dev/null +++ b/db/mssql/entrypoint.sh @@ -0,0 +1,2 @@ +# Run initialization script and start ms sql server +/usr/src/app/run-initialization.sh & /opt/mssql/bin/sqlservr \ No newline at end of file diff --git a/db/mssql/run-initialization.sh b/db/mssql/run-initialization.sh new file mode 100644 index 0000000..746b62b --- /dev/null +++ b/db/mssql/run-initialization.sh @@ -0,0 +1,6 @@ +# Wait to be sure that SQL Server came up +sleep 60s + +# Run the setup script to create the DB and the schema in the DB +# Note: make sure that your password matches what is in the Dockerfile +/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P $MSSQL_SA_PASSWORD -d master -i build/create-database.sql diff --git a/db/postgres/build/create-database.sql b/db/postgres/build/create-database.sql new file mode 100644 index 0000000..4321630 --- /dev/null +++ b/db/postgres/build/create-database.sql @@ -0,0 +1,6 @@ + CREATE DATABASE test_database; + + CREATE ROLE testadmin WITH PASSWORD 'm4k3rb0t#' LOGIN; + + GRANT ALL PRIVILEGES ON DATABASE test_database TO testadmin; + diff --git a/db/postgres/my-postgres.conf b/db/postgres/my-postgres.conf new file mode 100644 index 0000000..0c86555 --- /dev/null +++ b/db/postgres/my-postgres.conf @@ -0,0 +1,30 @@ +listen_addresses = '*' + +# DB Version: 14.0 +# OS Type: linux +# DB Type: test + +# Performance tweaks (sample values provided) +max_connections = 1000 +shared_buffers = 15872MB +effective_cache_size = 47616MB +maintenance_work_mem = 2GB +checkpoint_completion_target = 0.7 +wal_buffers = 16MB +default_statistics_target = 100 +random_page_cost = 1.1 +effective_io_concurrency = 200 +work_mem = 4063kB +min_wal_size = 1GB +max_wal_size = 4GB +max_worker_processes = 12 +max_parallel_workers_per_gather = 4 + +wal_level = logical # options are: replica, minimal, logical +hot_standby = on +max_wal_senders = 10 +max_replication_slots = 10 +hot_standby_feedback = on + +# For more information on WAL: +# https://www.postgresql.org/docs/current/runtime-config-wal.html \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 0619289..4264fea 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -7,3 +7,9 @@ services: volumes: - .:/app command: python3 -m pytest + networks: + - test-network + +networks: + test-network: + external: True diff --git a/mssql.yml b/mssql.yml new file mode 100644 index 0000000..dfd39a6 --- /dev/null +++ b/mssql.yml @@ -0,0 +1,23 @@ +services: + mssql: + build: ./db/mssql + hostname: mssql + environment: + - MSSQL_SA_PASSWORD=${MSSQL_SA_PASSWORD} + - MSSQL_SQL_CONNECTION=${MSSQL_SQL_CONNECTION} + - ACCEPT_EULA="Y" + ports: + - "1440:1433" + volumes: + - mssql-volume:/var/opt/mssql + restart: always + networks: + - test-network + +volumes: + mssql-volume: + external: True + +networks: + test-network: + external: True \ No newline at end of file diff --git a/psql.yml b/psql.yml new file mode 100644 index 0000000..6afb752 --- /dev/null +++ b/psql.yml @@ -0,0 +1,26 @@ +services: + psql: + image: postgres:14.1-alpine + hostname: postgres + environment: + - POSTGRES_DB=${POSTGRES_DB} + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - SQL_CONNECTION=${SQL_CONNECTION} + ports: + - "5440:5432" + volumes: + - postgres-volume:/var/lib/postgresql/data + - ./my-postgres.conf:/etc/postgresql/postgres.conf + - ./db/postgres/build/create_database.sql:/docker-entrypoint-initdb.d/create_database.sql + restart: always + networks: + - test-network + +volumes: + postgres-volume: + external: True + +networks: + test-network: + external: True