adding psql and mssql services, updated readme

.env

@@ -0,0 +1,9 @@
+POSTGRES_DB=postgres
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=p0stgr3s
+PSQL_CONNECTION=postgresql://postgres:p0stgr3s@psql:5432
+HOST_PSQL_CONNECTION=postgresql://postgres:p0stgr3s@localhost:5440
+
+MSSQL_SA_PASSWORD=mssql4docker#
+MSSQL_SQL_CONNECTION=mssql+pyodbc://sa:mssql4docker#@mssql:1433?driver=ODBC+Driver+17+for+SQL+Server
+HOST_MSSQL_CONNECTION=mssql+pyodbc://sa:mssql4docker#@localhost:1440?driver=ODBC+Driver+17+for+SQL+Server

.gitignore

@@ -102,7 +102,7 @@ celerybeat.pid
 *.sage.py
 
 # Environments
-.env
+# .env  using this to pass env variables to docker compose
 .venv
 env/
 venv/

README.md

@@ -21,3 +21,32 @@ Add your unit tests to files inside the `tests` folder ... name your files `test
 ## Data Flow
 High level description of data source(s) and sink(s), as well as the general pattern and data flow through the pipeline.
 Discuss any assumptions made.
+
+## Environment
+Compose supports declaring default environment variables in an environment file named .env placed in the project directory.
+
+The following syntax rules apply to the .env file:
+
+- Compose expects each line in an env file to be in VAR=VAL format.
+- Lines beginning with # are processed as comments and ignored.
+- Blank lines are ignored.
+- There is no special handling of quotation marks. This means that they are part of the VAL.
+
+## Database
+For persisting data into SQL, a sql container using an external volume allows
+for the database container to restart without losing data. 
+
+The database is added to a bridge network where other hosts on the network can 
+reach the database.
+
+`docker network create test-network`
+
+Start a PostgreSQL database
+
+`docker volume create postgres-volume`
+`docker-compose -f psql.yml up --build -d`
+
+Start a SQL Server database
+
+`docker volume create mssql-volume`
+`docker-compose -f mssql.yml up --build -d`

db/mssql/Dockerfile

@@ -0,0 +1,36 @@
+FROM mcr.microsoft.com/mssql/server:2019-latest
+
+USER root
+
+# Install dependencies for installing mssql-tools
+RUN apt-get update \
+    && apt-get install -y gnupg \
+    && apt-get -y install curl
+
+# adding custom MS repository
+RUN curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add -
+RUN curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | tee /etc/apt/sources.list.d/msprod.list
+
+# Install SQL Server Tools
+RUN apt-get update 
+RUN apt-get -y install mssql-tools 
+RUN echo 'export PATH="$PATH:/opt/mssql-tools/bin"' >> ~/.bashrc
+RUN #!/bin/bash source ~/.bashrc
+
+# copy over initialization scripts that create our sql databases
+RUN mkdir -p /usr/src/app
+WORKDIR /usr/src/app
+COPY ./entrypoint.sh /usr/src/app
+COPY ./run-initialization.sh /usr/src/app
+
+# set permissions allowing the shell script to be executed
+RUN chmod +x /usr/src/app/entrypoint.sh
+RUN chmod +x /usr/src/app/run-initialization.sh
+
+# expose the default mssql port from the container
+EXPOSE 1433
+
+USER mssql
+
+# override Dockerfile CMD entry from base image (which runs SQL server), with our own CMD command, which runs our custom script:
+CMD ["/bin/bash", "./entrypoint.sh"]

db/mssql/build/create-database.sql

@@ -0,0 +1,6 @@
+IF NOT EXISTS (SELECT name FROM master.sys.databases WHERE name = 'TEST_DATABASE')
+create database TEST_DATABASE;
+GO
+
+use TEST_DATABASE;
+GO

db/mssql/entrypoint.sh

@@ -0,0 +1,2 @@
+# Run initialization script and start ms sql server
+/usr/src/app/run-initialization.sh & /opt/mssql/bin/sqlservr

db/mssql/run-initialization.sh

@@ -0,0 +1,6 @@
+# Wait to be sure that SQL Server came up
+sleep 60s
+
+# Run the setup script to create the DB and the schema in the DB
+# Note: make sure that your password matches what is in the Dockerfile
+/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P $MSSQL_SA_PASSWORD -d master -i build/create-database.sql

db/postgres/build/create-database.sql

@@ -0,0 +1,6 @@
+	CREATE DATABASE test_database;
+
+	CREATE ROLE testadmin WITH PASSWORD 'm4k3rb0t#' LOGIN;
+
+	GRANT ALL PRIVILEGES ON DATABASE test_database TO testadmin;
+

db/postgres/my-postgres.conf

@@ -0,0 +1,30 @@
+listen_addresses = '*'
+
+# DB Version: 14.0
+# OS Type: linux
+# DB Type: test
+
+# Performance tweaks (sample values provided)
+max_connections = 1000
+shared_buffers = 15872MB
+effective_cache_size = 47616MB
+maintenance_work_mem = 2GB
+checkpoint_completion_target = 0.7
+wal_buffers = 16MB
+default_statistics_target = 100
+random_page_cost = 1.1
+effective_io_concurrency = 200
+work_mem = 4063kB
+min_wal_size = 1GB
+max_wal_size = 4GB
+max_worker_processes = 12
+max_parallel_workers_per_gather = 4
+
+wal_level = logical  # options are: replica, minimal, logical
+hot_standby = on
+max_wal_senders = 10
+max_replication_slots = 10
+hot_standby_feedback = on
+
+# For more information on WAL: 
+# https://www.postgresql.org/docs/current/runtime-config-wal.html

docker-compose.yaml

@@ -7,3 +7,9 @@ services:
     volumes:
       - .:/app
     command: python3 -m pytest
+    networks: 
+      - test-network
+
+networks:
+  test-network:
+    external: True

mssql.yml

@@ -0,0 +1,23 @@
+services:
+  mssql:
+    build: ./db/mssql 
+    hostname: mssql
+    environment:
+      - MSSQL_SA_PASSWORD=${MSSQL_SA_PASSWORD}
+      - MSSQL_SQL_CONNECTION=${MSSQL_SQL_CONNECTION}
+      - ACCEPT_EULA="Y"
+    ports:
+      - "1440:1433"
+    volumes:
+      - mssql-volume:/var/opt/mssql
+    restart: always
+    networks:
+      - test-network
+
+volumes:
+  mssql-volume:
+    external: True
+
+networks:
+  test-network:
+    external: True

psql.yml

@@ -0,0 +1,26 @@
+services:
+  psql:
+    image: postgres:14.1-alpine
+    hostname: postgres
+    environment:
+      - POSTGRES_DB=${POSTGRES_DB}
+      - POSTGRES_USER=${POSTGRES_USER}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - SQL_CONNECTION=${SQL_CONNECTION}
+    ports:
+      - "5440:5432"
+    volumes:
+      - postgres-volume:/var/lib/postgresql/data
+      - ./my-postgres.conf:/etc/postgresql/postgres.conf
+      - ./db/postgres/build/create_database.sql:/docker-entrypoint-initdb.d/create_database.sql
+    restart: always
+    networks:
+      - test-network
+
+volumes:
+  postgres-volume:
+    external: True
+
+networks:
+  test-network:
+    external: True