feat: chatbot to main

.env.example

@@ -1,25 +1,56 @@
-# Mail notifications
-ADMINS="Gabriel Milan,gabriel.gazola@poli.ufrj.br"
-EMAIL_HOST_USER="notifications@gmail.com"
-EMAIL_HOST_PASSWORD="password"
-# Django configurations
-DJANGO_SECRET_KEY="some-secret"
-DJANGO_SETTINGS_MODULE="backend.settings.local"
+# =============================================================================
+# ESSENTIAL SETTINGS FOR DEVELOPMENT
+# =============================================================================
+
+# Django
+DJANGO_SECRET_KEY=dev-secret-key-change-in-production
+DJANGO_SETTINGS_MODULE=backend.settings.local
+
+# Database (configured for Docker - do not change if using docker compose)
+DB_HOST=database
+DB_PORT=5432
+DB_NAME=postgres
+DB_USER=postgres
+DB_PASSWORD=postgres
+
+# Queue (configured for Docker - do not change if using docker compose)
+REDIS_HOST=queue
+REDIS_PORT=6379
+
+# Search Index (configured for Docker - do not change if using docker compose)
+ELASTICSEARCH_URL=http://index:9200
+
 # Logger
-LOGGER_LEVEL="DEBUG"
-LOGGER_IGNORE="faker,haystack"
-LOGGER_SERIALIZE=""
-# Database
-DB_HOST="localhost"
-DB_PORT="5432"
-DB_NAME="postgres"
-DB_USER="postgres"
-DB_PASSWORD="postgres"
-# Queue
-REDIS_HOST="localhost"
-REDIS_PORT="6379"
-# Index
-ELASTICSEARCH_URL=http://localhost:9200
-# Google
-GOOGLE_OAUTH_CLIENT_ID="google_key"
-GOOGLE_OAUTH_CLIENT_SECRET="google_secret"
+LOGGER_LEVEL=DEBUG
+LOGGER_IGNORE=faker,haystack
+LOGGER_SERIALIZE=
+
+# =============================================================================
+# OPTIONAL SETTINGS FOR DEVELOPMENT
+# =============================================================================
+
+# Mail notifications (only required if testing email sending)
+# ADMINS=Your Name,your.email@example.com
+# EMAIL_HOST_USER=your-notifications-email@gmail.com
+# EMAIL_HOST_PASSWORD=your-app-password
+
+# Google authentication
+# GOOGLE_OAUTH_CLIENT_ID="google_key"
+# GOOGLE_OAUTH_CLIENT_SECRET="google_secret"
+
+# Chatbot (only required if working with chatbot functionality)
+# Requires Google Cloud Platform credentials with BigQuery access
+# CHATBOT_CREDENTIALS=/app/credentials/service_account.json
+# BIGQUERY_PROJECT_ID=your-gcp-project
+# BASE_URL_BACKEND=http://localhost:8000
+# LANGSMITH_TRACING=true
+# LANGSMITH_API_KEY=your-langsmith-api-key
+# LANGSMITH_PROJECT=your-langsmith-project-name
+# For load testing only
+# MOCK_AGENT=false
+# CHATBOT_USER_EMAIL=example@email.com
+# CHATBOT_USER_PASSWORD=examplepassword
+
+# Local DB populating (only required to populate local database with production data)
+# METABASE_USER=your-metabase-user
+# METABASE_PASSWORD=your-metabase-password

.github/workflows/ci-python.yaml

@@ -11,7 +11,9 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Lint source code
-        uses: chartboost/ruff-action@v1
+        uses: astral-sh/ruff-action@v3
+        with:
+          version: "0.14.9"
   test:
     name: Test python
     runs-on: ubuntu-latest

.gitignore

@@ -249,10 +249,14 @@ fabric.properties
 .DS_Store
 /backend/media/
 /backend/notebooks/
+/backend/static/
 /backend/staticfiles/
 
 # Fixture
 fixtures*
 
 # Version manager
 .tool-versions
+
+# fetch_metabase script
+metabase_data

.pre-commit-config.yaml

@@ -19,13 +19,13 @@ repos:
     hooks:
       - id: poetry-check
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.2.0
+    rev: v0.14.9
     hooks:
-      - id: ruff
+      - id: ruff-check
         args: [--fix]
       - id: ruff-format
   - repo: https://github.com/tconbeer/sqlfmt
-    rev: v0.20.0
+    rev: v0.28.2
     hooks:
       - id: sqlfmt
         language_version: python
@@ -36,4 +36,5 @@ repos:
         name: yamlfix
         types: [yaml]
         language: system
-        entry: yamlfix --exclude "charts/**/*" .
+        entry: yamlfix
+        exclude: ^charts/

Dockerfile

@@ -1,15 +1,21 @@
 ARG PYTHON_VERSION=3.11-slim
 
-FROM python:${PYTHON_VERSION}
+FROM python:$PYTHON_VERSION
 
-# Install virtualenv and create a virtual environment
-RUN pip install --no-cache-dir -U virtualenv>=20.13.1 && virtualenv /env --python=python3.11
-ENV PATH /env/bin:$PATH
+# Set shell with pipefail for safer pipe operations
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
-# Install pip requirements
-WORKDIR /app
-COPY . .
-RUN /env/bin/pip install --no-cache-dir . && rm nginx.conf
+# Define where Poetry virtual environments will be stored
+ARG POETRY_VIRTUALENVS_PATH=/opt/pypoetry/virtualenvs
+
+# Ensure that the python output is sent straight to terminal (e.g. your container log)
+# without being first buffered and that you can see the output of your application (e.g. django logs)
+# in real time. Equivalent to python -u: https://docs.python.org/3/using/cmdline.html#cmdoption-u
+ENV PYTHONUNBUFFERED=1
+
+# Prevent Python from writing .pyc files to disc
+# https://docs.python.org/3/using/cmdline.html#envvar-PYTHONDONTWRITEBYTECODE
+ENV PYTHONDONTWRITEBYTECODE=1
 
 # Install make, nginx and copy configuration
 RUN apt-get update \
@@ -20,20 +26,26 @@ RUN apt-get update \
 RUN apt-get update && apt-get install -y postgresql postgresql-contrib
 COPY nginx.conf /etc/nginx/nginx.conf
 
-# Prevents Python from writing .pyc files to disc
-# https://docs.python.org/3/using/cmdline.html#envvar-PYTHONDONTWRITEBYTECODE
-ENV PYTHONDONTWRITEBYTECODE 1
+# Install Poetry and add it to PATH so its commands can be executed
+# from anywhere, without specifying the full path to its executable.
+RUN curl -sSL https://install.python-poetry.org | python3 - --version 2.1.3
+ENV PATH="/root/.local/bin:$PATH"
 
-# Ensures that the python output is sent straight to terminal (e.g. your container log)
-# without being first buffered and that you can see the output of your application (e.g. django logs)
-# in real time. Equivalent to python -u: https://docs.python.org/3/using/cmdline.html#cmdoption-u
-ENV PYTHONUNBUFFERED 1
+# Create the folder where Poetry virtual environments will be stored and make it
+# accessible to all users. This is needed by the 'www-data' user during server startup
+RUN mkdir -p $POETRY_VIRTUALENVS_PATH && chmod 755 $POETRY_VIRTUALENVS_PATH
+ENV POETRY_VIRTUALENVS_PATH=$POETRY_VIRTUALENVS_PATH
+
+# Copy and install project
+WORKDIR /app
+COPY . .
+RUN poetry install --only main && rm nginx.conf
 
-# Copy app, generate static and set permissions
-RUN /env/bin/python manage.py collectstatic --no-input --settings=backend.settings.base && \
+# Generate static and set permissions
+RUN poetry run python manage.py collectstatic --no-input --settings=backend.settings.base && \
     chown -R www-data:www-data /app
 
 # Expose and run app
 EXPOSE 80
 STOPSIGNAL SIGKILL
-CMD ["/app/start-server.sh"]
+CMD ["poetry", "run", "/app/start-server.sh"]

README.md

@@ -1,60 +1,82 @@
-# Base dos Dados API
-
-## Configuração de ambiente para desenvolvimento
-
-### Requisitos
-
-- Um editor de texto (recomendado VS Code)
-- Python 3.11
-- `pip`
-- (Opcional, mas recomendado) Um ambiente virtual para desenvolvimento (`miniconda`, `virtualenv` ou similares)
-
-### Procedimentos
-
-- Clonar esse repositório
-
-  ```
-  git clone https://github.com/basedosdados/backend.git
-  ```
-
-- Abrí-lo no seu editor de texto
-
-- No seu ambiente de desenvolvimento, instalar [poetry](https://python-poetry.org/) para gerenciamento de dependências
-
-    ```
-    pip3 install poetry
-    ```
-
-- Instalar as dependências para desenvolvimento
-
-    ```
-    poetry install
-    ```
-
-- Instalar os hooks de pré-commit (ver https://pre-commit.com/ para entendimento dos hooks)
-
-    ```
-    pre-commit install
-    ```
+<div align="center">
+    <a href="https://basedosdados.org">
+        <img src="https://storage.googleapis.com/basedosdados-website/logos/bd_minilogo.png" width="240" alt="Base dos Dados">
+    </a>
+</div>
+
+# Base dos Dados
+
+Backend da [Base dos Dados](https://basedosdados.org), a maior plataforma pública de dados do Brasil.
+
+## Tecnologias Utilizadas
+
+#### Core
+- [Django](https://www.djangoproject.com/) como framework web.
+- [PostgreSQL](https://www.postgresql.org/) como banco de dados relacional.
+- [huey](https://huey.readthedocs.io/en/latest/) + [Redis](https://redis.io/) para agendamento de tarefas.
+- [Elasticsearch](https://www.elastic.co/) como motor de busca.
+
+#### API de Dados
+- [GraphQL](https://graphql.org/) (via [Graphene-Django](https://docs.graphene-python.org/projects/django/en/latest/)) para a API de dados.
+- [Google Cloud Storage](https://cloud.google.com/storage) para armazenamento de mídia e acesso aos dados brutos.
+- [Google BigQuery](https://cloud.google.com/bigquery) para acesso aos dados tratados.
+
+#### Chatbot
+- [Django REST Framework](https://www.django-rest-framework.org/) para a API do chatbot.
+- [LangChain](https://docs.langchain.com/oss/python/langchain/overview) / [LangGraph](https://langchain-ai.github.io/langgraph/) para desenvolvimento de agentes de IA.
+- [Google BigQuery](https://cloud.google.com/bigquery) para acesso aos dados tratados.
+- [Vertex AI](https://cloud.google.com/vertex-ai) para acesso a LLMs.
+
+#### Pagamentos
+- [Stripe](https://stripe.com/) / [dj-stripe](https://dj-stripe.dev/) para o processamento de pagamentos.
+
+#### Ferramentas de Desenvolvimento
+- [Poetry](https://python-poetry.org/) para gerenciamento de dependências.
+- [Docker](https://www.docker.com/) para conteinerização.
+- [Pre-commit](https://pre-commit.com/) para gerenciamento de hooks de pre-commit.
+- [Ruff](https://docs.astral.sh/ruff/) para formatação.
+
+## Configuração do Ambiente de Desenvolvimento
+Para começar a desenvolver, configure o ambiente de desenvolvimento seguindo as instruções abaixo:
+
+1\. Clone o repositório e abra-o no seu editor de texto.
+```bash
+git clone https://github.com/basedosdados/backend.git
+```
 
-- Pronto! Seu ambiente está configurado para desenvolvimento.
+2\. Crie um ambiente virtual de desenvolvimento com o [Poetry](https://python-poetry.org/docs).
+```bash
+poetry install
+```
 
-* OBS1: É possível realizar a execução do servidor django um dos alias
-```sh
-    python manage.py makemigrations
-    python manage.py migrate
-    make run_docker
+3\. Instale os hooks de pre-commit.
+```
+pre-commit install
 ```
 
-* OBS2: É possível realizar a execução do servidor django via
-```sh
-    python manage.py migrate
-    python manage.py createsuperuser
-    python manage.py runserver 8080
+4\. Copie o arquivo `.env.example` e ajuste as variáveis conforme necessário.
+```bash
+cp .env.example .env
 ```
+> [!NOTE]
+> As variáveis de ambiente no arquivo `.env.example` já estão configuradas para execução com o Docker.
+>
+> Caso vá utilizar a funcionalidade do chatbot, a conta de serviço deve ser armazenada localmente em `~/.basedosdados/credentials`.
 
-* OBS3: É possível realizar a load e dump de fixtures via
-```sh
-    python manage.py dumpdata > fixture.json
-    python manage.py loadfixture fixture.json
+## Execução do Backend
+Utilize o [Docker](https://docs.docker.com/engine/install/) para executar o backend localmente:
+
+```bash
+docker compose up
 ```
+
+> [!TIP]
+> Você também pode utilizar a flag `-d --detach` e acompanhar os logs utilizando o comando `docker logs` com a flag `-f --follow`:
+> ```bash
+> docker compose up -d
+> docker compose logs api -f
+> ```
+> Para parar o serviço:
+> ```
+> docker compose down
+> ```

backend/apps/account/admin.py

@@ -10,7 +10,15 @@
 from django.utils.translation import gettext_lazy
 from faker import Faker
 
-from backend.apps.account.models import Account, BDGroup, BDRole, Team, Role, Career, Subscription
+from backend.apps.account.models import (
+    Account,
+    BDGroup,
+    BDRole,
+    Career,
+    Role,
+    Subscription,
+    Team,
+)
 from backend.apps.account.tasks import sync_subscription_task
 
 
@@ -204,6 +212,7 @@ class AccountAdmin(BaseAccountAdmin):
         "created_at",
         "is_admin",
         "is_subscriber",
+        "has_chatbot_access",
     )
     list_filter = (
         SuperUserListFilter,
@@ -264,6 +273,7 @@ class AccountAdmin(BaseAccountAdmin):
                     "is_active",
                     "is_admin",
                     "is_superuser",
+                    "has_chatbot_access",
                     "staff_groups",
                 )
             },

backend/apps/account/migrations/0019_role_team_alter_career_end_at_alter_career_level_and_more.py

@@ -1,7 +1,8 @@
+# -*- coding: utf-8 -*-
 # Generated by Django 4.2.18 on 2025-02-04 04:02
 
-from django.db import migrations, models
 import django.db.models.deletion
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):