chore: speed-up of CI tests

.github/actions/docker-cache/action.yml

@@ -0,0 +1,64 @@
+name: 'Docker Image Cache'
+description: 'Cache and load Docker images for CI jobs'
+
+inputs:
+  images:
+    description: 'Space-separated list of Docker images to cache'
+    required: true
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Generate cache key from images
+      id: cache-key
+      shell: bash
+      env:
+        IMAGES_INPUT: ${{ inputs.images }}
+      run: |
+        # Create a stable hash from the sorted image list
+        # Using env var to prevent script injection
+        IMAGES_HASH=$(echo "$IMAGES_INPUT" | tr ' ' '\n' | sort | md5sum | cut -d' ' -f1)
+        echo "key=docker-${{ runner.os }}-${IMAGES_HASH}" >> $GITHUB_OUTPUT
+
+    - name: Cache Docker images
+      uses: actions/cache@v5
+      id: docker-cache
+      with:
+        path: /tmp/docker-cache
+        key: ${{ steps.cache-key.outputs.key }}
+
+    - name: Load cached Docker images
+      if: steps.docker-cache.outputs.cache-hit == 'true'
+      shell: bash
+      run: |
+        echo "Loading cached images..."
+        for f in /tmp/docker-cache/*.tar.zst; do
+          zstd -d -c "$f" | docker load &
+        done
+        wait
+        docker images
+
+    - name: Pull and save Docker images
+      if: steps.docker-cache.outputs.cache-hit != 'true'
+      shell: bash
+      env:
+        IMAGES_INPUT: ${{ inputs.images }}
+      run: |
+        mkdir -p /tmp/docker-cache
+
+        echo "Pulling images in parallel..."
+        for img in $IMAGES_INPUT; do
+          docker pull "$img" &
+        done
+        wait
+
+        echo "Saving images with zstd compression..."
+        for img in $IMAGES_INPUT; do
+          # Create filename from image name (replace special chars)
+          filename=$(echo "$img" | tr '/:' '_')
+          docker save "$img" | zstd -T0 -3 > "/tmp/docker-cache/${filename}.tar.zst" &
+        done
+        wait
+
+        echo "Cache size:"
+        du -sh /tmp/docker-cache/

.github/workflows/backend-ci.yml

@@ -15,13 +15,64 @@ on:
       - 'docker-compose.ci.yaml'
   workflow_dispatch:
 
+# Pin image versions for cache key consistency
+env:
+  MONGO_IMAGE: mongo:8.0
+  REDIS_IMAGE: redis:7-alpine
+  KAFKA_IMAGE: apache/kafka:3.9.0
+  SCHEMA_REGISTRY_IMAGE: confluentinc/cp-schema-registry:7.5.0
+
 jobs:
+  unit:
+    name: Unit Tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          cache-dependency-glob: "backend/uv.lock"
+
+      - name: Install Python dependencies
+        run: |
+          cd backend
+          uv python install 3.12
+          uv sync --frozen
+
+      - name: Run unit tests
+        timeout-minutes: 5
+        run: |
+          cd backend
+          uv run pytest tests/unit -v -rs \
+            --cov=app \
+            --cov-report=xml --cov-report=term
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v5
+        if: always()
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: backend/coverage.xml
+          flags: backend-unit
+          name: backend-unit-coverage
+          fail_ci_if_error: false
+          verbose: true
+
   integration:
     name: Integration Tests
     runs-on: ubuntu-latest
+
     steps:
       - uses: actions/checkout@v6
 
+      - name: Cache and load Docker images
+        uses: ./.github/actions/docker-cache
+        with:
+          images: ${{ env.MONGO_IMAGE }} ${{ env.REDIS_IMAGE }} ${{ env.KAFKA_IMAGE }} ${{ env.SCHEMA_REGISTRY_IMAGE }}
+
       - name: Set up uv
         uses: astral-sh/setup-uv@v7
         with:
@@ -34,95 +85,137 @@ jobs:
           uv python install 3.12
           uv sync --frozen
 
-      - name: Setup Docker Buildx
-        uses: docker/setup-buildx-action@v3
+      - name: Start infrastructure services
+        run: |
+          docker compose -f docker-compose.ci.yaml up -d --wait --wait-timeout 120
+          docker compose -f docker-compose.ci.yaml ps
 
-      - name: Setup Kubernetes (k3s)
+      - name: Run integration tests
+        timeout-minutes: 10
+        env:
+          MONGO_ROOT_USER: root
+          MONGO_ROOT_PASSWORD: rootpassword
+          MONGODB_HOST: 127.0.0.1
+          MONGODB_PORT: 27017
+          MONGODB_URL: mongodb://root:rootpassword@127.0.0.1:27017/?authSource=admin
+          KAFKA_BOOTSTRAP_SERVERS: localhost:9092
+          SCHEMA_REGISTRY_URL: http://localhost:8081
+          REDIS_HOST: localhost
+          REDIS_PORT: 6379
+          SCHEMA_SUBJECT_PREFIX: "ci.${{ github.run_id }}."
         run: |
-          curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--disable=traefik --tls-san host.docker.internal" sh -
-          mkdir -p /home/runner/.kube
-          sudo k3s kubectl config view --raw > /home/runner/.kube/config
-          sudo chmod 600 /home/runner/.kube/config
-          export KUBECONFIG=/home/runner/.kube/config
-          timeout 90 bash -c 'until sudo k3s kubectl cluster-info; do sleep 5; done'
+          cd backend
+          uv run pytest tests/integration -v -rs \
+            --ignore=tests/integration/k8s \
+            --cov=app \
+            --cov-report=xml --cov-report=term
 
-      - name: Create kubeconfig for CI Docker containers
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v5
+        if: always()
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: backend/coverage.xml
+          flags: backend-integration
+          name: backend-integration-coverage
+          fail_ci_if_error: false
+          verbose: true
+
+      - name: Collect logs
+        if: failure()
         run: |
-          # Copy real k3s kubeconfig with valid credentials, but change server address
-          # from 127.0.0.1 to host.docker.internal for Docker container networking
-          # (k3s was started with --tls-san host.docker.internal so the cert is valid)
-          sed 's|https://127.0.0.1:6443|https://host.docker.internal:6443|g' \
-            /home/runner/.kube/config > backend/kubeconfig.yaml
-          chmod 644 backend/kubeconfig.yaml
-
-      - name: Setup CI Compose
-        uses: ./.github/actions/setup-ci-compose
+          mkdir -p logs
+          docker compose -f docker-compose.ci.yaml logs > logs/docker-compose.log 2>&1
+          docker compose -f docker-compose.ci.yaml logs kafka > logs/kafka.log 2>&1
+          docker compose -f docker-compose.ci.yaml logs schema-registry > logs/schema-registry.log 2>&1
+
+      - name: Upload logs
+        if: failure()
+        uses: actions/upload-artifact@v6
         with:
-          kubeconfig-path: /home/runner/.kube/config
+          name: backend-logs
+          path: logs/
+
+  e2e:
+    name: E2E Tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v6
 
-      - name: Build services
-        uses: docker/bake-action@v6
+      - name: Cache and load Docker images
+        uses: ./.github/actions/docker-cache
         with:
-          source: .
-          files: docker-compose.ci.yaml
-          load: true
-          set: |
-            *.cache-from=type=gha,scope=buildkit-${{ github.repository }}-${{ github.ref_name }}
-            *.cache-from=type=gha,scope=buildkit-${{ github.repository }}-main
-            *.cache-to=type=gha,mode=max,scope=buildkit-${{ github.repository }}-${{ github.ref_name }}
-            *.pull=true
-        env:
-          BUILDKIT_PROGRESS: plain
+          images: ${{ env.MONGO_IMAGE }} ${{ env.REDIS_IMAGE }} ${{ env.KAFKA_IMAGE }} ${{ env.SCHEMA_REGISTRY_IMAGE }}
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          cache-dependency-glob: "backend/uv.lock"
 
-      - name: Start services
+      - name: Install Python dependencies
         run: |
-          docker compose -f docker-compose.ci.yaml up -d --remove-orphans
-          docker compose -f docker-compose.ci.yaml ps
+          cd backend
+          uv python install 3.12
+          uv sync --frozen
 
-      - name: Wait for backend
+      - name: Start infrastructure services
         run: |
-          curl --retry 60 --retry-delay 5 --retry-all-errors -ksf https://127.0.0.1:443/api/v1/health/live
+          docker compose -f docker-compose.ci.yaml up -d --wait --wait-timeout 120
           docker compose -f docker-compose.ci.yaml ps
-          kubectl get pods -A -o wide
 
-      - name: Run integration tests
+      - name: Setup Kubernetes (k3s)
+        run: |
+          curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--disable=traefik" sh -
+          mkdir -p /home/runner/.kube
+          sudo k3s kubectl config view --raw > /home/runner/.kube/config
+          sudo chmod 600 /home/runner/.kube/config
+          export KUBECONFIG=/home/runner/.kube/config
+          timeout 90 bash -c 'until sudo k3s kubectl cluster-info; do sleep 5; done'
+          kubectl create namespace integr8scode --dry-run=client -o yaml | kubectl apply -f -
+
+      - name: Run E2E tests
         timeout-minutes: 10
         env:
-          BACKEND_BASE_URL: https://127.0.0.1:443
           MONGO_ROOT_USER: root
           MONGO_ROOT_PASSWORD: rootpassword
-          MONGODB_HOST: 127.0.0.1
-          MONGODB_PORT: 27017
           MONGODB_URL: mongodb://root:rootpassword@127.0.0.1:27017/?authSource=admin
+          KAFKA_BOOTSTRAP_SERVERS: localhost:9092
+          SCHEMA_REGISTRY_URL: http://localhost:8081
+          REDIS_HOST: localhost
+          REDIS_PORT: 6379
           SCHEMA_SUBJECT_PREFIX: "ci.${{ github.run_id }}."
+          KUBECONFIG: /home/runner/.kube/config
+          K8S_NAMESPACE: integr8scode
         run: |
           cd backend
-          uv run pytest tests/integration -v -rs --cov=app --cov-branch --cov-report=xml --cov-report=term
+          uv run pytest tests/integration/k8s -v -rs \
+            --cov=app \
+            --cov-report=xml --cov-report=term
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v5
         if: always()
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: backend/coverage.xml
-          flags: backend
-          name: backend-coverage
+          flags: backend-e2e
+          name: backend-e2e-coverage
           fail_ci_if_error: false
           verbose: true
 
       - name: Collect logs
         if: failure()
         run: |
           mkdir -p logs
-          docker compose -f docker-compose.ci.yaml logs > logs/docker-compose.log
-          docker compose -f docker-compose.ci.yaml logs backend > logs/backend.log
-          docker compose -f docker-compose.ci.yaml logs mongo > logs/mongo.log
-          kubectl get events --sort-by='.metadata.creationTimestamp' > logs/k8s-events.log 2>&1 || true
+          docker compose -f docker-compose.ci.yaml logs > logs/docker-compose.log 2>&1
+          kubectl get events --sort-by='.metadata.creationTimestamp' -A > logs/k8s-events.log 2>&1 || true
           kubectl describe pods -A > logs/k8s-describe-pods.log 2>&1 || true
 
       - name: Upload logs
         if: failure()
         uses: actions/upload-artifact@v6
         with:
-          name: backend-logs
+          name: k8s-logs
           path: logs/