From f65cbad32309c951e0a83bd516af7475836ffc68 Mon Sep 17 00:00:00 2001
From: Charlie Le <charlie_le@apple.com>
Date: Thu, 19 Jun 2025 13:04:49 -0700
Subject: [PATCH 01/12] Test modernized CI workflows

Signed-off-by: Charlie Le <charlie_le@apple.com>
---
 .github/workflows/build-image.yml |  90 ++++--
 .github/workflows/ci.yml          | 441 ++++++++++++++++++++++++++++++
 .github/workflows/performance.yml | 238 ++++++++++++++++
 .github/workflows/release.yml     | 328 ++++++++++++++++++++++
 .github/workflows/security.yml    | 163 +++++++++++
 5 files changed, 1243 insertions(+), 17 deletions(-)
 create mode 100644 .github/workflows/ci.yml
 create mode 100644 .github/workflows/performance.yml
 create mode 100644 .github/workflows/release.yml
 create mode 100644 .github/workflows/security.yml

diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml
index bea9ac2d682..70a5f04c830 100644
--- a/.github/workflows/build-image.yml
+++ b/.github/workflows/build-image.yml
@@ -2,45 +2,101 @@ name: Build Image
 
 on:
   push:
-    branches: [ master ]
+    branches: [master, main]
     paths:
       - 'build-image/**'
       - '.github/workflows/build-image.yml'
   pull_request:
-    branches: [ master ]
+    branches: [master, main]
     paths:
       - 'build-image/**'
       - '.github/workflows/build-image.yml'
+  workflow_dispatch:
 
-permissions: 
+permissions:
   contents: read
+  packages: write
+
+env:
+  REGISTRY: quay.io
+  IMAGE_NAME: cortexproject/build-image
 
 jobs:
   build:
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-latest
+    outputs:
+      image-digest: ${{ steps.build.outputs.digest }}
+      image-tag: ${{ steps.meta.outputs.tags }}
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        name: Checkout
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@18ce135bb5112fa8ce4ed6c17ab05699d7f3a5e0 # v3.11.0
+        uses: docker/setup-buildx-action@v3
 
-      - name: Save image
-        run: make save-multiarch-build-image
+      - name: Login to Container Registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ secrets.QUAY_REGISTRY_USER }}
+          password: ${{ secrets.QUAY_REGISTRY_PASSWORD }}
 
-      - name: Upload Docker Images Artifacts
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+      - name: Extract metadata
+        id: meta
+        uses: docker/metadata-action@v5
         with:
-          name: build-image
-          path: |
-            ./build-image-amd64.tar
-            ./build-image-arm64.tar
-          if-no-files-found: error
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=sha,prefix={{branch}}-
+            type=raw,value=latest,enable={{is_default_branch}}
+
+      - name: Build and push
+        id: build
+        uses: docker/build-push-action@v6
+        with:
+          context: ./build-image
+          platforms: linux/amd64,linux/arm64
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          # Security features
+          provenance: true
+          sbom: true
+
+      - name: Generate attestation
+        if: github.event_name != 'pull_request'
+        uses: actions/attest-build-provenance@v1
+        with:
+          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          subject-digest: ${{ steps.build.outputs.digest }}
+          push-to-registry: true
+
+  # Test the build image
+  test:
+    runs-on: ubuntu-latest
+    needs: build
+    if: github.event_name == 'pull_request'
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Test build image functionality
+        run: |
+          # Test that the build image contains expected tools
+          docker run --rm ${{ needs.build.outputs.image-tag }} which go
+          docker run --rm ${{ needs.build.outputs.image-tag }} which make
+          docker run --rm ${{ needs.build.outputs.image-tag }} which git
+          
+          # Test Go version
+          docker run --rm ${{ needs.build.outputs.image-tag }} go version
 
   push:
     needs: build
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 00000000000..d8d19a66a6c
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,441 @@
+name: CI
+
+on:
+  push:
+    branches: [master, main]
+    tags:
+      - v[0-9]+.[0-9]+.[0-9]+**
+    paths-ignore:
+      - 'build-image/**'
+      - '.github/workflows/build-image.yml'
+      - 'docs/**'
+      - '*.md'
+  pull_request:
+    paths-ignore:
+      - 'build-image/**'
+      - '.github/workflows/build-image.yml'
+      - 'docs/**'
+      - '*.md'
+
+env:
+  GO_VERSION: '1.24.0'
+  REGISTRY: quay.io
+  IMAGE_PREFIX: quay.io/cortexproject/
+
+permissions:
+  contents: read
+  security-events: write
+  actions: read
+
+jobs:
+  # Detect changes to optimize what we build/test
+  changes:
+    runs-on: ubuntu-latest
+    outputs:
+      go: ${{ steps.changes.outputs.go }}
+      docker: ${{ steps.changes.outputs.docker }}
+      integration: ${{ steps.changes.outputs.integration }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
+        id: changes
+        with:
+          filters: |
+            go:
+              - '**/*.go'
+              - 'go.mod'
+              - 'go.sum'
+              - 'Makefile'
+              - '.golangci.yml'
+            docker:
+              - '**/Dockerfile'
+              - 'build-image/**'
+            integration:
+              - 'integration/**'
+              - 'cmd/**'
+
+  # Modern Go setup with caching
+  setup-go:
+    runs-on: ubuntu-latest
+    needs: changes
+    if: needs.changes.outputs.go == 'true'
+    outputs:
+      cache-key: ${{ steps.cache-key.outputs.key }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: true
+      
+      - name: Generate cache key
+        id: cache-key
+        run: |
+          echo "key=go-mod-${{ hashFiles('go.sum') }}" >> $GITHUB_OUTPUT
+      
+      - name: Download dependencies
+        run: go mod download
+
+  # Lint and static analysis
+  lint:
+    runs-on: ubuntu-latest
+    needs: [changes, setup-go]
+    if: needs.changes.outputs.go == 'true'
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # Full history for better analysis
+      
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: true
+      
+      # Modern linting with latest golangci-lint
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v6
+        with:
+          version: latest
+          args: --timeout=10m
+      
+      # Additional security and quality checks
+      - name: Run gosec security scanner
+        uses: securecodewarrior/github-action-gosec@master
+        with:
+          args: '-no-fail -fmt sarif -out gosec.sarif ./...'
+      
+      - name: Upload SARIF file
+        if: always()
+        uses: github/codeql-action/upload-sarif@v3
+        with:
+          sarif_file: gosec.sarif
+      
+      - name: Check dependencies
+        run: |
+          go mod verify
+          go mod tidy
+          git diff --exit-code -- go.sum go.mod
+      
+      - name: Check generated files
+        run: |
+          make check-protos
+          make check-doc
+          make check-white-noise
+
+  # Unit tests with coverage
+  test:
+    runs-on: ubuntu-latest
+    needs: [changes, setup-go]
+    if: needs.changes.outputs.go == 'true'
+    strategy:
+      matrix:
+        test-type: [unit, unit-no-race]
+    steps:
+      - uses: actions/checkout@v4
+      
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: true
+      
+      - name: Run tests
+        run: |
+          if [[ "${{ matrix.test-type }}" == "unit" ]]; then
+            go test -race -coverprofile=coverage.out -covermode=atomic -timeout=30m ./...
+          else
+            go test -coverprofile=coverage-no-race.out -covermode=atomic -timeout=30m ./...
+          fi
+      
+      - name: Upload coverage to Codecov
+        if: matrix.test-type == 'unit'
+        uses: codecov/codecov-action@v4
+        with:
+          file: coverage.out
+          fail_ci_if_error: false
+
+  # Build Docker images efficiently
+  build:
+    runs-on: ubuntu-latest
+    needs: [changes, lint, test]
+    if: always() && (needs.lint.result == 'success' || needs.lint.result == 'skipped') && (needs.test.result == 'success' || needs.test.result == 'skipped')
+    outputs:
+      image-tag: ${{ steps.meta.outputs.tags }}
+      image-digest: ${{ steps.build.outputs.digest }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: true
+      
+      # Modern Docker setup
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      
+      # Login for pushing (only on main/tags)
+      - name: Login to Container Registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ secrets.QUAY_REGISTRY_USER }}
+          password: ${{ secrets.QUAY_REGISTRY_PASSWORD }}
+      
+      # Generate metadata
+      - name: Extract metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.IMAGE_PREFIX }}cortex
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=sha
+      
+      # Build Go binaries
+      - name: Build binaries
+        run: |
+          make dist
+      
+      # Build and push multi-arch images
+      - name: Build and push
+        id: build
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          # Modern build features
+          provenance: true
+          sbom: true
+      
+      # Upload binaries as artifacts
+      - name: Upload binaries
+        uses: actions/upload-artifact@v4
+        with:
+          name: binaries
+          path: dist/
+          retention-days: 7
+
+  # Integration tests with better parallelization
+  integration:
+    runs-on: ubuntu-latest
+    needs: [changes, build]
+    if: needs.changes.outputs.integration == 'true' || needs.changes.outputs.go == 'true'
+    strategy:
+      fail-fast: false
+      matrix:
+        suite:
+          - name: docker
+            tags: requires_docker
+          - name: alertmanager
+            tags: integration_alertmanager
+          - name: backward-compatibility
+            tags: integration_backward_compatibility
+          - name: memberlist
+            tags: integration_memberlist
+          - name: querier
+            tags: integration_querier
+          - name: ruler
+            tags: integration_ruler
+          - name: query-fuzz
+            tags: integration_query_fuzz
+    steps:
+      - uses: actions/checkout@v4
+      
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: true
+      
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      
+      # Pre-pull common images to avoid timeouts
+      - name: Pre-pull Docker images
+        run: |
+          docker pull minio/minio:RELEASE.2024-05-28T17-19-04Z &
+          docker pull consul:1.8.4 &
+          docker pull gcr.io/etcd-development/etcd:v3.4.7 &
+          docker pull memcached:1.6.1 &
+          docker pull redis:7.0.4-alpine &
+          
+          # Suite-specific images
+          if [[ "${{ matrix.suite.name }}" == "backward-compatibility" ]]; then
+            for version in v1.13.1 v1.13.2 v1.14.0 v1.14.1 v1.15.0 v1.15.1 v1.15.2 v1.15.3 v1.16.0 v1.16.1 v1.17.0 v1.17.1 v1.18.0 v1.18.1; do
+              docker pull quay.io/cortexproject/cortex:$version &
+            done
+          elif [[ "${{ matrix.suite.name }}" == "query-fuzz" ]]; then
+            docker pull quay.io/cortexproject/cortex:v1.18.1 &
+            docker pull quay.io/prometheus/prometheus:v2.51.0 &
+            docker pull quay.io/prometheus/prometheus:v2.55.1 &
+          fi
+          
+          wait # Wait for all background pulls
+      
+      - name: Run integration tests
+        env:
+          CORTEX_IMAGE_PREFIX: ${{ env.IMAGE_PREFIX }}
+          IMAGE_TAG: ${{ needs.build.outputs.image-tag }}
+        run: |
+          export CORTEX_IMAGE="${CORTEX_IMAGE_PREFIX}cortex:${IMAGE_TAG}"
+          echo "Running integration tests with image: $CORTEX_IMAGE"
+          go test -tags=integration,${{ matrix.suite.tags }} -timeout 40m -v -count=1 ./integration/...
+
+  # Database integration tests
+  integration-configs-db:
+    runs-on: ubuntu-latest
+    needs: [changes, build]
+    if: needs.changes.outputs.integration == 'true' || needs.changes.outputs.go == 'true'
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_DB: configs_test
+          POSTGRES_PASSWORD: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    steps:
+      - uses: actions/checkout@v4
+      
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: true
+      
+      - name: Run database integration tests
+        env:
+          DB_ADDR: localhost:5432
+          DB_USER: postgres
+          DB_PASSWORD: postgres
+          MIGRATIONS_DIR: ${{ github.workspace }}/cmd/cortex/migrations
+        run: |
+          go test -v -tags 'netgo integration' -timeout 10m ./pkg/configs/... ./pkg/ruler/...
+
+  # Security scanning
+  security:
+    runs-on: ubuntu-latest
+    needs: changes
+    if: needs.changes.outputs.go == 'true'
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      
+      # CodeQL analysis
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: go
+          queries: security-extended,security-and-quality
+      
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v3
+      
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
+        with:
+          category: "/language:go"
+
+  # Dependency scanning
+  dependency-review:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/dependency-review-action@v4
+        with:
+          fail-on-severity: moderate
+
+  # Website deployment (only on main branch)
+  deploy-website:
+    runs-on: ubuntu-latest
+    needs: [build, test]
+    if: github.ref == 'refs/heads/master' && github.repository == 'cortexproject/cortex'
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ssh-key: ${{ secrets.WEBSITE_DEPLOY_SSH_PRIVATE_KEY }}
+          submodules: recursive
+      
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: true
+      
+      - name: Setup Hugo
+        uses: peaceiris/actions-hugo@v3
+        with:
+          hugo-version: 'latest'
+          extended: true
+      
+      - name: Build website
+        run: |
+          cd website
+          HUGO_ENV=production hugo --config config.toml --minify -v
+      
+      - name: Deploy to GitHub Pages
+        uses: peaceiris/actions-gh-pages@v4
+        with:
+          deploy_key: ${{ secrets.WEBSITE_DEPLOY_SSH_PRIVATE_KEY }}
+          publish_dir: ./website/public
+          external_repository: cortexproject/cortexproject.github.io
+          publish_branch: master
+
+  # Container image deployment
+  deploy:
+    runs-on: ubuntu-latest
+    needs: [build, test, lint, integration, integration-configs-db]
+    if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex'
+    steps:
+      - uses: actions/checkout@v4
+      
+      - name: Login to additional registries
+        if: env.DOCKER_REGISTRY_PASSWORD != ''
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_REGISTRY_USER }}
+          password: ${{ secrets.DOCKER_REGISTRY_PASSWORD }}
+        env:
+          DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }}
+      
+      - name: Multi-registry push
+        if: needs.build.outputs.image-digest != ''
+        run: |
+          # Images are already built and pushed to Quay in build job
+          # Additional registry pushes can be added here if needed
+          echo "Image successfully deployed: ${{ needs.build.outputs.image-tag }}"
+          echo "Digest: ${{ needs.build.outputs.image-digest }}"
+
+  # Generate SLSA provenance
+  provenance:
+    needs: [build, deploy]
+    if: startsWith(github.ref, 'refs/tags/')
+    permissions:
+      actions: read
+      id-token: write
+      contents: write
+    uses: slsa-framework/slsa-github-generator/.github/workflows/generator_container_slsa3.yml@v2.0.0
+    with:
+      image: ${{ needs.build.outputs.image-tag }}
+      digest: ${{ needs.build.outputs.image-digest }}
+    secrets:
+      registry-username: ${{ secrets.QUAY_REGISTRY_USER }}
+      registry-password: ${{ secrets.QUAY_REGISTRY_PASSWORD }}
\ No newline at end of file
diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
new file mode 100644
index 00000000000..180712607aa
--- /dev/null
+++ b/.github/workflows/performance.yml
@@ -0,0 +1,238 @@
+name: Performance & Benchmarks
+
+on:
+  push:
+    branches: [master]
+    paths:
+      - '**/*.go'
+      - 'go.mod'
+      - 'go.sum'
+  pull_request:
+    paths:
+      - '**/*.go'
+      - 'go.mod'
+      - 'go.sum'
+  schedule:
+    # Run weekly performance tests
+    - cron: '0 3 * * 1'
+  workflow_dispatch:
+    inputs:
+      benchmark_filter:
+        description: 'Benchmark filter pattern'
+        required: false
+        default: '.'
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  # Go benchmarks
+  benchmarks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2 # Need previous commit for comparison
+      
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.24.0'
+          cache: true
+      
+      - name: Run benchmarks
+        run: |
+          go test -bench="${{ github.event.inputs.benchmark_filter || '.' }}" \
+            -benchmem -count=5 -timeout=30m ./... | tee benchmark-results.txt
+      
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: 'go'
+          output-file-path: benchmark-results.txt
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: ${{ github.event_name != 'pull_request' }}
+          comment-on-alert: true
+          alert-threshold: '150%'
+          fail-on-alert: false
+      
+      - name: Upload benchmark results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-results
+          path: benchmark-results.txt
+
+  # Memory profiling
+  memory-profile:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    steps:
+      - uses: actions/checkout@v4
+      
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.24.0'
+          cache: true
+      
+      - name: Install pprof tools
+        run: |
+          go install github.com/google/pprof@latest
+      
+      - name: Run memory profiling tests
+        run: |
+          # Run specific memory-intensive tests with profiling
+          go test -memprofile=mem.prof -run=TestMemoryIntensive ./...
+          
+          # Generate memory profile report
+          go tool pprof -text mem.prof > memory-profile.txt || true
+      
+      - name: Upload memory profile
+        uses: actions/upload-artifact@v4
+        with:
+          name: memory-profile
+          path: memory-profile.txt
+
+  # Load testing with k6 (if applicable)
+  load-test:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    services:
+      cortex:
+        image: quay.io/cortexproject/cortex:latest
+        ports:
+          - 9009:9009
+        options: >-
+          --health-cmd "wget --no-verbose --tries=1 --spider http://localhost:9009/ready || exit 1"
+          --health-interval 30s
+          --health-timeout 10s
+          --health-retries 5
+    steps:
+      - uses: actions/checkout@v4
+      
+      - name: Setup k6
+        uses: grafana/setup-k6-action@v1
+      
+      - name: Run load tests
+        if: hashFiles('tests/load/*.js') != ''
+        run: |
+          k6 run tests/load/basic-load-test.js || echo "Load test completed"
+      
+      - name: Upload load test results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: load-test-results
+          path: |
+            summary.json
+            result.html
+
+  # Binary size tracking
+  binary-size:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+      
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.24.0'
+          cache: true
+      
+      - name: Build binaries
+        run: |
+          make dist
+      
+      - name: Check binary sizes
+        run: |
+          echo "Binary sizes:" > binary-sizes.txt
+          for binary in dist/cortex-*; do
+            size=$(stat --format="%s" "$binary")
+            echo "$(basename "$binary"): ${size} bytes" >> binary-sizes.txt
+          done
+          cat binary-sizes.txt
+      
+      - name: Compare with previous version
+        if: github.event_name == 'pull_request'
+        run: |
+          # Get previous commit binary sizes for comparison
+          git checkout HEAD~1
+          make dist
+          echo "Previous binary sizes:" > prev-binary-sizes.txt
+          for binary in dist/cortex-*; do
+            size=$(stat --format="%s" "$binary")
+            echo "$(basename "$binary"): ${size} bytes" >> prev-binary-sizes.txt
+          done
+          
+          # Compare sizes
+          echo "Size comparison:" > size-comparison.txt
+          # Add comparison logic here
+      
+      - name: Upload size reports
+        uses: actions/upload-artifact@v4
+        with:
+          name: binary-size-report
+          path: |
+            binary-sizes.txt
+            prev-binary-sizes.txt
+            size-comparison.txt
+
+  # Performance regression detection
+  performance-regression:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.24.0'
+          cache: true
+      
+      - name: Install benchstat
+        run: go install golang.org/x/perf/cmd/benchstat@latest
+      
+      - name: Run benchmarks for base commit
+        run: |
+          git checkout ${{ github.event.pull_request.base.sha }}
+          go test -bench=. -count=10 -benchmem ./... > base-bench.txt
+      
+      - name: Run benchmarks for PR commit
+        run: |
+          git checkout ${{ github.event.pull_request.head.sha }}
+          go test -bench=. -count=10 -benchmem ./... > pr-bench.txt
+      
+      - name: Compare benchmarks
+        run: |
+          benchstat base-bench.txt pr-bench.txt > bench-comparison.txt
+          cat bench-comparison.txt
+      
+      - name: Comment PR with benchmark results
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            try {
+              const comparison = fs.readFileSync('bench-comparison.txt', 'utf8');
+              const comment = `
+            ## 📊 Benchmark Comparison
+            
+            \`\`\`
+            ${comparison}
+            \`\`\`
+            
+            > This compares the performance of the base branch with your changes.
+            > Significant regressions are highlighted above.
+              `;
+              
+              github.rest.issues.createComment({
+                issue_number: context.issue.number,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body: comment
+              });
+            } catch (error) {
+              console.log('No benchmark comparison available');
+            }
\ No newline at end of file
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
new file mode 100644
index 00000000000..17e058a2e04
--- /dev/null
+++ b/.github/workflows/release.yml
@@ -0,0 +1,328 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - 'v*'
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Release version (e.g., v1.2.3)'
+        required: true
+      prerelease:
+        description: 'Mark as pre-release'
+        required: false
+        default: false
+        type: boolean
+
+permissions:
+  contents: write
+  packages: write
+  attestations: write
+  id-token: write
+
+env:
+  REGISTRY: quay.io
+  IMAGE_PREFIX: quay.io/cortexproject/
+
+jobs:
+  # Validate release
+  validate:
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.version.outputs.version }}
+      is-prerelease: ${{ steps.version.outputs.is-prerelease }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      
+      - name: Determine version
+        id: version
+        run: |
+          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            VERSION="${{ github.event.inputs.version }}"
+            IS_PRERELEASE="${{ github.event.inputs.prerelease }}"
+          else
+            VERSION=${GITHUB_REF#refs/tags/}
+            # Check if it's a pre-release (contains alpha, beta, rc)
+            if [[ $VERSION =~ (alpha|beta|rc) ]]; then
+              IS_PRERELEASE=true
+            else
+              IS_PRERELEASE=false
+            fi
+          fi
+          
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+          echo "is-prerelease=$IS_PRERELEASE" >> $GITHUB_OUTPUT
+          echo "Releasing version: $VERSION (prerelease: $IS_PRERELEASE)"
+      
+      - name: Validate version format
+        run: |
+          if [[ ! "${{ steps.version.outputs.version }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]; then
+            echo "Invalid version format. Expected: vX.Y.Z"
+            exit 1
+          fi
+
+  # Build release artifacts
+  build:
+    runs-on: ubuntu-latest
+    needs: validate
+    strategy:
+      matrix:
+        include:
+          - os: linux
+            arch: amd64
+          - os: linux
+            arch: arm64
+          - os: darwin
+            arch: amd64
+          - os: darwin
+            arch: arm64
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.24.0'
+          cache: true
+      
+      - name: Build binary
+        env:
+          VERSION: ${{ needs.validate.outputs.version }}
+          GOOS: ${{ matrix.os }}
+          GOARCH: ${{ matrix.arch }}
+        run: |
+          CGO_ENABLED=0 go build \
+            -ldflags "-X main.Branch=${GITHUB_REF#refs/heads/} -X main.Revision=${GITHUB_SHA} -X main.Version=${VERSION} -extldflags '-static' -s -w" \
+            -tags netgo \
+            -o cortex-${GOOS}-${GOARCH} \
+            ./cmd/cortex
+          
+          # Create archive
+          if [[ "${{ matrix.os }}" == "windows" ]]; then
+            zip cortex-${VERSION}-${GOOS}-${GOARCH}.zip cortex-${GOOS}-${GOARCH}
+          else
+            tar -czf cortex-${VERSION}-${GOOS}-${GOARCH}.tar.gz cortex-${GOOS}-${GOARCH}
+          fi
+          
+          # Generate checksums
+          if [[ "${{ matrix.os }}" == "windows" ]]; then
+            sha256sum cortex-${VERSION}-${GOOS}-${GOARCH}.zip > cortex-${VERSION}-${GOOS}-${GOARCH}.zip.sha256
+          else
+            sha256sum cortex-${VERSION}-${GOOS}-${GOARCH}.tar.gz > cortex-${VERSION}-${GOOS}-${GOARCH}.tar.gz.sha256
+          fi
+      
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: cortex-${{ matrix.os }}-${{ matrix.arch }}
+          path: |
+            cortex-${{ needs.validate.outputs.version }}-${{ matrix.os }}-${{ matrix.arch }}.*
+
+  # Build and push container images
+  container:
+    runs-on: ubuntu-latest
+    needs: validate
+    outputs:
+      image-digest: ${{ steps.build.outputs.digest }}
+      image-uri: ${{ steps.build.outputs.image-uri }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      
+      - name: Login to Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ secrets.QUAY_REGISTRY_USER }}
+          password: ${{ secrets.QUAY_REGISTRY_PASSWORD }}
+      
+      - name: Extract metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.IMAGE_PREFIX }}cortex
+          tags: |
+            type=semver,pattern={{version}},value=${{ needs.validate.outputs.version }}
+            type=semver,pattern={{major}}.{{minor}},value=${{ needs.validate.outputs.version }}
+            type=semver,pattern={{major}},value=${{ needs.validate.outputs.version }}
+      
+      - name: Build and push
+        id: build
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          # Enhanced security features
+          provenance: true
+          sbom: true
+          build-args: |
+            VERSION=${{ needs.validate.outputs.version }}
+            GIT_REVISION=${{ github.sha }}
+      
+      - name: Generate attestation
+        uses: actions/attest-build-provenance@v1
+        with:
+          subject-name: ${{ env.IMAGE_PREFIX }}cortex
+          subject-digest: ${{ steps.build.outputs.digest }}
+          push-to-registry: true
+
+  # Build packages (DEB/RPM)
+  packages:
+    runs-on: ubuntu-latest
+    needs: [validate, build]
+    strategy:
+      matrix:
+        arch: [amd64, arm64]
+    steps:
+      - uses: actions/checkout@v4
+      
+      - name: Download binary artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: cortex-linux-${{ matrix.arch }}
+      
+      - name: Install packaging tools
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y rpm
+          gem install fpm
+      
+      - name: Create packages
+        env:
+          VERSION: ${{ needs.validate.outputs.version }}
+          ARCH: ${{ matrix.arch }}
+        run: |
+          # Prepare binary
+          chmod +x cortex-linux-${ARCH}
+          mkdir -p package-root/usr/local/bin
+          cp cortex-linux-${ARCH} package-root/usr/local/bin/cortex
+          
+          # Create DEB package
+          fpm -s dir -t deb -n cortex -v ${VERSION#v} \
+            --architecture ${ARCH} \
+            --description "Cortex: A horizontally scalable, highly available, multi-tenant, long term Prometheus." \
+            --url "https://github.com/cortexproject/cortex" \
+            --license "Apache-2.0" \
+            --maintainer "Cortex Team <cortex-team@googlegroups.com>" \
+            --after-install packaging/deb/control/postinst \
+            --before-remove packaging/deb/control/prerm \
+            --config-files /etc/cortex/single-process-config.yaml \
+            --config-files /etc/default/cortex \
+            -C package-root \
+            usr/local/bin/cortex=/usr/local/bin/cortex \
+            docs/configuration/single-process-config-blocks.yaml=/etc/cortex/single-process-config.yaml \
+            packaging/deb/default/cortex=/etc/default/cortex \
+            packaging/deb/systemd/cortex.service=/etc/systemd/system/cortex.service
+          
+          # Create RPM package
+          fpm -s dir -t rpm -n cortex -v ${VERSION#v} \
+            --architecture ${ARCH} \
+            --description "Cortex: A horizontally scalable, highly available, multi-tenant, long term Prometheus." \
+            --url "https://github.com/cortexproject/cortex" \
+            --license "Apache-2.0" \
+            --maintainer "Cortex Team <cortex-team@googlegroups.com>" \
+            --after-install packaging/rpm/control/post \
+            --before-remove packaging/rpm/control/preun \
+            --config-files /etc/cortex/single-process-config.yaml \
+            --config-files /etc/sysconfig/cortex \
+            -C package-root \
+            usr/local/bin/cortex=/usr/local/bin/cortex \
+            docs/configuration/single-process-config-blocks.yaml=/etc/cortex/single-process-config.yaml \
+            packaging/rpm/sysconfig/cortex=/etc/sysconfig/cortex \
+            packaging/rpm/systemd/cortex.service=/etc/systemd/system/cortex.service
+      
+      - name: Upload packages
+        uses: actions/upload-artifact@v4
+        with:
+          name: packages-${{ matrix.arch }}
+          path: |
+            *.deb
+            *.rpm
+
+  # Create GitHub release
+  release:
+    runs-on: ubuntu-latest
+    needs: [validate, build, container, packages]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      
+      - name: Download all artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: artifacts
+      
+      - name: Prepare release assets
+        run: |
+          mkdir -p release-assets
+          find artifacts -type f \( -name "*.tar.gz" -o -name "*.zip" -o -name "*.sha256" -o -name "*.deb" -o -name "*.rpm" \) \
+            -exec cp {} release-assets/ \;
+      
+      - name: Generate release notes
+        id: release-notes
+        run: |
+          # Generate changelog since last tag
+          PREVIOUS_TAG=$(git tag --sort=-version:refname | grep -v "${{ needs.validate.outputs.version }}" | head -n1)
+          if [[ -n "$PREVIOUS_TAG" ]]; then
+            echo "## Changes since $PREVIOUS_TAG" > release-notes.md
+            git log --pretty=format:"- %s (%h)" ${PREVIOUS_TAG}..${{ github.sha }} >> release-notes.md
+          else
+            echo "## Initial Release" > release-notes.md
+          fi
+          
+          # Add container image info
+          echo "" >> release-notes.md
+          echo "## Container Images" >> release-notes.md
+          echo "- \`${{ env.IMAGE_PREFIX }}cortex:${{ needs.validate.outputs.version }}\`" >> release-notes.md
+          echo "- \`${{ env.IMAGE_PREFIX }}cortex:latest\`" >> release-notes.md
+          echo "" >> release-notes.md
+          echo "**Digest:** \`${{ needs.container.outputs.image-digest }}\`" >> release-notes.md
+      
+      - name: Create Release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: ${{ needs.validate.outputs.version }}
+          name: Release ${{ needs.validate.outputs.version }}
+          body_path: release-notes.md
+          draft: false
+          prerelease: ${{ needs.validate.outputs.is-prerelease == 'true' }}
+          files: release-assets/*
+          generate_release_notes: true
+      
+      - name: Update latest tag
+        if: needs.validate.outputs.is-prerelease == 'false'
+        run: |
+          git tag -f latest
+          git push -f origin latest
+
+  # SLSA provenance
+  provenance:
+    needs: [validate, container]
+    permissions:
+      actions: read
+      id-token: write
+      contents: write
+    uses: slsa-framework/slsa-github-generator/.github/workflows/generator_container_slsa3.yml@v2.0.0
+    with:
+      image: ${{ needs.container.outputs.image-uri }}
+      digest: ${{ needs.container.outputs.image-digest }}
+    secrets:
+      registry-username: ${{ secrets.QUAY_REGISTRY_USER }}
+      registry-password: ${{ secrets.QUAY_REGISTRY_PASSWORD }}
\ No newline at end of file
diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml
new file mode 100644
index 00000000000..17297409cf4
--- /dev/null
+++ b/.github/workflows/security.yml
@@ -0,0 +1,163 @@
+name: Security & Dependencies
+
+on:
+  schedule:
+    # Run daily at 6 AM UTC
+    - cron: '0 6 * * *'
+  workflow_dispatch:
+  push:
+    branches: [master]
+    paths:
+      - 'go.mod'
+      - 'go.sum'
+      - '.github/workflows/security.yml'
+
+permissions:
+  contents: read
+  security-events: write
+  pull-requests: write
+
+jobs:
+  # Vulnerability scanning
+  trivy-scan:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      
+      - name: Run Trivy vulnerability scanner in repo mode
+        uses: aquasecurity/trivy-action@master
+        with:
+          scan-type: 'fs'
+          scan-ref: '.'
+          format: 'sarif'
+          output: 'trivy-results.sarif'
+      
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v3
+        if: always()
+        with:
+          sarif_file: 'trivy-results.sarif'
+
+  # Container image scanning (if images exist)
+  container-scan:
+    runs-on: ubuntu-latest
+    if: github.event_name != 'schedule' # Skip for scheduled runs to avoid rate limits
+    steps:
+      - uses: actions/checkout@v4
+      
+      - name: Build image for scanning
+        run: |
+          docker build -t cortex-scan:latest .
+      
+      - name: Run Trivy vulnerability scanner on image
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: 'cortex-scan:latest'
+          format: 'sarif'
+          output: 'trivy-image-results.sarif'
+      
+      - name: Upload image scan results
+        uses: github/codeql-action/upload-sarif@v3
+        if: always()
+        with:
+          sarif_file: 'trivy-image-results.sarif'
+
+  # Dependency updates (Dependabot alternative for more control)
+  dependency-update:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+      
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.24.0'
+          cache: true
+      
+      - name: Update Go dependencies
+        run: |
+          go get -u ./...
+          go mod tidy
+      
+      - name: Check for changes
+        id: changes
+        run: |
+          if git diff --quiet go.mod go.sum; then
+            echo "changed=false" >> $GITHUB_OUTPUT
+          else
+            echo "changed=true" >> $GITHUB_OUTPUT
+          fi
+      
+      - name: Create Pull Request
+        if: steps.changes.outputs.changed == 'true'
+        uses: peter-evans/create-pull-request@v6
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: 'chore: update Go dependencies'
+          title: 'chore: update Go dependencies'
+          body: |
+            This PR updates Go dependencies to their latest versions.
+            
+            Changes:
+            - Updated Go module dependencies
+            - Ran `go mod tidy` to clean up go.sum
+            
+            Please review the changes and ensure all tests pass.
+          branch: chore/update-dependencies
+          delete-branch: true
+
+  # License compliance check
+  license-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.24.0'
+          cache: true
+      
+      - name: Install go-licenses
+        run: go install github.com/google/go-licenses@latest
+      
+      - name: Check licenses
+        run: |
+          go-licenses check ./...
+      
+      - name: Generate license report
+        run: |
+          go-licenses report ./... > license-report.txt
+      
+      - name: Upload license report
+        uses: actions/upload-artifact@v4
+        with:
+          name: license-report
+          path: license-report.txt
+
+  # Supply chain security with OpenSSF Scorecard
+  scorecard:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    permissions:
+      security-events: write
+      id-token: write
+      contents: read
+      actions: read
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      
+      - name: Run analysis
+        uses: ossf/scorecard-action@v2.4.0
+        with:
+          results_file: results.sarif
+          results_format: sarif
+          publish_results: true
+      
+      - name: Upload SARIF results
+        uses: github/codeql-action/upload-sarif@v3
+        with:
+          sarif_file: results.sarif
\ No newline at end of file

From 3f378f61462df4d35e11839fc5eddd5b8949a18b Mon Sep 17 00:00:00 2001
From: Charlie Le <charlie_le@apple.com>
Date: Thu, 19 Jun 2025 13:10:38 -0700
Subject: [PATCH 02/12] Remove SLSA provenance

Signed-off-by: Charlie Le <charlie_le@apple.com>
---
 .github/workflows/ci.yml | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d8d19a66a6c..44101c5e3c7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -425,17 +425,17 @@ jobs:
           echo "Digest: ${{ needs.build.outputs.image-digest }}"
 
   # Generate SLSA provenance
-  provenance:
-    needs: [build, deploy]
-    if: startsWith(github.ref, 'refs/tags/')
-    permissions:
-      actions: read
-      id-token: write
-      contents: write
-    uses: slsa-framework/slsa-github-generator/.github/workflows/generator_container_slsa3.yml@v2.0.0
-    with:
-      image: ${{ needs.build.outputs.image-tag }}
-      digest: ${{ needs.build.outputs.image-digest }}
-    secrets:
-      registry-username: ${{ secrets.QUAY_REGISTRY_USER }}
-      registry-password: ${{ secrets.QUAY_REGISTRY_PASSWORD }}
\ No newline at end of file
+#  provenance:
+#    needs: [build, deploy]
+#    if: startsWith(github.ref, 'refs/tags/')
+#    permissions:
+#      actions: read
+#      id-token: write
+#      contents: write
+#    uses: slsa-framework/slsa-github-generator/.github/workflows/generator_container_slsa3.yml@v2.0.0
+#    with:
+#      image: ${{ needs.build.outputs.image-tag }}
+#      digest: ${{ needs.build.outputs.image-digest }}
+#    secrets:
+#      registry-username: ${{ secrets.QUAY_REGISTRY_USER }}
+#      registry-password: ${{ secrets.QUAY_REGISTRY_PASSWORD }}

From 25d518b0c46529b1f204d9bdf3aeed92d1dfd461 Mon Sep 17 00:00:00 2001
From: Charlie Le <charlie_le@apple.com>
Date: Thu, 19 Jun 2025 13:24:54 -0700
Subject: [PATCH 03/12] just build linux/arm64 for faster iterations. make .go
 change to test changes path

Signed-off-by: Charlie Le <charlie_le@apple.com>
---
 Makefile                        | 10 +++++-----
 integration/e2ecortex/client.go |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/Makefile b/Makefile
index 705e005dac1..542d5be5e62 100644
--- a/Makefile
+++ b/Makefile
@@ -320,14 +320,14 @@ dist: dist/$(UPTODATE)
 dist/$(UPTODATE):
 	rm -fr ./dist
 	mkdir -p ./dist
-	for os in linux darwin; do \
-      for arch in amd64 arm64; do \
+	for os in linux ; do \
+      for arch in  arm64; do \
         echo "Building Cortex for $$os/$$arch"; \
         GOOS=$$os GOARCH=$$arch CGO_ENABLED=0 go build $(GO_FLAGS) -o ./dist/cortex-$$os-$$arch ./cmd/cortex; \
         sha256sum ./dist/cortex-$$os-$$arch | cut -d ' ' -f 1 > ./dist/cortex-$$os-$$arch-sha-256; \
-        echo "Building query-tee for $$os/$$arch"; \
-        GOOS=$$os GOARCH=$$arch CGO_ENABLED=0 go build $(GO_FLAGS) -o ./dist/query-tee-$$os-$$arch ./cmd/query-tee; \
-        sha256sum ./dist/query-tee-$$os-$$arch | cut -d ' ' -f 1 > ./dist/query-tee-$$os-$$arch-sha-256; \
+#        echo "Building query-tee for $$os/$$arch"; \
+#        GOOS=$$os GOARCH=$$arch CGO_ENABLED=0 go build $(GO_FLAGS) -o ./dist/query-tee-$$os-$$arch ./cmd/query-tee; \
+#        sha256sum ./dist/query-tee-$$os-$$arch | cut -d ' ' -f 1 > ./dist/query-tee-$$os-$$arch-sha-256; \
       done; \
     done; \
     touch $@
diff --git a/integration/e2ecortex/client.go b/integration/e2ecortex/client.go
index 9067b60c078..81acd3fc450 100644
--- a/integration/e2ecortex/client.go
+++ b/integration/e2ecortex/client.go
@@ -40,7 +40,7 @@ var ErrNotFound = errors.New("not found")
 
 var DefaultFilter = RuleFilter{}
 
-// Client is a client used to interact with Cortex in integration tests
+// Client is a client used to interact with Cortex in integration tests.
 type Client struct {
 	alertmanagerClient  promapi.Client
 	querierAddress      string

From 724f95555cc2f517caec7f42dd8799a3f805fa37 Mon Sep 17 00:00:00 2001
From: Charlie Le <charlie_le@apple.com>
Date: Thu, 19 Jun 2025 13:29:49 -0700
Subject: [PATCH 04/12] fix lint ci

Signed-off-by: Charlie Le <charlie_le@apple.com>
---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 44101c5e3c7..5c21b1b0a11 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -100,7 +100,7 @@ jobs:
       
       # Additional security and quality checks
       - name: Run gosec security scanner
-        uses: securecodewarrior/github-action-gosec@master
+        uses: securego/gosec@master
         with:
           args: '-no-fail -fmt sarif -out gosec.sarif ./...'
       

From f396a36ee1b7e53ff629ce886569c8b948bb689d Mon Sep 17 00:00:00 2001
From: Charlie Le <charlie_le@apple.com>
Date: Thu, 19 Jun 2025 13:35:51 -0700
Subject: [PATCH 05/12] Fix image name for build-image

Signed-off-by: Charlie Le <charlie_le@apple.com>
---
 .github/workflows/build-image.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml
index 70a5f04c830..6e5896a1889 100644
--- a/.github/workflows/build-image.yml
+++ b/.github/workflows/build-image.yml
@@ -19,7 +19,7 @@ permissions:
 
 env:
   REGISTRY: quay.io
-  IMAGE_NAME: cortexproject/build-image
+  IMAGE_NAME: cortexproject/build-image:master-7ce1d1b12
 
 jobs:
   build:

From 1ea195f625ed4b14dd14121dfaae693eef964409 Mon Sep 17 00:00:00 2001
From: Charlie Le <charlie_le@apple.com>
Date: Thu, 19 Jun 2025 13:45:16 -0700
Subject: [PATCH 06/12] fixes

Error: Failed to run: Error: Command failed: /home/runner/golangci-lint-1.64.8-linux-amd64/golangci-lint config verify
jsonschema: "output.formats" does not validate with "/properties/output/properties/formats/type": got object, want array
jsonschema: "linters.exclusions" does not validate with "/properties/linters/properties/exclusions/additionalProperties": additional properties 'presets' not allowed
jsonschema: "linters" does not validate with "/properties/linters/additionalProperties": additional properties 'settings' not allowed
jsonschema: "" does not validate with "/additionalProperties": additional properties 'formatters', 'version' not allowed
Failed executing command with error: the configuration contains invalid elements
, Error: Command failed: /home/runner/golangci-lint-1.64.8-linux-amd64/golangci-lint config verify
jsonschema: "output.formats" does not validate with "/properties/output/properties/formats/type": got object, want array
jsonschema: "linters.exclusions" does not validate with "/properties/linters/properties/exclusions/additionalProperties": additional properties 'presets' not allowed
jsonschema: "linters" does not validate with "/properties/linters/additionalProperties": additional properties 'settings' not allowed
jsonschema: "" does not validate with "/additionalProperties": additional properties 'formatters', 'version' not allowed
Failed executing command with error: the configuration contains invalid elements

build
buildx failed with: ERROR: invalid tag "quay.io/cortexproject/build-image:master-7ce1d1b12:pr-83": invalid reference format

Signed-off-by: Charlie Le <charlie_le@apple.com>
---
 .github/workflows/build-image.yml |  2 +-
 .github/workflows/ci.yml          | 13 +++++++------
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml
index 6e5896a1889..70a5f04c830 100644
--- a/.github/workflows/build-image.yml
+++ b/.github/workflows/build-image.yml
@@ -19,7 +19,7 @@ permissions:
 
 env:
   REGISTRY: quay.io
-  IMAGE_NAME: cortexproject/build-image:master-7ce1d1b12
+  IMAGE_NAME: cortexproject/build-image
 
 jobs:
   build:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5c21b1b0a11..0f63276771a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -81,6 +81,8 @@ jobs:
     runs-on: ubuntu-latest
     needs: [changes, setup-go]
     if: needs.changes.outputs.go == 'true'
+    env:
+      GO111MODULE: on
     steps:
       - uses: actions/checkout@v4
         with:
@@ -93,22 +95,21 @@ jobs:
       
       # Modern linting with latest golangci-lint
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v6
+        uses: golangci/golangci-lint-action@v8
         with:
-          version: latest
-          args: --timeout=10m
+          version: v2.1
       
       # Additional security and quality checks
       - name: Run gosec security scanner
         uses: securego/gosec@master
         with:
-          args: '-no-fail -fmt sarif -out gosec.sarif ./...'
+          args: '-no-fail -fmt sarif -out results.sarif ./...'
       
       - name: Upload SARIF file
         if: always()
-        uses: github/codeql-action/upload-sarif@v3
+        uses: github/codeql-action/upload-sarif@v2
         with:
-          sarif_file: gosec.sarif
+          sarif_file: results.sarif
       
       - name: Check dependencies
         run: |

From f35e8e3bf629520a067f388ef4660541998590af Mon Sep 17 00:00:00 2001
From: Charlie Le <charlie_le@apple.com>
Date: Thu, 19 Jun 2025 13:54:32 -0700
Subject: [PATCH 07/12] fix branch

Signed-off-by: Charlie Le <charlie_le@apple.com>
---
 .github/workflows/build-image.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml
index 70a5f04c830..b982c1f942b 100644
--- a/.github/workflows/build-image.yml
+++ b/.github/workflows/build-image.yml
@@ -54,7 +54,7 @@ jobs:
           tags: |
             type=ref,event=branch
             type=ref,event=pr
-            type=sha,prefix={{branch}}-
+            type=sha,enable={{is_default_branch}},prefix={{branch}}-
             type=raw,value=latest,enable={{is_default_branch}}
 
       - name: Build and push

From 350c4bffa963e443b81e89ac1c8a3a72c86376a4 Mon Sep 17 00:00:00 2001
From: Charlie Le <charlie_le@apple.com>
Date: Thu, 19 Jun 2025 14:09:47 -0700
Subject: [PATCH 08/12] parallelize

Signed-off-by: Charlie Le <charlie_le@apple.com>
---
 .github/workflows/ci.yml | 160 ++++++++++++++++++++++++++++++++++-----
 1 file changed, 142 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 0f63276771a..3b825812eaf 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -123,42 +123,165 @@ jobs:
           make check-doc
           make check-white-noise
 
-  # Unit tests with coverage
+  # Unit tests with coverage - parallelized by package groups
   test:
     runs-on: ubuntu-latest
     needs: [changes, setup-go]
     if: needs.changes.outputs.go == 'true'
     strategy:
+      fail-fast: false
       matrix:
-        test-type: [unit, unit-no-race]
+        include:
+          # Core storage and ingestion (heavy tests)
+          - name: "storage-ingestion"
+            packages: "./pkg/storage/... ./pkg/ingester/... ./pkg/chunk/..."
+            race: true
+          - name: "storage-ingestion-no-race"
+            packages: "./pkg/storage/... ./pkg/ingester/... ./pkg/chunk/..."
+            race: false
+
+          # Query path (heavy tests)
+          - name: "querier-frontend"
+            packages: "./pkg/querier/... ./pkg/frontend/... ./pkg/scheduler/..."
+            race: true
+          - name: "querier-frontend-no-race"
+            packages: "./pkg/querier/... ./pkg/frontend/... ./pkg/scheduler/..."
+            race: false
+
+          # Ruler and alertmanager (moderate tests)
+          - name: "ruler-alertmanager"
+            packages: "./pkg/ruler/... ./pkg/alertmanager/..."
+            race: true
+          - name: "ruler-alertmanager-no-race"
+            packages: "./pkg/ruler/... ./pkg/alertmanager/..."
+            race: false
+
+          # Distributor and ring (moderate tests)
+          - name: "distributor-ring"
+            packages: "./pkg/distributor/... ./pkg/ring/... ./pkg/ha/..."
+            race: true
+          - name: "distributor-ring-no-race"
+            packages: "./pkg/distributor/... ./pkg/ring/... ./pkg/ha/..."
+            race: false
+
+          # Compactor and store gateway (moderate tests)
+          - name: "compactor-storegateway"
+            packages: "./pkg/compactor/... ./pkg/storegateway/..."
+            race: true
+          - name: "compactor-storegateway-no-race"
+            packages: "./pkg/compactor/... ./pkg/storegateway/..."
+            race: false
+
+          # Utilities and smaller packages (lighter tests)
+          - name: "utilities"
+            packages: "./pkg/util/... ./pkg/tenant/... ./pkg/tracing/... ./pkg/api/... ./pkg/configs/... ./pkg/cortex/... ./pkg/flusher/... ./pkg/purger/..."
+            race: true
+          - name: "utilities-no-race"
+            packages: "./pkg/util/... ./pkg/tenant/... ./pkg/tracing/... ./pkg/api/... ./pkg/configs/... ./pkg/cortex/... ./pkg/flusher/... ./pkg/purger/..."
+            race: false
+
+          # Command line tools and remaining packages
+          - name: "cmd-tools"
+            packages: "./cmd/... ./tools/... ./pkg/cortexpb/... ./pkg/engine/... ./pkg/parquetconverter/... ./pkg/querysharding/... ./pkg/testexporter/..."
+            race: true
+          - name: "cmd-tools-no-race"
+            packages: "./cmd/... ./tools/... ./pkg/cortexpb/... ./pkg/engine/... ./pkg/parquetconverter/... ./pkg/querysharding/... ./pkg/testexporter/..."
+            race: false
+
     steps:
       - uses: actions/checkout@v4
-      
+
       - uses: actions/setup-go@v5
         with:
           go-version: ${{ env.GO_VERSION }}
           cache: true
-      
+
       - name: Run tests
+        env:
+          RACE_FLAG: ${{ matrix.race && '-race' || '' }}
+          COVERAGE_FILE: ${{ matrix.race && 'coverage' || 'coverage-no-race' }}-${{ matrix.name }}.out
         run: |
-          if [[ "${{ matrix.test-type }}" == "unit" ]]; then
-            go test -race -coverprofile=coverage.out -covermode=atomic -timeout=30m ./...
-          else
-            go test -coverprofile=coverage-no-race.out -covermode=atomic -timeout=30m ./...
-          fi
-      
+          echo "Running tests for: ${{ matrix.name }}"
+          echo "Packages: ${{ matrix.packages }}"
+          echo "Race detection: ${{ matrix.race }}"
+          
+          # Create coverage directory
+          mkdir -p coverage
+          
+          # Run tests with appropriate flags
+          go test $RACE_FLAG \
+            -coverprofile=coverage/$COVERAGE_FILE \
+            -covermode=atomic \
+            -timeout=30m \
+            -v \
+            ${{ matrix.packages }}
+
       - name: Upload coverage to Codecov
-        if: matrix.test-type == 'unit'
+        if: matrix.race == true
         uses: codecov/codecov-action@v4
         with:
-          file: coverage.out
+          file: coverage/coverage-${{ matrix.name }}.out
+          flags: ${{ matrix.name }}
+          name: ${{ matrix.name }}
           fail_ci_if_error: false
 
-  # Build Docker images efficiently
+      - name: Upload coverage artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-${{ matrix.name }}-${{ matrix.race && 'race' || 'no-race' }}
+          path: coverage/
+          retention-days: 1
+
+  # Combine coverage reports
+  coverage:
+    runs-on: ubuntu-latest
+    needs: test
+    if: always() && needs.test.result == 'success'
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: true
+
+      - name: Download all coverage artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: coverage-*-race
+          path: coverage-reports
+          merge-multiple: true
+
+      - name: Install gocovmerge
+        run: go install github.com/wadey/gocovmerge@latest
+
+      - name: Merge coverage reports
+        run: |
+          find coverage-reports -name "coverage-*.out" -exec echo "Found: {}" \;
+          gocovmerge coverage-reports/coverage-*.out > merged-coverage.out
+
+      - name: Upload merged coverage
+        uses: codecov/codecov-action@v4
+        with:
+          file: merged-coverage.out
+          name: merged-coverage
+          fail_ci_if_error: false
+
+      - name: Generate coverage HTML
+        run: |
+          go tool cover -html=merged-coverage.out -o coverage.html
+
+      - name: Upload coverage HTML
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-html
+          path: coverage.html
+
+    # Build Docker images efficiently
   build:
     runs-on: ubuntu-latest
-    needs: [changes, lint, test]
-    if: always() && (needs.lint.result == 'success' || needs.lint.result == 'skipped') && (needs.test.result == 'success' || needs.test.result == 'skipped')
+    needs: [changes, lint, coverage]
+    if: always() && (needs.lint.result == 'success' || needs.lint.result == 'skipped') && (needs.coverage.result == 'success' || needs.coverage.result == 'skipped')
     outputs:
       image-tag: ${{ steps.meta.outputs.tags }}
       image-digest: ${{ steps.build.outputs.digest }}
@@ -365,10 +488,10 @@ jobs:
         with:
           fail-on-severity: moderate
 
-  # Website deployment (only on main branch)
+    # Website deployment (only on main branch)
   deploy-website:
     runs-on: ubuntu-latest
-    needs: [build, test]
+    needs: [build, coverage]
     if: github.ref == 'refs/heads/master' && github.repository == 'cortexproject/cortex'
     steps:
       - uses: actions/checkout@v4
@@ -400,10 +523,11 @@ jobs:
           external_repository: cortexproject/cortexproject.github.io
           publish_branch: master
 
+  # Container image deployment
   # Container image deployment
   deploy:
     runs-on: ubuntu-latest
-    needs: [build, test, lint, integration, integration-configs-db]
+    needs: [build, coverage, lint, integration, integration-configs-db]
     if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex'
     steps:
       - uses: actions/checkout@v4

From 446cbd8798edfb6db32a92e6212fc95265ed42b0 Mon Sep 17 00:00:00 2001
From: Charlie Le <charlie_le@apple.com>
Date: Thu, 19 Jun 2025 14:31:57 -0700
Subject: [PATCH 09/12] build immediately

Signed-off-by: Charlie Le <charlie_le@apple.com>
---
 .github/workflows/ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3b825812eaf..e74a1d3873f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -280,8 +280,8 @@ jobs:
     # Build Docker images efficiently
   build:
     runs-on: ubuntu-latest
-    needs: [changes, lint, coverage]
-    if: always() && (needs.lint.result == 'success' || needs.lint.result == 'skipped') && (needs.coverage.result == 'success' || needs.coverage.result == 'skipped')
+    needs: [changes]
+    if: always()
     outputs:
       image-tag: ${{ steps.meta.outputs.tags }}
       image-digest: ${{ steps.build.outputs.digest }}

From ebc15b4fed70bdb194e12f5a3f8c8ab8a9f94ff7 Mon Sep 17 00:00:00 2001
From: Charlie Le <charlie_le@apple.com>
Date: Thu, 19 Jun 2025 14:45:42 -0700
Subject: [PATCH 10/12] Specify Dockerfile path in build step

Signed-off-by: Charlie Le <charlie_le@apple.com>
---
 .github/workflows/ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e74a1d3873f..179093b72d9 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -335,6 +335,7 @@ jobs:
         uses: docker/build-push-action@v6
         with:
           context: .
+          file: cmd/cortex/Dockerfile
           platforms: linux/amd64,linux/arm64
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}

From bbde17d83dd7bd7defa36f01e02abad9643363f5 Mon Sep 17 00:00:00 2001
From: Charlie Le <charlie_le@apple.com>
Date: Thu, 19 Jun 2025 14:48:53 -0700
Subject: [PATCH 11/12] Update Docker build platforms to only include
 linux/arm64

Signed-off-by: Charlie Le <charlie_le@apple.com>
---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 179093b72d9..4098703af9c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -336,7 +336,7 @@ jobs:
         with:
           context: .
           file: cmd/cortex/Dockerfile
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/arm64
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

From 911cbb6c76af20ef5b6a60702a64fa74359eacd7 Mon Sep 17 00:00:00 2001
From: Charlie Le <charlie_le@apple.com>
Date: Thu, 19 Jun 2025 14:57:31 -0700
Subject: [PATCH 12/12] Update CI workflow to build for arm64 and adjust
 Makefile architecture

Signed-off-by: Charlie Le <charlie_le@apple.com>
---
 .github/workflows/ci.yml | 5 ++---
 Makefile                 | 2 +-
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4098703af9c..88114560d98 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -327,15 +327,14 @@ jobs:
       # Build Go binaries
       - name: Build binaries
         run: |
-          make dist
+          make BUILD_IN_CONTAINER=false
       
       # Build and push multi-arch images
       - name: Build and push
         id: build
         uses: docker/build-push-action@v6
         with:
-          context: .
-          file: cmd/cortex/Dockerfile
+          context: cmd/cortex
           platforms: linux/arm64
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
diff --git a/Makefile b/Makefile
index 542d5be5e62..24580ed2f1d 100644
--- a/Makefile
+++ b/Makefile
@@ -10,7 +10,7 @@ VERSION=$(shell cat "./VERSION" 2> /dev/null)
 GOPROXY_VALUE=$(shell go env GOPROXY)
 
 # ARCHS
-ARCHS       = amd64 arm64
+ARCHS       = arm64
 
 # Boiler plate for building Docker containers.
 # All this must go at top of file I'm afraid.