Fix high priority issues: version sync, portable paths, and CI/CD

.github/workflows/quality.yml

@@ -0,0 +1,114 @@
+name: Code Quality
+
+on:
+  push:
+    branches: [ main, develop, claude/* ]
+  pull_request:
+    branches: [ main, develop ]
+
+permissions:
+  contents: read
+
+jobs:
+  lint:
+    name: Lint and Format Check
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install uv
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+
+      - name: Create virtual environment
+        run: uv venv
+
+      - name: Install ruff
+        run: |
+          source .venv/bin/activate
+          uv pip install ruff
+
+      - name: Run ruff linter
+        run: |
+          source .venv/bin/activate
+          ruff check src/ tests/ scripts/ --output-format=github
+        continue-on-error: true  # Don't fail the build on linting issues initially
+
+      - name: Run ruff formatter check
+        run: |
+          source .venv/bin/activate
+          ruff format --check src/ tests/ scripts/
+        continue-on-error: true  # Don't fail the build on formatting issues initially
+
+  type-check:
+    name: Type Checking
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install uv
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+
+      - name: Create virtual environment
+        run: uv venv
+
+      - name: Install dependencies
+        run: |
+          source .venv/bin/activate
+          uv pip install -e ".[dev]"
+          uv pip install mypy types-PyYAML types-requests
+
+      - name: Run mypy
+        run: |
+          source .venv/bin/activate
+          mypy src/ --ignore-missing-imports --no-strict-optional
+        continue-on-error: true  # Don't fail the build on type errors initially
+
+  security:
+    name: Security Scan
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install uv
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+
+      - name: Create virtual environment
+        run: uv venv
+
+      - name: Install safety
+        run: |
+          source .venv/bin/activate
+          uv pip install safety
+
+      - name: Check for known vulnerabilities
+        run: |
+          source .venv/bin/activate
+          safety check --json || echo "Security check completed with warnings"
+        continue-on-error: true

.github/workflows/test.yml

@@ -0,0 +1,72 @@
+name: Tests
+
+on:
+  push:
+    branches: [ main, develop, claude/* ]
+  pull_request:
+    branches: [ main, develop ]
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: Test on Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install uv
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+
+      - name: Create virtual environment
+        run: uv venv
+
+      - name: Install dependencies
+        run: |
+          source .venv/bin/activate
+          uv pip install -e ".[dev]"
+
+      - name: Create config directory
+        run: |
+          mkdir -p config
+          cp config/credentials.yaml.example config/credentials.yaml || echo "No credentials example found"
+          cp config/system_profile.yaml.example config/system_profile.yaml || echo "No system profile example found"
+
+      - name: Run unit tests
+        run: |
+          source .venv/bin/activate
+          pytest tests/unit/ -v --tb=short
+
+      - name: Run integration tests
+        run: |
+          source .venv/bin/activate
+          pytest tests/integration/ -v --tb=short
+        continue-on-error: true  # Integration tests may require external resources
+
+      - name: Generate coverage report
+        if: matrix.python-version == '3.11'
+        run: |
+          source .venv/bin/activate
+          pytest tests/unit/ --cov=src --cov-report=xml --cov-report=html --cov-report=term
+
+      - name: Upload coverage report
+        if: matrix.python-version == '3.11'
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-report
+          path: htmlcov/
+          retention-days: 30

config/pipeline_config.yaml

@@ -3,19 +3,25 @@
 # This file IS COMMITTED to git - use relative paths or generic defaults
 
 # Data Lake Root (Medallion Architecture: Landing → Bronze → Silver → Gold)
-data_lake_root: /Volumes/sandisk/quantmini-lake
+# IMPORTANT: For production use with large datasets, override this path:
+#   1. Set DATA_LAKE_ROOT environment variable, OR
+#   2. Edit config/paths.yaml (gitignored) with your environment-specific paths, OR
+#   3. Edit config/system_profile.yaml (gitignored) with your personal paths
+# Default: Uses relative "data" directory in project root
+data_lake_root: data
 
 # Layer-specific paths
-landing_path: /Volumes/sandisk/quantmini-lake/landing
-bronze_path: /Volumes/sandisk/quantmini-lake/bronze
-silver_path: /Volumes/sandisk/quantmini-lake/silver
-gold_path: /Volumes/sandisk/quantmini-lake/gold
+# Leave empty to auto-derive from data_lake_root (recommended)
+# Or specify custom paths for each layer
+landing_path: data/landing
+bronze_path: data/bronze
+silver_path: data/silver
+gold_path: data/gold
 
 # Legacy data root (deprecated - kept for backward compatibility)
-# Default: /Volumes/sandisk/quantmini-data (external drive)
 # For your personal path: Edit config/system_profile.yaml (gitignored)
 # Or set DATA_ROOT environment variable
-data_root: /Volumes/sandisk/quantmini-data
+data_root: data
 
 pipeline:
   # Processing mode: adaptive, streaming, batch, or parallel

src/cli/main.py

@@ -21,7 +21,7 @@
 
 
 @click.group()
-@click.version_option(version='0.1.0', prog_name='quantmini')
+@click.version_option(version='0.2.0', prog_name='quantmini')
 @click.pass_context
 def cli(ctx):
     """

src/transform/qlib_binary_validator.py

@@ -133,7 +133,15 @@ def _validate_instruments(self, data_type: str) -> Dict[str, Any]:
                 return result
 
             with open(instruments_file) as f:
-                symbols = [line.strip() for line in f if line.strip()]
+                # Instruments file format: SYMBOL\tstart_date\tend_date
+                # Extract just the symbol (first field)
+                symbols = []
+                for line in f:
+                    line = line.strip()
+                    if line:
+                        # Split by tab and take first field (symbol)
+                        symbol = line.split('\t')[0]
+                        symbols.append(symbol)
 
             if len(symbols) == 0:
                 result['errors'] = result.get('errors', [])

tests/unit/test_base_ingestor.py

@@ -13,7 +13,7 @@
 from src.ingest.base_ingestor import BaseIngestor, IngestionError
 
 
-class TestIngestor(BaseIngestor):
+class MockIngestor(BaseIngestor):
     """Concrete implementation for testing"""
 
     def ingest_date(self, date, data, symbols=None):
@@ -36,7 +36,7 @@ def test_config():
 @pytest.fixture
 def test_ingestor(tmp_path, test_config):
     """Create test ingestor instance"""
-    return TestIngestor(
+    return MockIngestor(
         data_type='stocks_daily',
         output_root=tmp_path / 'parquet',
         config=test_config
@@ -182,7 +182,7 @@ def test_memory_monitor_integration(test_ingestor):
 def test_repr(test_ingestor):
     """Test string representation"""
     repr_str = repr(test_ingestor)
-    assert 'TestIngestor' in repr_str
+    assert 'MockIngestor' in repr_str
     assert 'stocks_daily' in repr_str