diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml new file mode 100644 index 0000000..cca673f --- /dev/null +++ b/.github/workflows/quality.yml @@ -0,0 +1,114 @@ +name: Code Quality + +on: + push: + branches: [ main, develop, claude/* ] + pull_request: + branches: [ main, develop ] + +permissions: + contents: read + +jobs: + lint: + name: Lint and Format Check + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + + - name: Create virtual environment + run: uv venv + + - name: Install ruff + run: | + source .venv/bin/activate + uv pip install ruff + + - name: Run ruff linter + run: | + source .venv/bin/activate + ruff check src/ tests/ scripts/ --output-format=github + continue-on-error: true # Don't fail the build on linting issues initially + + - name: Run ruff formatter check + run: | + source .venv/bin/activate + ruff format --check src/ tests/ scripts/ + continue-on-error: true # Don't fail the build on formatting issues initially + + type-check: + name: Type Checking + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + + - name: Create virtual environment + run: uv venv + + - name: Install dependencies + run: | + source .venv/bin/activate + uv pip install -e ".[dev]" + uv pip install mypy types-PyYAML types-requests + + - name: Run mypy + run: | + source .venv/bin/activate + mypy src/ --ignore-missing-imports --no-strict-optional + continue-on-error: true # Don't fail the build on type errors initially + + security: + name: Security Scan + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + + - name: Create virtual environment + run: uv venv + + - name: Install safety + run: | + source .venv/bin/activate + uv pip install safety + + - name: Check for known vulnerabilities + run: | + source .venv/bin/activate + safety check --json || echo "Security check completed with warnings" + continue-on-error: true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..b0761e3 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,72 @@ +name: Tests + +on: + push: + branches: [ main, develop, claude/* ] + pull_request: + branches: [ main, develop ] + +permissions: + contents: read + +jobs: + test: + name: Test on Python ${{ matrix.python-version }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12"] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + + - name: Create virtual environment + run: uv venv + + - name: Install dependencies + run: | + source .venv/bin/activate + uv pip install -e ".[dev]" + + - name: Create config directory + run: | + mkdir -p config + cp config/credentials.yaml.example config/credentials.yaml || echo "No credentials example found" + cp config/system_profile.yaml.example config/system_profile.yaml || echo "No system profile example found" + + - name: Run unit tests + run: | + source .venv/bin/activate + pytest tests/unit/ -v --tb=short + + - name: Run integration tests + run: | + source .venv/bin/activate + pytest tests/integration/ -v --tb=short + continue-on-error: true # Integration tests may require external resources + + - name: Generate coverage report + if: matrix.python-version == '3.11' + run: | + source .venv/bin/activate + pytest tests/unit/ --cov=src --cov-report=xml --cov-report=html --cov-report=term + + - name: Upload coverage report + if: matrix.python-version == '3.11' + uses: actions/upload-artifact@v4 + with: + name: coverage-report + path: htmlcov/ + retention-days: 30 diff --git a/config/pipeline_config.yaml b/config/pipeline_config.yaml index 3d36f10..bde7db1 100755 --- a/config/pipeline_config.yaml +++ b/config/pipeline_config.yaml @@ -3,19 +3,25 @@ # This file IS COMMITTED to git - use relative paths or generic defaults # Data Lake Root (Medallion Architecture: Landing → Bronze → Silver → Gold) -data_lake_root: /Volumes/sandisk/quantmini-lake +# IMPORTANT: For production use with large datasets, override this path: +# 1. Set DATA_LAKE_ROOT environment variable, OR +# 2. Edit config/paths.yaml (gitignored) with your environment-specific paths, OR +# 3. Edit config/system_profile.yaml (gitignored) with your personal paths +# Default: Uses relative "data" directory in project root +data_lake_root: data # Layer-specific paths -landing_path: /Volumes/sandisk/quantmini-lake/landing -bronze_path: /Volumes/sandisk/quantmini-lake/bronze -silver_path: /Volumes/sandisk/quantmini-lake/silver -gold_path: /Volumes/sandisk/quantmini-lake/gold +# Leave empty to auto-derive from data_lake_root (recommended) +# Or specify custom paths for each layer +landing_path: data/landing +bronze_path: data/bronze +silver_path: data/silver +gold_path: data/gold # Legacy data root (deprecated - kept for backward compatibility) -# Default: /Volumes/sandisk/quantmini-data (external drive) # For your personal path: Edit config/system_profile.yaml (gitignored) # Or set DATA_ROOT environment variable -data_root: /Volumes/sandisk/quantmini-data +data_root: data pipeline: # Processing mode: adaptive, streaming, batch, or parallel diff --git a/src/cli/main.py b/src/cli/main.py index 37ee01e..761c153 100644 --- a/src/cli/main.py +++ b/src/cli/main.py @@ -21,7 +21,7 @@ @click.group() -@click.version_option(version='0.1.0', prog_name='quantmini') +@click.version_option(version='0.2.0', prog_name='quantmini') @click.pass_context def cli(ctx): """ diff --git a/src/transform/qlib_binary_validator.py b/src/transform/qlib_binary_validator.py index 8a93537..169a439 100755 --- a/src/transform/qlib_binary_validator.py +++ b/src/transform/qlib_binary_validator.py @@ -133,7 +133,15 @@ def _validate_instruments(self, data_type: str) -> Dict[str, Any]: return result with open(instruments_file) as f: - symbols = [line.strip() for line in f if line.strip()] + # Instruments file format: SYMBOL\tstart_date\tend_date + # Extract just the symbol (first field) + symbols = [] + for line in f: + line = line.strip() + if line: + # Split by tab and take first field (symbol) + symbol = line.split('\t')[0] + symbols.append(symbol) if len(symbols) == 0: result['errors'] = result.get('errors', []) diff --git a/tests/unit/test_base_ingestor.py b/tests/unit/test_base_ingestor.py index 20bcca7..00786fe 100755 --- a/tests/unit/test_base_ingestor.py +++ b/tests/unit/test_base_ingestor.py @@ -13,7 +13,7 @@ from src.ingest.base_ingestor import BaseIngestor, IngestionError -class TestIngestor(BaseIngestor): +class MockIngestor(BaseIngestor): """Concrete implementation for testing""" def ingest_date(self, date, data, symbols=None): @@ -36,7 +36,7 @@ def test_config(): @pytest.fixture def test_ingestor(tmp_path, test_config): """Create test ingestor instance""" - return TestIngestor( + return MockIngestor( data_type='stocks_daily', output_root=tmp_path / 'parquet', config=test_config @@ -182,7 +182,7 @@ def test_memory_monitor_integration(test_ingestor): def test_repr(test_ingestor): """Test string representation""" repr_str = repr(test_ingestor) - assert 'TestIngestor' in repr_str + assert 'MockIngestor' in repr_str assert 'stocks_daily' in repr_str