diff --git a/package.json b/package.json index 766e241ff8d6d..8d4e23b334340 100644 --- a/package.json +++ b/package.json @@ -129,7 +129,8 @@ "test:php": "node ./tools/local-env/scripts/docker.js run --rm php ./vendor/bin/phpunit", "test:coverage": "npm run test:php -- --coverage-html ./coverage/html/ --coverage-php ./coverage/php/report.php --coverage-text=./coverage/text/report.txt", "test:e2e": "wp-scripts test-playwright --config tests/e2e/playwright.config.js", - "test:visual": "wp-scripts test-playwright --config tests/visual-regression/playwright.config.js", + "test:visual": "bash tests/visual-regression/compare-branches.sh", + "test:visual:quick": "bash tests/visual-regression/compare-branches.sh --skip-baselines", "gutenberg:checkout": "node tools/gutenberg/checkout-gutenberg.js", "gutenberg:build": "node tools/gutenberg/build-gutenberg.js", "gutenberg:copy": "node tools/gutenberg/copy-gutenberg-build.js", diff --git a/tests/visual-regression/README.md b/tests/visual-regression/README.md index d7ef71e64324b..0d54abd4f88b6 100644 --- a/tests/visual-regression/README.md +++ b/tests/visual-regression/README.md @@ -2,10 +2,80 @@ These tests make use of Playwright, with a setup very similar to that of the e2e tests. +## Prerequisites + +- Node.js >= 20.10.0 +- A running WordPress environment (`npm run env:start`) +- Playwright browsers installed (`npx playwright install chromium`) + ## How to Run the Tests Locally -1. Check out trunk. -2. Run `npm run test:visual` to generate some base snapshots. +From a feature branch with a clean working tree, run: + +```bash +npm run test:visual +``` + +This will automatically: +1. Check out trunk to generate baseline snapshots. +2. Return to your feature branch. +3. Compare the current branch against those baselines. +4. Print a visual impact summary in the terminal. +5. Open the HTML report with side-by-side visual diffs. + +### Quick Re-run (Skip Baseline Generation) + +After the first run, baselines from trunk are already saved. To re-compare without regenerating them: + +```bash +npm run test:visual:quick +``` + +This is useful when iterating on CSS — no need to commit changes or wait for trunk baselines to regenerate each time. + +**Typical workflow:** +```bash +npm run test:visual # First run: generates baselines from trunk +# ... tweak CSS ... +npm run test:visual:quick # Fast: reuses existing baselines +# ... tweak more CSS ... +npm run test:visual:quick # Fast again +``` + +### Specifying a Base Branch + +By default, baselines are generated from `trunk`. To compare against a different branch: + +```bash +npm run test:visual -- some-other-branch +``` + +### Manual 2-Step Workflow + +You can also generate baselines and compare manually: + +1. Check out the base branch (e.g. trunk). +2. Run `npx playwright test --config tests/visual-regression/playwright.config.js --update-snapshots` 3. Check out the feature branch to be tested. -4. Run `npm run test:visual` again. If any tests fail, the diff images can be found in `artifacts/` +4. Run `npx playwright test --config tests/visual-regression/playwright.config.js` + +## Impact Summary + +After each run, a summary is printed to the terminal showing which pages changed and which remained unchanged: + +``` +──────────────────────────────────────────────── +Visual Impact Summary +──────────────────────────────────────────────── +Changed: 27 of 35 pages +Unchanged: 8 of 35 pages + +Changed: + Dashboard, All Posts, Categories, Tags, ... + +Unchanged: + Login, Media Settings, ... +──────────────────────────────────────────────── +``` +The HTML report also opens automatically with side-by-side visual diffs for detailed inspection. diff --git a/tests/visual-regression/compare-branches.sh b/tests/visual-regression/compare-branches.sh new file mode 100755 index 0000000000000..aa51ab1213dcb --- /dev/null +++ b/tests/visual-regression/compare-branches.sh @@ -0,0 +1,58 @@ +#!/bin/bash +set -e + +SKIP_BASELINES=false +BASELINE_BRANCH="trunk" + +# Parse arguments. +for arg in "$@"; do + case "$arg" in + --skip-baselines) + SKIP_BASELINES=true + ;; + *) + BASELINE_BRANCH="$arg" + ;; + esac +done + +CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CONFIG="$SCRIPT_DIR/playwright.config.js" + +if [ "$CURRENT_BRANCH" = "$BASELINE_BRANCH" ]; then + echo "Error: Already on $BASELINE_BRANCH. Checkout a feature branch first." + exit 1 +fi + +if [ "$SKIP_BASELINES" = true ]; then + # Verify baselines exist before skipping regeneration. + SNAPSHOT_DIR="$SCRIPT_DIR/specs/__snapshots__" + if [ ! -d "$SNAPSHOT_DIR" ] || [ -z "$(ls -A "$SNAPSHOT_DIR" 2>/dev/null)" ]; then + echo "Error: No baselines found. Run 'npm run test:visual' first to generate them." + exit 1 + fi + + echo "Skipping baseline generation (reusing existing baselines)..." + echo "Ensuring Playwright browsers are installed..." + npx playwright install --with-deps chromium + + echo "Comparing against existing baselines..." + npx playwright test --config "$CONFIG" || true +else + if ! git diff-index --quiet HEAD --; then + echo "Error: Uncommitted changes detected. Please commit or stash before running." + exit 1 + fi + + echo "Ensuring Playwright browsers are installed..." + npx playwright install --with-deps chromium + + echo "Generating baselines from $BASELINE_BRANCH..." + git checkout "$BASELINE_BRANCH" + npx playwright test --config "$CONFIG" --update-snapshots + + echo "Comparing against $CURRENT_BRANCH..." + git checkout "$CURRENT_BRANCH" + npx playwright test --config "$CONFIG" || true +fi diff --git a/tests/visual-regression/playwright.config.js b/tests/visual-regression/playwright.config.js index 759d887bf71c2..af4fcf9d1ee56 100644 --- a/tests/visual-regression/playwright.config.js +++ b/tests/visual-regression/playwright.config.js @@ -15,9 +15,25 @@ process.env.STORAGE_STATE_PATH ??= path.join( 'storage-states/admin.json' ); +const reporter = [ + [ 'list' ], + [ + 'html', + { + open: process.env.CI ? 'never' : 'always', + outputFolder: path.join( + process.env.WP_ARTIFACTS_PATH, + 'visual-report' + ), + }, + ], + [ './reporters/impact-summary.js' ], +]; + const config = defineConfig( { ...baseConfig, globalSetup: undefined, + reporter, webServer: { ...baseConfig.webServer, command: 'npm run env:start', diff --git a/tests/visual-regression/reporters/impact-summary.js b/tests/visual-regression/reporters/impact-summary.js new file mode 100644 index 0000000000000..6163007daf8a6 --- /dev/null +++ b/tests/visual-regression/reporters/impact-summary.js @@ -0,0 +1,61 @@ +/** + * Impact Summary Reporter + * + * A custom Playwright reporter that prints a visual impact summary + * at the end of the test run, showing which pages changed and which + * remained unchanged. + */ +class ImpactSummaryReporter { + constructor() { + this.changed = []; + this.unchanged = []; + } + + onTestEnd( test, result ) { + const name = test.title; + + if ( result.status === 'passed' ) { + this.unchanged.push( name ); + } else { + this.changed.push( name ); + } + } + + onEnd() { + const total = this.changed.length + this.unchanged.length; + + if ( total === 0 ) { + return; + } + + const separator = '─'.repeat( 48 ); + + console.log( '' ); + console.log( separator ); + console.log( 'Visual Impact Summary' ); + console.log( separator ); + console.log( + `Changed: ${ this.changed.length } of ${ total } pages` + ); + console.log( + `Unchanged: ${ this.unchanged.length } of ${ total } pages` + ); + + if ( this.changed.length > 0 ) { + console.log( '' ); + console.log( 'Changed:' ); + console.log( ` ${ this.changed.join( ', ' ) }` ); + } + + if ( this.unchanged.length > 0 ) { + console.log( '' ); + console.log( 'Unchanged:' ); + console.log( ` ${ this.unchanged.join( ', ' ) }` ); + } + + console.log( separator ); + console.log( '' ); + } +} + +module.exports = ImpactSummaryReporter; diff --git a/tests/visual-regression/specs/visual-snapshots.test.js b/tests/visual-regression/specs/visual-snapshots.test.js index 048a6bc0b47cb..85e97d42cad9d 100644 --- a/tests/visual-regression/specs/visual-snapshots.test.js +++ b/tests/visual-regression/specs/visual-snapshots.test.js @@ -163,4 +163,102 @@ test.describe( 'Admin Visual Snapshots', () => { mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), }); } ); + + test( 'Dashboard', async ({ admin, page }) => { + await admin.visitAdminPage( '/index.php' ); + await expect( page ).toHaveScreenshot( 'Dashboard.png', { + mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), + }); + } ); + + test( 'Themes', async ({ admin, page }) => { + await admin.visitAdminPage( '/themes.php' ); + await expect( page ).toHaveScreenshot( 'Themes.png', { + mask: [ + ...elementsToHide, + '.theme-screenshot img', + ].map( ( selector ) => page.locator( selector ) ), + }); + } ); + + test( 'General Settings', async ({ admin, page }) => { + await admin.visitAdminPage( '/options-general.php' ); + await expect( page ).toHaveScreenshot( 'General Settings.png', { + mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), + }); + } ); + + test( 'Writing Settings', async ({ admin, page }) => { + await admin.visitAdminPage( '/options-writing.php' ); + await expect( page ).toHaveScreenshot( 'Writing Settings.png', { + mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), + }); + } ); + + test( 'Permalink Settings', async ({ admin, page }) => { + await admin.visitAdminPage( '/options-permalink.php' ); + await expect( page ).toHaveScreenshot( 'Permalink Settings.png', { + mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), + }); + } ); + + test( 'Add New Post', async ({ admin, page }) => { + await admin.visitAdminPage( '/post-new.php' ); + await expect( page ).toHaveScreenshot( 'Add New Post.png', { + mask: [ + ...elementsToHide, + '#wp-content-editor-container', + ].map( ( selector ) => page.locator( selector ) ), + }); + } ); + + test( 'Edit Post', async ({ admin, page, requestUtils }) => { + const post = await requestUtils.rest( { + method: 'POST', + path: '/wp/v2/posts', + data: { + title: 'Visual Regression Test Post', + content: 'Test content for visual regression.', + status: 'publish', + }, + } ); + + await admin.visitAdminPage( '/post.php', `post=${ post.id }&action=edit` ); + await expect( page ).toHaveScreenshot( 'Edit Post.png', { + mask: [ + ...elementsToHide, + '#wp-content-editor-container', + ].map( ( selector ) => page.locator( selector ) ), + }); + } ); + + test( 'Site Health', async ({ admin, page }) => { + await admin.visitAdminPage( '/site-health.php' ); + await expect( page ).toHaveScreenshot( 'Site Health.png', { + mask: [ + ...elementsToHide, + '.site-health-issues .health-check-accordion', + ].map( ( selector ) => page.locator( selector ) ), + }); + } ); + + test( 'Updates', async ({ admin, page }) => { + await admin.visitAdminPage( '/update-core.php' ); + await expect( page ).toHaveScreenshot( 'Updates.png', { + mask: [ + ...elementsToHide, + 'form.upgrade', + '.last-checked', + ].map( ( selector ) => page.locator( selector ) ), + }); + } ); +} ); + +test.describe( 'Unauthenticated Visual Snapshots', () => { + test.use( { storageState: {} } ); + + test( 'Login', async ({ page }) => { + await page.goto( '/wp-login.php' ); + await expect( page ).toHaveScreenshot( 'Login.png' ); + } ); } );