From 43bf5a98781112926763ff014517831160451ab8 Mon Sep 17 00:00:00 2001 From: Joe Fusco Date: Thu, 12 Feb 2026 11:38:46 -0500 Subject: [PATCH 1/5] Tests: Add visual regression tests for admin reskin pages. Add 7 new visual regression snapshot tests covering admin pages most impacted by the CSS reskin that previously had no automated visual coverage: - Dashboard (/index.php) - Themes (/themes.php) - General Settings (/options-general.php) - Writing Settings (/options-writing.php) - Permalink Settings (/options-permalink.php) - Add New Post (/post-new.php) - Edit Post (/post.php?post=&action=edit) Theme screenshots and editor content areas are masked to avoid false positives from dynamic content. See #64308. --- .../specs/visual-snapshots.test.js | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/tests/visual-regression/specs/visual-snapshots.test.js b/tests/visual-regression/specs/visual-snapshots.test.js index 048a6bc0b47cb..553ebb9ddc87a 100644 --- a/tests/visual-regression/specs/visual-snapshots.test.js +++ b/tests/visual-regression/specs/visual-snapshots.test.js @@ -163,4 +163,72 @@ test.describe( 'Admin Visual Snapshots', () => { mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), }); } ); + + test( 'Dashboard', async ({ admin, page }) => { + await admin.visitAdminPage( '/index.php' ); + await expect( page ).toHaveScreenshot( 'Dashboard.png', { + mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), + }); + } ); + + test( 'Themes', async ({ admin, page }) => { + await admin.visitAdminPage( '/themes.php' ); + await expect( page ).toHaveScreenshot( 'Themes.png', { + mask: [ + ...elementsToHide, + '.theme-screenshot img', + ].map( ( selector ) => page.locator( selector ) ), + }); + } ); + + test( 'General Settings', async ({ admin, page }) => { + await admin.visitAdminPage( '/options-general.php' ); + await expect( page ).toHaveScreenshot( 'General Settings.png', { + mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), + }); + } ); + + test( 'Writing Settings', async ({ admin, page }) => { + await admin.visitAdminPage( '/options-writing.php' ); + await expect( page ).toHaveScreenshot( 'Writing Settings.png', { + mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), + }); + } ); + + test( 'Permalink Settings', async ({ admin, page }) => { + await admin.visitAdminPage( '/options-permalink.php' ); + await expect( page ).toHaveScreenshot( 'Permalink Settings.png', { + mask: elementsToHide.map( ( selector ) => page.locator( selector ) ), + }); + } ); + + test( 'Add New Post', async ({ admin, page }) => { + await admin.visitAdminPage( '/post-new.php' ); + await expect( page ).toHaveScreenshot( 'Add New Post.png', { + mask: [ + ...elementsToHide, + '#wp-content-editor-container', + ].map( ( selector ) => page.locator( selector ) ), + }); + } ); + + test( 'Edit Post', async ({ admin, page, requestUtils }) => { + const post = await requestUtils.rest( { + method: 'POST', + path: '/wp/v2/posts', + data: { + title: 'Visual Regression Test Post', + content: 'Test content for visual regression.', + status: 'publish', + }, + } ); + + await admin.visitAdminPage( '/post.php', `post=${ post.id }&action=edit` ); + await expect( page ).toHaveScreenshot( 'Edit Post.png', { + mask: [ + ...elementsToHide, + '#wp-content-editor-container', + ].map( ( selector ) => page.locator( selector ) ), + }); + } ); } ); From 928a324cf8067449ccf33cf3cc9588f969516245 Mon Sep 17 00:00:00 2001 From: Joe Fusco Date: Thu, 12 Feb 2026 16:14:23 -0500 Subject: [PATCH 2/5] Add single-command visual regression workflow Add compare-branches.sh that automates the baseline generation and comparison workflow into a single npm run test:visual command. The script checks out trunk (or a specified base branch), generates baseline snapshots, switches back to the feature branch, and runs the comparison. --- package.json | 2 +- tests/visual-regression/README.md | 32 ++++++++++++++++++--- tests/visual-regression/compare-branches.sh | 25 ++++++++++++++++ 3 files changed, 54 insertions(+), 5 deletions(-) create mode 100755 tests/visual-regression/compare-branches.sh diff --git a/package.json b/package.json index 766e241ff8d6d..6755c2a020dba 100644 --- a/package.json +++ b/package.json @@ -129,7 +129,7 @@ "test:php": "node ./tools/local-env/scripts/docker.js run --rm php ./vendor/bin/phpunit", "test:coverage": "npm run test:php -- --coverage-html ./coverage/html/ --coverage-php ./coverage/php/report.php --coverage-text=./coverage/text/report.txt", "test:e2e": "wp-scripts test-playwright --config tests/e2e/playwright.config.js", - "test:visual": "wp-scripts test-playwright --config tests/visual-regression/playwright.config.js", + "test:visual": "bash tests/visual-regression/compare-branches.sh", "gutenberg:checkout": "node tools/gutenberg/checkout-gutenberg.js", "gutenberg:build": "node tools/gutenberg/build-gutenberg.js", "gutenberg:copy": "node tools/gutenberg/copy-gutenberg-build.js", diff --git a/tests/visual-regression/README.md b/tests/visual-regression/README.md index d7ef71e64324b..bbcae932fbfe8 100644 --- a/tests/visual-regression/README.md +++ b/tests/visual-regression/README.md @@ -4,8 +4,32 @@ These tests make use of Playwright, with a setup very similar to that of the e2e ## How to Run the Tests Locally -1. Check out trunk. -2. Run `npm run test:visual` to generate some base snapshots. -3. Check out the feature branch to be tested. -4. Run `npm run test:visual` again. If any tests fail, the diff images can be found in `artifacts/` +From a feature branch with a clean working tree, run: + +```bash +npm run test:visual +``` + +This will automatically: +1. Check out trunk to generate baseline snapshots. +2. Return to your feature branch. +3. Compare the current branch against those baselines. + +If any tests fail, diff images can be found in `artifacts/`. + +### Specifying a Base Branch +By default, baselines are generated from `trunk`. To compare against a different branch: + +```bash +npm run test:visual -- some-other-branch +``` + +### Manual 2-Step Workflow + +You can also generate baselines and compare manually: + +1. Check out the base branch (e.g. trunk). +2. Run `npx playwright test --config tests/visual-regression/playwright.config.js --update-snapshots` +3. Check out the feature branch to be tested. +4. Run `npx playwright test --config tests/visual-regression/playwright.config.js` diff --git a/tests/visual-regression/compare-branches.sh b/tests/visual-regression/compare-branches.sh new file mode 100755 index 0000000000000..c852aec333bcf --- /dev/null +++ b/tests/visual-regression/compare-branches.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -e + +CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) +BASELINE_BRANCH="${1:-trunk}" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CONFIG="$SCRIPT_DIR/playwright.config.js" + +if [ "$CURRENT_BRANCH" = "$BASELINE_BRANCH" ]; then + echo "Error: Already on $BASELINE_BRANCH. Checkout a feature branch first." + exit 1 +fi + +if ! git diff-index --quiet HEAD --; then + echo "Error: Uncommitted changes detected. Please commit or stash before running." + exit 1 +fi + +echo "Generating baselines from $BASELINE_BRANCH..." +git checkout "$BASELINE_BRANCH" +npx playwright test --config "$CONFIG" --update-snapshots + +echo "Comparing against $CURRENT_BRANCH..." +git checkout "$CURRENT_BRANCH" +npx playwright test --config "$CONFIG" From 1a4e1e9d966f7c65fd6840d3548d0ca864a4a8a9 Mon Sep 17 00:00:00 2001 From: Joe Fusco Date: Thu, 12 Feb 2026 16:30:43 -0500 Subject: [PATCH 3/5] Tests: Add reporter configuration to visual regression config Configure list and HTML reporters for the visual regression Playwright config. The HTML report opens automatically on failure and outputs to the artifacts directory. --- tests/visual-regression/playwright.config.js | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/visual-regression/playwright.config.js b/tests/visual-regression/playwright.config.js index 759d887bf71c2..2f0289797e971 100644 --- a/tests/visual-regression/playwright.config.js +++ b/tests/visual-regression/playwright.config.js @@ -15,9 +15,24 @@ process.env.STORAGE_STATE_PATH ??= path.join( 'storage-states/admin.json' ); +const reporter = [ + [ 'list' ], + [ + 'html', + { + open: 'on-failure', + outputFolder: path.join( + process.env.WP_ARTIFACTS_PATH, + 'visual-report' + ), + }, + ], +]; + const config = defineConfig( { ...baseConfig, globalSetup: undefined, + reporter, webServer: { ...baseConfig.webServer, command: 'npm run env:start', From 656f69894fd71ae8c21d1988e909e12cb1fdb170 Mon Sep 17 00:00:00 2001 From: Joe Fusco Date: Fri, 13 Feb 2026 10:58:16 -0500 Subject: [PATCH 4/5] Tests: Improve visual regression workflow for admin reskin iteration Add impact summary reporter, fast re-run mode, and auto-open HTML report to streamline CSS iteration during the admin reskin effort. --- package.json | 1 + tests/visual-regression/README.md | 48 ++++++++++++++- tests/visual-regression/compare-branches.sh | 55 +++++++++++++---- tests/visual-regression/playwright.config.js | 3 +- .../reporters/impact-summary.js | 61 +++++++++++++++++++ 5 files changed, 155 insertions(+), 13 deletions(-) create mode 100644 tests/visual-regression/reporters/impact-summary.js diff --git a/package.json b/package.json index 6755c2a020dba..8d4e23b334340 100644 --- a/package.json +++ b/package.json @@ -130,6 +130,7 @@ "test:coverage": "npm run test:php -- --coverage-html ./coverage/html/ --coverage-php ./coverage/php/report.php --coverage-text=./coverage/text/report.txt", "test:e2e": "wp-scripts test-playwright --config tests/e2e/playwright.config.js", "test:visual": "bash tests/visual-regression/compare-branches.sh", + "test:visual:quick": "bash tests/visual-regression/compare-branches.sh --skip-baselines", "gutenberg:checkout": "node tools/gutenberg/checkout-gutenberg.js", "gutenberg:build": "node tools/gutenberg/build-gutenberg.js", "gutenberg:copy": "node tools/gutenberg/copy-gutenberg-build.js", diff --git a/tests/visual-regression/README.md b/tests/visual-regression/README.md index bbcae932fbfe8..0d54abd4f88b6 100644 --- a/tests/visual-regression/README.md +++ b/tests/visual-regression/README.md @@ -2,6 +2,12 @@ These tests make use of Playwright, with a setup very similar to that of the e2e tests. +## Prerequisites + +- Node.js >= 20.10.0 +- A running WordPress environment (`npm run env:start`) +- Playwright browsers installed (`npx playwright install chromium`) + ## How to Run the Tests Locally From a feature branch with a clean working tree, run: @@ -14,8 +20,27 @@ This will automatically: 1. Check out trunk to generate baseline snapshots. 2. Return to your feature branch. 3. Compare the current branch against those baselines. +4. Print a visual impact summary in the terminal. +5. Open the HTML report with side-by-side visual diffs. + +### Quick Re-run (Skip Baseline Generation) -If any tests fail, diff images can be found in `artifacts/`. +After the first run, baselines from trunk are already saved. To re-compare without regenerating them: + +```bash +npm run test:visual:quick +``` + +This is useful when iterating on CSS — no need to commit changes or wait for trunk baselines to regenerate each time. + +**Typical workflow:** +```bash +npm run test:visual # First run: generates baselines from trunk +# ... tweak CSS ... +npm run test:visual:quick # Fast: reuses existing baselines +# ... tweak more CSS ... +npm run test:visual:quick # Fast again +``` ### Specifying a Base Branch @@ -33,3 +58,24 @@ You can also generate baselines and compare manually: 2. Run `npx playwright test --config tests/visual-regression/playwright.config.js --update-snapshots` 3. Check out the feature branch to be tested. 4. Run `npx playwright test --config tests/visual-regression/playwright.config.js` + +## Impact Summary + +After each run, a summary is printed to the terminal showing which pages changed and which remained unchanged: + +``` +──────────────────────────────────────────────── +Visual Impact Summary +──────────────────────────────────────────────── +Changed: 27 of 35 pages +Unchanged: 8 of 35 pages + +Changed: + Dashboard, All Posts, Categories, Tags, ... + +Unchanged: + Login, Media Settings, ... +──────────────────────────────────────────────── +``` + +The HTML report also opens automatically with side-by-side visual diffs for detailed inspection. diff --git a/tests/visual-regression/compare-branches.sh b/tests/visual-regression/compare-branches.sh index c852aec333bcf..aa51ab1213dcb 100755 --- a/tests/visual-regression/compare-branches.sh +++ b/tests/visual-regression/compare-branches.sh @@ -1,8 +1,22 @@ #!/bin/bash set -e +SKIP_BASELINES=false +BASELINE_BRANCH="trunk" + +# Parse arguments. +for arg in "$@"; do + case "$arg" in + --skip-baselines) + SKIP_BASELINES=true + ;; + *) + BASELINE_BRANCH="$arg" + ;; + esac +done + CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) -BASELINE_BRANCH="${1:-trunk}" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" CONFIG="$SCRIPT_DIR/playwright.config.js" @@ -11,15 +25,34 @@ if [ "$CURRENT_BRANCH" = "$BASELINE_BRANCH" ]; then exit 1 fi -if ! git diff-index --quiet HEAD --; then - echo "Error: Uncommitted changes detected. Please commit or stash before running." - exit 1 -fi +if [ "$SKIP_BASELINES" = true ]; then + # Verify baselines exist before skipping regeneration. + SNAPSHOT_DIR="$SCRIPT_DIR/specs/__snapshots__" + if [ ! -d "$SNAPSHOT_DIR" ] || [ -z "$(ls -A "$SNAPSHOT_DIR" 2>/dev/null)" ]; then + echo "Error: No baselines found. Run 'npm run test:visual' first to generate them." + exit 1 + fi -echo "Generating baselines from $BASELINE_BRANCH..." -git checkout "$BASELINE_BRANCH" -npx playwright test --config "$CONFIG" --update-snapshots + echo "Skipping baseline generation (reusing existing baselines)..." + echo "Ensuring Playwright browsers are installed..." + npx playwright install --with-deps chromium -echo "Comparing against $CURRENT_BRANCH..." -git checkout "$CURRENT_BRANCH" -npx playwright test --config "$CONFIG" + echo "Comparing against existing baselines..." + npx playwright test --config "$CONFIG" || true +else + if ! git diff-index --quiet HEAD --; then + echo "Error: Uncommitted changes detected. Please commit or stash before running." + exit 1 + fi + + echo "Ensuring Playwright browsers are installed..." + npx playwright install --with-deps chromium + + echo "Generating baselines from $BASELINE_BRANCH..." + git checkout "$BASELINE_BRANCH" + npx playwright test --config "$CONFIG" --update-snapshots + + echo "Comparing against $CURRENT_BRANCH..." + git checkout "$CURRENT_BRANCH" + npx playwright test --config "$CONFIG" || true +fi diff --git a/tests/visual-regression/playwright.config.js b/tests/visual-regression/playwright.config.js index 2f0289797e971..af4fcf9d1ee56 100644 --- a/tests/visual-regression/playwright.config.js +++ b/tests/visual-regression/playwright.config.js @@ -20,13 +20,14 @@ const reporter = [ [ 'html', { - open: 'on-failure', + open: process.env.CI ? 'never' : 'always', outputFolder: path.join( process.env.WP_ARTIFACTS_PATH, 'visual-report' ), }, ], + [ './reporters/impact-summary.js' ], ]; const config = defineConfig( { diff --git a/tests/visual-regression/reporters/impact-summary.js b/tests/visual-regression/reporters/impact-summary.js new file mode 100644 index 0000000000000..6163007daf8a6 --- /dev/null +++ b/tests/visual-regression/reporters/impact-summary.js @@ -0,0 +1,61 @@ +/** + * Impact Summary Reporter + * + * A custom Playwright reporter that prints a visual impact summary + * at the end of the test run, showing which pages changed and which + * remained unchanged. + */ +class ImpactSummaryReporter { + constructor() { + this.changed = []; + this.unchanged = []; + } + + onTestEnd( test, result ) { + const name = test.title; + + if ( result.status === 'passed' ) { + this.unchanged.push( name ); + } else { + this.changed.push( name ); + } + } + + onEnd() { + const total = this.changed.length + this.unchanged.length; + + if ( total === 0 ) { + return; + } + + const separator = '─'.repeat( 48 ); + + console.log( '' ); + console.log( separator ); + console.log( 'Visual Impact Summary' ); + console.log( separator ); + console.log( + `Changed: ${ this.changed.length } of ${ total } pages` + ); + console.log( + `Unchanged: ${ this.unchanged.length } of ${ total } pages` + ); + + if ( this.changed.length > 0 ) { + console.log( '' ); + console.log( 'Changed:' ); + console.log( ` ${ this.changed.join( ', ' ) }` ); + } + + if ( this.unchanged.length > 0 ) { + console.log( '' ); + console.log( 'Unchanged:' ); + console.log( ` ${ this.unchanged.join( ', ' ) }` ); + } + + console.log( separator ); + console.log( '' ); + } +} + +module.exports = ImpactSummaryReporter; From 2bf69cb6315f716bdcd292452db17da97e1414ad Mon Sep 17 00:00:00 2001 From: Joe Fusco Date: Fri, 13 Feb 2026 10:58:16 -0500 Subject: [PATCH 5/5] Tests: Improve visual regression workflow for admin reskin iteration Add impact summary reporter, fast re-run mode, and auto-open HTML report to streamline CSS iteration during the admin reskin effort. --- package.json | 1 + tests/visual-regression/README.md | 48 ++++++++++++++- tests/visual-regression/compare-branches.sh | 55 +++++++++++++---- tests/visual-regression/playwright.config.js | 3 +- .../reporters/impact-summary.js | 61 +++++++++++++++++++ .../specs/visual-snapshots.test.js | 30 +++++++++ 6 files changed, 185 insertions(+), 13 deletions(-) create mode 100644 tests/visual-regression/reporters/impact-summary.js diff --git a/package.json b/package.json index 6755c2a020dba..8d4e23b334340 100644 --- a/package.json +++ b/package.json @@ -130,6 +130,7 @@ "test:coverage": "npm run test:php -- --coverage-html ./coverage/html/ --coverage-php ./coverage/php/report.php --coverage-text=./coverage/text/report.txt", "test:e2e": "wp-scripts test-playwright --config tests/e2e/playwright.config.js", "test:visual": "bash tests/visual-regression/compare-branches.sh", + "test:visual:quick": "bash tests/visual-regression/compare-branches.sh --skip-baselines", "gutenberg:checkout": "node tools/gutenberg/checkout-gutenberg.js", "gutenberg:build": "node tools/gutenberg/build-gutenberg.js", "gutenberg:copy": "node tools/gutenberg/copy-gutenberg-build.js", diff --git a/tests/visual-regression/README.md b/tests/visual-regression/README.md index bbcae932fbfe8..0d54abd4f88b6 100644 --- a/tests/visual-regression/README.md +++ b/tests/visual-regression/README.md @@ -2,6 +2,12 @@ These tests make use of Playwright, with a setup very similar to that of the e2e tests. +## Prerequisites + +- Node.js >= 20.10.0 +- A running WordPress environment (`npm run env:start`) +- Playwright browsers installed (`npx playwright install chromium`) + ## How to Run the Tests Locally From a feature branch with a clean working tree, run: @@ -14,8 +20,27 @@ This will automatically: 1. Check out trunk to generate baseline snapshots. 2. Return to your feature branch. 3. Compare the current branch against those baselines. +4. Print a visual impact summary in the terminal. +5. Open the HTML report with side-by-side visual diffs. + +### Quick Re-run (Skip Baseline Generation) -If any tests fail, diff images can be found in `artifacts/`. +After the first run, baselines from trunk are already saved. To re-compare without regenerating them: + +```bash +npm run test:visual:quick +``` + +This is useful when iterating on CSS — no need to commit changes or wait for trunk baselines to regenerate each time. + +**Typical workflow:** +```bash +npm run test:visual # First run: generates baselines from trunk +# ... tweak CSS ... +npm run test:visual:quick # Fast: reuses existing baselines +# ... tweak more CSS ... +npm run test:visual:quick # Fast again +``` ### Specifying a Base Branch @@ -33,3 +58,24 @@ You can also generate baselines and compare manually: 2. Run `npx playwright test --config tests/visual-regression/playwright.config.js --update-snapshots` 3. Check out the feature branch to be tested. 4. Run `npx playwright test --config tests/visual-regression/playwright.config.js` + +## Impact Summary + +After each run, a summary is printed to the terminal showing which pages changed and which remained unchanged: + +``` +──────────────────────────────────────────────── +Visual Impact Summary +──────────────────────────────────────────────── +Changed: 27 of 35 pages +Unchanged: 8 of 35 pages + +Changed: + Dashboard, All Posts, Categories, Tags, ... + +Unchanged: + Login, Media Settings, ... +──────────────────────────────────────────────── +``` + +The HTML report also opens automatically with side-by-side visual diffs for detailed inspection. diff --git a/tests/visual-regression/compare-branches.sh b/tests/visual-regression/compare-branches.sh index c852aec333bcf..aa51ab1213dcb 100755 --- a/tests/visual-regression/compare-branches.sh +++ b/tests/visual-regression/compare-branches.sh @@ -1,8 +1,22 @@ #!/bin/bash set -e +SKIP_BASELINES=false +BASELINE_BRANCH="trunk" + +# Parse arguments. +for arg in "$@"; do + case "$arg" in + --skip-baselines) + SKIP_BASELINES=true + ;; + *) + BASELINE_BRANCH="$arg" + ;; + esac +done + CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) -BASELINE_BRANCH="${1:-trunk}" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" CONFIG="$SCRIPT_DIR/playwright.config.js" @@ -11,15 +25,34 @@ if [ "$CURRENT_BRANCH" = "$BASELINE_BRANCH" ]; then exit 1 fi -if ! git diff-index --quiet HEAD --; then - echo "Error: Uncommitted changes detected. Please commit or stash before running." - exit 1 -fi +if [ "$SKIP_BASELINES" = true ]; then + # Verify baselines exist before skipping regeneration. + SNAPSHOT_DIR="$SCRIPT_DIR/specs/__snapshots__" + if [ ! -d "$SNAPSHOT_DIR" ] || [ -z "$(ls -A "$SNAPSHOT_DIR" 2>/dev/null)" ]; then + echo "Error: No baselines found. Run 'npm run test:visual' first to generate them." + exit 1 + fi -echo "Generating baselines from $BASELINE_BRANCH..." -git checkout "$BASELINE_BRANCH" -npx playwright test --config "$CONFIG" --update-snapshots + echo "Skipping baseline generation (reusing existing baselines)..." + echo "Ensuring Playwright browsers are installed..." + npx playwright install --with-deps chromium -echo "Comparing against $CURRENT_BRANCH..." -git checkout "$CURRENT_BRANCH" -npx playwright test --config "$CONFIG" + echo "Comparing against existing baselines..." + npx playwright test --config "$CONFIG" || true +else + if ! git diff-index --quiet HEAD --; then + echo "Error: Uncommitted changes detected. Please commit or stash before running." + exit 1 + fi + + echo "Ensuring Playwright browsers are installed..." + npx playwright install --with-deps chromium + + echo "Generating baselines from $BASELINE_BRANCH..." + git checkout "$BASELINE_BRANCH" + npx playwright test --config "$CONFIG" --update-snapshots + + echo "Comparing against $CURRENT_BRANCH..." + git checkout "$CURRENT_BRANCH" + npx playwright test --config "$CONFIG" || true +fi diff --git a/tests/visual-regression/playwright.config.js b/tests/visual-regression/playwright.config.js index 2f0289797e971..af4fcf9d1ee56 100644 --- a/tests/visual-regression/playwright.config.js +++ b/tests/visual-regression/playwright.config.js @@ -20,13 +20,14 @@ const reporter = [ [ 'html', { - open: 'on-failure', + open: process.env.CI ? 'never' : 'always', outputFolder: path.join( process.env.WP_ARTIFACTS_PATH, 'visual-report' ), }, ], + [ './reporters/impact-summary.js' ], ]; const config = defineConfig( { diff --git a/tests/visual-regression/reporters/impact-summary.js b/tests/visual-regression/reporters/impact-summary.js new file mode 100644 index 0000000000000..6163007daf8a6 --- /dev/null +++ b/tests/visual-regression/reporters/impact-summary.js @@ -0,0 +1,61 @@ +/** + * Impact Summary Reporter + * + * A custom Playwright reporter that prints a visual impact summary + * at the end of the test run, showing which pages changed and which + * remained unchanged. + */ +class ImpactSummaryReporter { + constructor() { + this.changed = []; + this.unchanged = []; + } + + onTestEnd( test, result ) { + const name = test.title; + + if ( result.status === 'passed' ) { + this.unchanged.push( name ); + } else { + this.changed.push( name ); + } + } + + onEnd() { + const total = this.changed.length + this.unchanged.length; + + if ( total === 0 ) { + return; + } + + const separator = '─'.repeat( 48 ); + + console.log( '' ); + console.log( separator ); + console.log( 'Visual Impact Summary' ); + console.log( separator ); + console.log( + `Changed: ${ this.changed.length } of ${ total } pages` + ); + console.log( + `Unchanged: ${ this.unchanged.length } of ${ total } pages` + ); + + if ( this.changed.length > 0 ) { + console.log( '' ); + console.log( 'Changed:' ); + console.log( ` ${ this.changed.join( ', ' ) }` ); + } + + if ( this.unchanged.length > 0 ) { + console.log( '' ); + console.log( 'Unchanged:' ); + console.log( ` ${ this.unchanged.join( ', ' ) }` ); + } + + console.log( separator ); + console.log( '' ); + } +} + +module.exports = ImpactSummaryReporter; diff --git a/tests/visual-regression/specs/visual-snapshots.test.js b/tests/visual-regression/specs/visual-snapshots.test.js index 553ebb9ddc87a..85e97d42cad9d 100644 --- a/tests/visual-regression/specs/visual-snapshots.test.js +++ b/tests/visual-regression/specs/visual-snapshots.test.js @@ -231,4 +231,34 @@ test.describe( 'Admin Visual Snapshots', () => { ].map( ( selector ) => page.locator( selector ) ), }); } ); + + test( 'Site Health', async ({ admin, page }) => { + await admin.visitAdminPage( '/site-health.php' ); + await expect( page ).toHaveScreenshot( 'Site Health.png', { + mask: [ + ...elementsToHide, + '.site-health-issues .health-check-accordion', + ].map( ( selector ) => page.locator( selector ) ), + }); + } ); + + test( 'Updates', async ({ admin, page }) => { + await admin.visitAdminPage( '/update-core.php' ); + await expect( page ).toHaveScreenshot( 'Updates.png', { + mask: [ + ...elementsToHide, + 'form.upgrade', + '.last-checked', + ].map( ( selector ) => page.locator( selector ) ), + }); + } ); +} ); + +test.describe( 'Unauthenticated Visual Snapshots', () => { + test.use( { storageState: {} } ); + + test( 'Login', async ({ page }) => { + await page.goto( '/wp-login.php' ); + await expect( page ).toHaveScreenshot( 'Login.png' ); + } ); } );