Tests: Add visual regression tests for admin reskin pages

package.json

@@ -129,7 +129,8 @@
 		"test:php": "node ./tools/local-env/scripts/docker.js run --rm php ./vendor/bin/phpunit",
 		"test:coverage": "npm run test:php -- --coverage-html ./coverage/html/ --coverage-php ./coverage/php/report.php --coverage-text=./coverage/text/report.txt",
 		"test:e2e": "wp-scripts test-playwright --config tests/e2e/playwright.config.js",
-		"test:visual": "wp-scripts test-playwright --config tests/visual-regression/playwright.config.js",
+		"test:visual": "bash tests/visual-regression/compare-branches.sh",
+		"test:visual:quick": "bash tests/visual-regression/compare-branches.sh --skip-baselines",
 		"gutenberg:checkout": "node tools/gutenberg/checkout-gutenberg.js",
 		"gutenberg:build": "node tools/gutenberg/build-gutenberg.js",
 		"gutenberg:copy": "node tools/gutenberg/copy-gutenberg-build.js",

tests/visual-regression/README.md

@@ -2,10 +2,80 @@
 
 These tests make use of Playwright, with a setup very similar to that of the e2e tests.
 
+## Prerequisites
+
+- Node.js >= 20.10.0
+- A running WordPress environment (`npm run env:start`)
+- Playwright browsers installed (`npx playwright install chromium`)
+
 ## How to Run the Tests Locally
 
-1. Check out trunk.
-2. Run `npm run test:visual` to generate some base snapshots.
+From a feature branch with a clean working tree, run:
+
+```bash
+npm run test:visual
+```
+
+This will automatically:
+1. Check out trunk to generate baseline snapshots.
+2. Return to your feature branch.
+3. Compare the current branch against those baselines.
+4. Print a visual impact summary in the terminal.
+5. Open the HTML report with side-by-side visual diffs.
+
+### Quick Re-run (Skip Baseline Generation)
+
+After the first run, baselines from trunk are already saved. To re-compare without regenerating them:
+
+```bash
+npm run test:visual:quick
+```
+
+This is useful when iterating on CSS — no need to commit changes or wait for trunk baselines to regenerate each time.
+
+**Typical workflow:**
+```bash
+npm run test:visual          # First run: generates baselines from trunk
+# ... tweak CSS ...
+npm run test:visual:quick    # Fast: reuses existing baselines
+# ... tweak more CSS ...
+npm run test:visual:quick    # Fast again
+```
+
+### Specifying a Base Branch
+
+By default, baselines are generated from `trunk`. To compare against a different branch:
+
+```bash
+npm run test:visual -- some-other-branch
+```
+
+### Manual 2-Step Workflow
+
+You can also generate baselines and compare manually:
+
+1. Check out the base branch (e.g. trunk).
+2. Run `npx playwright test --config tests/visual-regression/playwright.config.js --update-snapshots`
 3. Check out the feature branch to be tested.
-4. Run `npm run test:visual` again. If any tests fail, the diff images can be found in `artifacts/`
+4. Run `npx playwright test --config tests/visual-regression/playwright.config.js`
+
+## Impact Summary
+
+After each run, a summary is printed to the terminal showing which pages changed and which remained unchanged:
+
+```
+────────────────────────────────────────────────
+Visual Impact Summary
+────────────────────────────────────────────────
+Changed:   27 of 35 pages
+Unchanged: 8 of 35 pages
+
+Changed:
+  Dashboard, All Posts, Categories, Tags, ...
+
+Unchanged:
+  Login, Media Settings, ...
+────────────────────────────────────────────────
+```
 
+The HTML report also opens automatically with side-by-side visual diffs for detailed inspection.

tests/visual-regression/compare-branches.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+set -e
+
+SKIP_BASELINES=false
+BASELINE_BRANCH="trunk"
+
+# Parse arguments.
+for arg in "$@"; do
+	case "$arg" in
+		--skip-baselines)
+			SKIP_BASELINES=true
+			;;
+		*)
+			BASELINE_BRANCH="$arg"
+			;;
+	esac
+done
+
+CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+CONFIG="$SCRIPT_DIR/playwright.config.js"
+
+if [ "$CURRENT_BRANCH" = "$BASELINE_BRANCH" ]; then
+	echo "Error: Already on $BASELINE_BRANCH. Checkout a feature branch first."
+	exit 1
+fi
+
+if [ "$SKIP_BASELINES" = true ]; then
+	# Verify baselines exist before skipping regeneration.
+	SNAPSHOT_DIR="$SCRIPT_DIR/specs/__snapshots__"
+	if [ ! -d "$SNAPSHOT_DIR" ] || [ -z "$(ls -A "$SNAPSHOT_DIR" 2>/dev/null)" ]; then
+		echo "Error: No baselines found. Run 'npm run test:visual' first to generate them."
+		exit 1
+	fi
+
+	echo "Skipping baseline generation (reusing existing baselines)..."
+	echo "Ensuring Playwright browsers are installed..."
+	npx playwright install --with-deps chromium
+
+	echo "Comparing against existing baselines..."
+	npx playwright test --config "$CONFIG" || true
+else
+	if ! git diff-index --quiet HEAD --; then
+		echo "Error: Uncommitted changes detected. Please commit or stash before running."
+		exit 1
+	fi
+
+	echo "Ensuring Playwright browsers are installed..."
+	npx playwright install --with-deps chromium
+
+	echo "Generating baselines from $BASELINE_BRANCH..."
+	git checkout "$BASELINE_BRANCH"
+	npx playwright test --config "$CONFIG" --update-snapshots
+
+	echo "Comparing against $CURRENT_BRANCH..."
+	git checkout "$CURRENT_BRANCH"
+	npx playwright test --config "$CONFIG" || true
+fi

tests/visual-regression/playwright.config.js

@@ -15,9 +15,25 @@ process.env.STORAGE_STATE_PATH ??= path.join(
 	'storage-states/admin.json'
 );
 
+const reporter = [
+	[ 'list' ],
+	[
+		'html',
+		{
+			open: process.env.CI ? 'never' : 'always',
+			outputFolder: path.join(
+				process.env.WP_ARTIFACTS_PATH,
+				'visual-report'
+			),
+		},
+	],
+	[ './reporters/impact-summary.js' ],
+];
+
 const config = defineConfig( {
 	...baseConfig,
 	globalSetup: undefined,
+	reporter,
 	webServer: {
 		...baseConfig.webServer,
 		command: 'npm run env:start',

tests/visual-regression/reporters/impact-summary.js

@@ -0,0 +1,61 @@
+/**
+ * Impact Summary Reporter
+ *
+ * A custom Playwright reporter that prints a visual impact summary
+ * at the end of the test run, showing which pages changed and which
+ * remained unchanged.
+ */
+class ImpactSummaryReporter {
+	constructor() {
+		this.changed = [];
+		this.unchanged = [];
+	}
+
+	onTestEnd( test, result ) {
+		const name = test.title;
+
+		if ( result.status === 'passed' ) {
+			this.unchanged.push( name );
+		} else {
+			this.changed.push( name );
+		}
+	}
+
+	onEnd() {
+		const total = this.changed.length + this.unchanged.length;
+
+		if ( total === 0 ) {
+			return;
+		}
+
+		const separator = '─'.repeat( 48 );
+
+		console.log( '' );
+		console.log( separator );
+		console.log( 'Visual Impact Summary' );
+		console.log( separator );
+		console.log(
+			`Changed:   ${ this.changed.length } of ${ total } pages`
+		);
+		console.log(
+			`Unchanged: ${ this.unchanged.length } of ${ total } pages`
+		);
+
+		if ( this.changed.length > 0 ) {
+			console.log( '' );
+			console.log( 'Changed:' );
+			console.log( `  ${ this.changed.join( ', ' ) }` );
+		}
+
+		if ( this.unchanged.length > 0 ) {
+			console.log( '' );
+			console.log( 'Unchanged:' );
+			console.log( `  ${ this.unchanged.join( ', ' ) }` );
+		}
+
+		console.log( separator );
+		console.log( '' );
+	}
+}
+
+module.exports = ImpactSummaryReporter;

tests/visual-regression/specs/visual-snapshots.test.js

@@ -163,4 +163,102 @@ test.describe( 'Admin Visual Snapshots', () => {
 			mask: elementsToHide.map( ( selector ) => page.locator( selector ) ),
 		});
 	} );
+
+	test( 'Dashboard', async ({ admin, page }) => {
+		await admin.visitAdminPage( '/index.php' );
+		await expect( page ).toHaveScreenshot( 'Dashboard.png', {
+			mask: elementsToHide.map( ( selector ) => page.locator( selector ) ),
+		});
+	} );
+
+	test( 'Themes', async ({ admin, page }) => {
+		await admin.visitAdminPage( '/themes.php' );
+		await expect( page ).toHaveScreenshot( 'Themes.png', {
+			mask: [
+				...elementsToHide,
+				'.theme-screenshot img',
+			].map( ( selector ) => page.locator( selector ) ),
+		});
+	} );
+
+	test( 'General Settings', async ({ admin, page }) => {
+		await admin.visitAdminPage( '/options-general.php' );
+		await expect( page ).toHaveScreenshot( 'General Settings.png', {
+			mask: elementsToHide.map( ( selector ) => page.locator( selector ) ),
+		});
+	} );
+
+	test( 'Writing Settings', async ({ admin, page }) => {
+		await admin.visitAdminPage( '/options-writing.php' );
+		await expect( page ).toHaveScreenshot( 'Writing Settings.png', {
+			mask: elementsToHide.map( ( selector ) => page.locator( selector ) ),
+		});
+	} );
+
+	test( 'Permalink Settings', async ({ admin, page }) => {
+		await admin.visitAdminPage( '/options-permalink.php' );
+		await expect( page ).toHaveScreenshot( 'Permalink Settings.png', {
+			mask: elementsToHide.map( ( selector ) => page.locator( selector ) ),
+		});
+	} );
+
+	test( 'Add New Post', async ({ admin, page }) => {
+		await admin.visitAdminPage( '/post-new.php' );
+		await expect( page ).toHaveScreenshot( 'Add New Post.png', {
+			mask: [
+				...elementsToHide,
+				'#wp-content-editor-container',
+			].map( ( selector ) => page.locator( selector ) ),
+		});
+	} );
+
+	test( 'Edit Post', async ({ admin, page, requestUtils }) => {
+		const post = await requestUtils.rest( {
+			method: 'POST',
+			path: '/wp/v2/posts',
+			data: {
+				title: 'Visual Regression Test Post',
+				content: 'Test content for visual regression.',
+				status: 'publish',
+			},
+		} );
+
+		await admin.visitAdminPage( '/post.php', `post=${ post.id }&action=edit` );
+		await expect( page ).toHaveScreenshot( 'Edit Post.png', {
+			mask: [
+				...elementsToHide,
+				'#wp-content-editor-container',
+			].map( ( selector ) => page.locator( selector ) ),
+		});
+	} );
+
+	test( 'Site Health', async ({ admin, page }) => {
+		await admin.visitAdminPage( '/site-health.php' );
+		await expect( page ).toHaveScreenshot( 'Site Health.png', {
+			mask: [
+				...elementsToHide,
+				'.site-health-issues .health-check-accordion',
+			].map( ( selector ) => page.locator( selector ) ),
+		});
+	} );
+
+	test( 'Updates', async ({ admin, page }) => {
+		await admin.visitAdminPage( '/update-core.php' );
+		await expect( page ).toHaveScreenshot( 'Updates.png', {
+			mask: [
+				...elementsToHide,
+				'form.upgrade',
+				'.last-checked',
+			].map( ( selector ) => page.locator( selector ) ),
+		});
+	} );
+} );
+
+test.describe( 'Unauthenticated Visual Snapshots', () => {
+	test.use( { storageState: {} } );
+
+	test( 'Login', async ({ page }) => {
+		await page.goto( '/wp-login.php' );
+		await expect( page ).toHaveScreenshot( 'Login.png' );
+	} );
 } );