test: separate acceptance tests onto Fast and Slow suites

.github/workflows/testing.yml

@@ -22,10 +22,18 @@ jobs:
       test-command: composer test:func
       download-binaries: true
 
-  acceptance:
-    name: Acceptance Testing
+  acceptance-slow:
+    name: Acceptance Testing (Slow)
     uses: ./.github/workflows/run-test-suite.yml
     with:
       fail-fast: false
-      test-command: composer test:accept
+      test-command: composer test:accept-slow
+      download-binaries: true
+
+  acceptance-fast:
+    name: Acceptance Testing (Fast)
+    uses: ./.github/workflows/run-test-suite.yml
+    with:
+      fail-fast: false
+      test-command: composer test:accept-fast
       download-binaries: true

.run/Acceptance Fast.run.xml

@@ -0,0 +1,6 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Acceptance Fast" type="PHPUnitRunConfigurationType" factoryName="PHPUnit">
+    <TestRunner bootstrap_file="$PROJECT_DIR$/tests/bootstrap.php" configuration_file="$PROJECT_DIR$/phpunit.xml.dist" directory="$PROJECT_DIR$/tests/Acceptance" scope="XML" options="--testsuite=&quot;Acceptance-Fast&quot; --log-junit=runtime/phpunit-acceptance-fast-junit.xml" />
+    <method v="2" />
+  </configuration>
+</component>

.run/Acceptance Slow.run.xml

@@ -0,0 +1,6 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Acceptance Slow" type="PHPUnitRunConfigurationType" factoryName="PHPUnit">
+    <TestRunner bootstrap_file="$PROJECT_DIR$/tests/bootstrap.php" configuration_file="$PROJECT_DIR$/phpunit.xml.dist" directory="$PROJECT_DIR$/tests/Acceptance" scope="XML" options="--testsuite=&quot;Acceptance-Slow&quot; --log-junit=runtime/phpunit-acceptance-slow-junit.xml" />
+    <method v="2" />
+  </configuration>
+</component>

.run/Acceptance.run.xml

@@ -1,6 +1,6 @@
 <component name="ProjectRunConfigurationManager">
   <configuration default="false" name="Acceptance" type="PHPUnitRunConfigurationType" factoryName="PHPUnit">
-    <TestRunner bootstrap_file="$PROJECT_DIR$/tests/bootstrap.php" configuration_file="$PROJECT_DIR$/phpunit.xml.dist" directory="$PROJECT_DIR$/tests/Acceptance" scope="XML" options="--testsuite=Acceptance" />
+    <TestRunner bootstrap_file="$PROJECT_DIR$/tests/bootstrap.php" configuration_file="$PROJECT_DIR$/phpunit.xml.dist" directory="$PROJECT_DIR$/tests/Acceptance" scope="XML" options="--testsuite=Acceptance --log-junit=runtime/phpunit-acceptance-junit.xml" />
     <method v="2" />
   </configuration>
-</component>
+</component>

composer.json

@@ -102,7 +102,9 @@
         "test:unit": "tests/runner.php vendor/bin/phpunit --testsuite=Unit --color=always --testdox",
         "test:func": "tests/runner.php vendor/bin/phpunit --testsuite=Functional --color=always --testdox",
         "test:arch": "phpunit --testsuite=Arch --color=always --testdox",
-        "test:accept": "tests/runner.php vendor/bin/phpunit --testsuite=Acceptance --color=always --testdox"
+        "test:accept": "tests/runner.php vendor/bin/phpunit --testsuite=Acceptance --color=always --testdox",
+        "test:accept-slow": "tests/runner.php vendor/bin/phpunit --testsuite=\"Acceptance-Slow\" --color=always --testdox",
+        "test:accept-fast": "tests/runner.php vendor/bin/phpunit --testsuite=\"Acceptance-Fast\" --color=always --testdox"
     },
     "config": {
         "sort-packages": true,

phpunit.xml.dist

@@ -14,9 +14,56 @@
          displayDetailsOnTestsThatTriggerDeprecations="true"
          displayDetailsOnTestsThatTriggerErrors="true"
          displayDetailsOnTestsThatTriggerNotices="true"
-         displayDetailsOnTestsThatTriggerWarnings="true"
->
+         displayDetailsOnTestsThatTriggerWarnings="true">
     <testsuites>
+        <testsuite name="Acceptance-Fast">
+            <directory suffix="Test.php">tests/Acceptance/Extra</directory>
+            <directory suffix="Test.php">tests/Acceptance/Harness</directory>
+            <exclude>tests/Acceptance/Harness/Update/SelfTest.php</exclude>
+            <exclude>tests/Acceptance/Harness/Query/TimeoutDueToNoActiveWorkersTest.php</exclude>
+            <exclude>tests/Acceptance/Extra/Activity/ActivityPausedTest.php</exclude>
+            <exclude>tests/Acceptance/Extra/Versioning/DeploymentTest.php</exclude>
+            <exclude>tests/Acceptance/Harness/EagerWorkflow/SuccessfulStartTest.php</exclude>
+            <exclude>tests/Acceptance/Harness/Activity/CancelTryCancelTest.php</exclude>
+            <exclude>tests/Acceptance/Extra/Stability/ResetWorkerTest.php</exclude>
+            <exclude>tests/Acceptance/Harness/Activity/RetryOnErrorTest.php</exclude>
+            <exclude>tests/Acceptance/Harness/Update/WorkerRestartTest.php</exclude>
+            <exclude>tests/Acceptance/Harness/Schedule/BasicTest.php</exclude>
+            <exclude>tests/Acceptance/Harness/Update/AsyncAcceptTest.php</exclude>
+            <exclude>tests/Acceptance/Extra/Workflow/BuiltInPrefixedHandlersTest.php</exclude>
+            <exclude>tests/Acceptance/Harness/Schedule/TriggerTest.php</exclude>
+            <exclude>tests/Acceptance/Extra/Workflow/InitMethodTest.php</exclude>
+            <exclude>tests/Acceptance/Harness/Signal/PreventCloseTest.php</exclude>
+            <exclude>tests/Acceptance/Extra/Update/UntypedStubTest.php</exclude>
+            <exclude>tests/Acceptance/Extra/Update/TimeoutTest.php</exclude>
+            <exclude>tests/Acceptance/Extra/Workflow/DateTimeZoneWorkflowTest.php</exclude>
+            <exclude>tests/Acceptance/Extra/Schedule/ScheduleUpdateTest.php</exclude>
+            <exclude>tests/Acceptance/Harness/ChildWorkflow/CancelAbandonTest.php</exclude>
+            <exclude>tests/Acceptance/Harness/ContinueAsNew/ContinueAsSameTest.php</exclude>
+        </testsuite>
+        <testsuite name="Acceptance-Slow">
+            <file>tests/Acceptance/Harness/Update/SelfTest.php</file>
+            <file>tests/Acceptance/Harness/Query/TimeoutDueToNoActiveWorkersTest.php</file>
+            <file>tests/Acceptance/Extra/Activity/ActivityPausedTest.php</file>
+            <file>tests/Acceptance/Extra/Versioning/DeploymentTest.php</file>
+            <file>tests/Acceptance/Harness/EagerWorkflow/SuccessfulStartTest.php</file>
+            <file>tests/Acceptance/Harness/Activity/CancelTryCancelTest.php</file>
+            <file>tests/Acceptance/Extra/Stability/ResetWorkerTest.php</file>
+            <file>tests/Acceptance/Harness/Activity/RetryOnErrorTest.php</file>
+            <file>tests/Acceptance/Harness/Update/WorkerRestartTest.php</file>
+            <file>tests/Acceptance/Harness/Schedule/BasicTest.php</file>
+            <file>tests/Acceptance/Harness/Update/AsyncAcceptTest.php</file>
+            <file>tests/Acceptance/Extra/Workflow/BuiltInPrefixedHandlersTest.php</file>
+            <file>tests/Acceptance/Harness/Schedule/TriggerTest.php</file>
+            <file>tests/Acceptance/Extra/Workflow/InitMethodTest.php</file>
+            <file>tests/Acceptance/Harness/Signal/PreventCloseTest.php</file>
+            <file>tests/Acceptance/Extra/Update/UntypedStubTest.php</file>
+            <file>tests/Acceptance/Extra/Update/TimeoutTest.php</file>
+            <file>tests/Acceptance/Extra/Workflow/DateTimeZoneWorkflowTest.php</file>
+            <file>tests/Acceptance/Extra/Schedule/ScheduleUpdateTest.php</file>
+            <file>tests/Acceptance/Harness/ChildWorkflow/CancelAbandonTest.php</file>
+            <file>tests/Acceptance/Harness/ContinueAsNew/ContinueAsSameTest.php</file>
+        </testsuite>
         <testsuite name="Acceptance">
             <directory suffix="Test.php">tests/Acceptance/Extra</directory>
             <directory suffix="Test.php">tests/Acceptance/Harness</directory>
@@ -45,4 +92,4 @@
             <directory>src</directory>
         </include>
     </source>
-</phpunit>
+</phpunit>

tests/bootstrap.php

@@ -35,7 +35,10 @@
     return null;
 })($GLOBALS['argv'] ?? []);
 
+$suite = \substr($suite, 0, \strpos($suite, '-') ?: \strlen($suite));
+
 # Include related bootstrap
-$suite === null or (static fn(string $file) => \is_file($file) and include $file)(
-    __DIR__ . DIRECTORY_SEPARATOR . $suite . DIRECTORY_SEPARATOR . 'bootstrap.php',
-);
+$file = __DIR__ . DIRECTORY_SEPARATOR . $suite . DIRECTORY_SEPARATOR . 'bootstrap.php';
+if (\is_file($file)) {
+    include $file;
+}

tests/phpunit-generate.php

@@ -0,0 +1,151 @@
+#!/usr/bin/env php
+<?php
+
+declare(strict_types=1);
+
+$threshold = 1.0;
+$junitFiles = [
+    __DIR__ . '/../runtime/phpunit-acceptance-junit.xml',
+    __DIR__ . '/../runtime/phpunit-acceptance-fast-junit.xml',
+    __DIR__ . '/../runtime/phpunit-acceptance-slow-junit.xml',
+];
+$outputFile = $argv[2] ?? __DIR__ . '/../phpunit.xml.dist';
+
+$junitFiles = \array_filter($junitFiles, 'file_exists');
+if (empty($junitFiles)) {
+    echo "No junit files found to analyze.\n";
+    exit(1);
+}
+
+foreach ($argv as $arg) {
+    if (\str_starts_with($arg, '--threshold=')) {
+        $threshold = (float) \substr($arg, 12);
+    }
+}
+
+$tests = [];
+
+foreach ($junitFiles as $junitFile) {
+    $xml = \simplexml_load_file($junitFile);
+    if ($xml === false) {
+        continue;
+    }
+    foreach ($xml->xpath('//testcase') as $testcase) {
+        $file = (string) $testcase['file'];
+        $time = (float) $testcase['time'];
+
+        if (!isset($tests[$file])) {
+            $tests[$file] = 0.0;
+        }
+        $tests[$file] = \max($tests[$file], $time);  // max instead of sum
+    }
+}
+
+$slow = [];
+$fast = [];
+
+foreach ($tests as $file => $time) {
+    if ($time >= $threshold) {
+        $slow[$file] = $time;
+    } else {
+        $fast[$file] = $time;
+    }
+}
+
+\arsort($slow);
+
+$extractPath = static function (string $file): string {
+    if (\preg_match('#(tests/Acceptance/.+)$#', $file, $matches)) {
+        return $matches[1];
+    }
+    return $file;
+};
+
+$slowFiles = \array_map($extractPath, \array_keys($slow));
+
+$excludeLines = \implode("\n", \array_map(
+    static fn(string $file): string => "            <exclude>$file</exclude>",
+    $slowFiles,
+));
+
+$fileLines = \implode("\n", \array_map(
+    static fn(string $file): string => "            <file>$file</file>",
+    $slowFiles,
+));
+
+$template = <<<'XML'
+<?xml version="1.0" encoding="UTF-8"?>
+<phpunit xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:noNamespaceSchemaLocation="vendor/phpunit/phpunit/phpunit.xsd"
+         bootstrap="tests/bootstrap.php"
+         cacheResultFile="runtime/.phpunit.result.cache"
+         backupGlobals="false"
+         colors="true"
+         processIsolation="false"
+         stopOnFailure="false"
+         stopOnError="false"
+         stderr="true"
+         displayDetailsOnIncompleteTests="true"
+         displayDetailsOnSkippedTests="true"
+         displayDetailsOnTestsThatTriggerDeprecations="true"
+         displayDetailsOnTestsThatTriggerErrors="true"
+         displayDetailsOnTestsThatTriggerNotices="true"
+         displayDetailsOnTestsThatTriggerWarnings="true">
+    <testsuites>
+        <testsuite name="Acceptance-Fast">
+            <directory suffix="Test.php">tests/Acceptance/Extra</directory>
+            <directory suffix="Test.php">tests/Acceptance/Harness</directory>
+{{EXCLUDES}}
+        </testsuite>
+        <testsuite name="Acceptance-Slow">
+{{FILES}}
+        </testsuite>
+        <testsuite name="Acceptance">
+            <directory suffix="Test.php">tests/Acceptance/Extra</directory>
+            <directory suffix="Test.php">tests/Acceptance/Harness</directory>
+        </testsuite>
+        <testsuite name="Arch">
+            <directory suffix="Test.php">tests/Arch</directory>
+        </testsuite>
+        <testsuite name="Unit">
+            <directory suffix="TestCase.php">tests/Unit</directory>
+        </testsuite>
+        <testsuite name="Functional">
+            <directory suffix="TestCase.php">tests/Functional</directory>
+        </testsuite>
+    </testsuites>
+    <groups>
+        <exclude>
+            <group>skip-on-test-server</group>
+        </exclude>
+    </groups>
+    <php>
+        <ini name="error_reporting" value="-1"/>
+        <ini name="memory_limit" value="-1"/>
+    </php>
+    <source>
+        <include>
+            <directory>src</directory>
+        </include>
+    </source>
+</phpunit>
+XML;
+
+$output = \str_replace(
+    ['{{EXCLUDES}}', '{{FILES}}'],
+    [$excludeLines, $fileLines],
+    $template,
+);
+
+\file_put_contents($outputFile, $output);
+
+$slowTime = \array_sum($slow);
+$fastTime = \array_sum($fast);
+
+echo "Generated: $outputFile (threshold: {$threshold}s)\n\n";
+echo "Fast: " . \count($fast) . " files (~" . \round($fastTime, 1) . "s)\n";
+echo "Slow: " . \count($slow) . " files (~" . \round($slowTime, 1) . "s)\n\n";
+
+foreach ($slow as $file => $time) {
+    echo \sprintf("%6.2fs  %s\n", $time, $extractPath($file));
+}