diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index e10f13ebd2..f4347d9639 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -54,16 +54,16 @@ jobs: # we want to run the full build on all os: don't cancel running jobs even if one fails fail-fast: false matrix: - os: [macos-14, ubuntu-24.04, windows-2022] + os: [macos-15, ubuntu-24.04, windows-2022] browser: [chromium, firefox, chrome] include: # only test WebKit on macOS - - os: macos-14 + - os: macos-15 browser: webkit # only test Edge on Windows - os: windows-2022 browser: msedge - - os: macos-14 + - os: macos-15 browser: chrome steps: - name: Checkout @@ -71,22 +71,8 @@ jobs: - name: Free disk space on Ubuntu runner if: runner.os == 'Linux' uses: ./.github/actions/free-runner-disk-space-ubuntu -# START temporary workaround -# Browser download failure with Node 24. See https://github.com/microsoft/playwright/issues/41185 and https://github.com/microsoft/playwright-cli/issues/419#issuecomment-4646958277 (that mentions that it should work with Node 22) -# - name: Build Setup -# uses: ./.github/actions/build-setup - - name: Setup node - uses: actions/setup-node@v6 - with: -# node-version-file: '.nvmrc' - node-version: '22' - registry-url: ${{ inputs.registry-url }} - - name: Install dependencies - uses: bahmutov/npm-install@v1 - # if: inputs.install-dependencies == 'true' - with: - install-command: npm ci --ignore-scripts --prefer-offline --audit false -# END OF temporary workaround + - name: Build Setup + uses: ./.github/actions/build-setup - name: Install ${{matrix.browser}} uses: ./.github/actions/install-playwright-browser with: diff --git a/.github/workflows/test-npm-package.yml b/.github/workflows/test-npm-package.yml index 1c44e2c947..0c61ecb1b5 100644 --- a/.github/workflows/test-npm-package.yml +++ b/.github/workflows/test-npm-package.yml @@ -47,22 +47,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v7 - # START temporary workaround - # Browser download failure with Node 24. See https://github.com/microsoft/playwright/issues/41185 and https://github.com/microsoft/playwright-cli/issues/419#issuecomment-4646958277 (that mentions that it should work with Node 22) - # - name: Build Setup - # uses: ./.github/actions/build-setup - - name: Setup node - uses: actions/setup-node@v6 - with: - # node-version-file: '.nvmrc' - node-version: '22' - registry-url: ${{ inputs.registry-url }} - - name: Install dependencies - uses: bahmutov/npm-install@v1 - # if: inputs.install-dependencies == 'true' - with: - install-command: npm ci --ignore-scripts --prefer-offline --audit false - # END OF temporary workaround + - name: Build Setup + uses: ./.github/actions/build-setup - name: Build npm package run: npm pack - name: List the size of the npm package bundles diff --git a/.github/workflows/test-performance.yml b/.github/workflows/test-performance.yml index ca52026a7b..eacd464bed 100644 --- a/.github/workflows/test-performance.yml +++ b/.github/workflows/test-performance.yml @@ -31,7 +31,7 @@ jobs: # triggerUncaughtException(err, true /* fromPromise */); # ^ # cdpSession.detach: Browser closed. - os: [macos-14, ubuntu-24.04] + os: [macos-15, ubuntu-24.04] env: # Performance tests rely on chromium API browser: chromium diff --git a/CLAUDE.md b/CLAUDE.md index 57b50312c3..5925984e86 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -186,6 +186,33 @@ GraphConfigurator **E2E Pattern:** Heavy use of image snapshots (in `__image_snapshots__/`) to verify visual rendering correctness across features. +### E2E Visual Regression Thresholds + +E2E tests compare a freshly rendered screenshot against a reference image snapshot using `jest-image-snapshot` (SSIM comparison). A test fails when the measured difference exceeds its allowed `failureThreshold` (a percentage of differing pixels). Because rendering differs slightly by browser engine and OS (mostly font rendering, a few pixels, not visible to a human), thresholds are configured per browser family and per platform. + +**How thresholds are resolved** (see `test/e2e/helpers/visu/image-snapshot-config.ts`): +- `MultiBrowserImageSnapshotThresholds` holds a default per browser family (`chromium`, `firefox`, `webkit`), passed to `super({ chromium, firefox, webkit })`. +- Each test file subclasses it and overrides `getChromiumThresholds()` / `getFirefoxThresholds()` / `getWebkitThresholds()` to return a `Map` of per-snapshot overrides. +- Each map entry is keyed by the snapshot identifier (the BPMN diagram name, sometimes with a suffix like `.ignored` / `.not-ignored`) and holds per-platform values: `{ linux?, macos?, windows? }`. +- At runtime, the browser family selects the map, `getSimplePlatformName()` selects the platform key, and the value found there wins. If there is no dedicated entry or no matching platform key, the browser-family default is used. +- On CI, macOS runs the `webkit` family, so webkit macOS thresholds come from the `macos` key inside `getWebkitThresholds()`. + +**Threshold value convention:** +- Values are written as `X / 100` (a percentage expressed as a fraction), for example `macos: 0.53 / 100`. +- Add a trailing comment with the actual observed diff percentage from the report, for example `macos: 0.53 / 100, // 0.5228413635451014%`. The fit test file prefixes it with `max` because one entry covers several test variations sharing the snapshot key. +- Set the value by rounding the observed diff UP to 2 decimals (ceil), so the threshold sits just above the observed diff. +- Keep entries ordered by snapshot key to match the surrounding file. + +**Updating thresholds from a CI test-results report** (recurring task): +1. The report bundle (downloaded artifact) contains `index-single-page.html` (jest-html-reporters) and a `__diff_output__/` folder with `-diff.png` per failure. +2. Parse `index-single-page.html`. Each failed block contains: the suite/title, `was % different from snapshot ... Failure threshold was set to %`, and a `__diff_output__/-diff.png` link. The relative path (minus `-diff.png`) identifies the snapshot; note some names contain hyphens (`.not-ignored`) and the fit tests live in nested subfolders, so capture the full path up to `-diff.png`. +3. Group by snapshot key and keep the MAX actual diff (several test variations can share one threshold key, e.g. `with.outside.labels` in the fit tests). +4. For each failing snapshot, locate the matching entry in the relevant test file's `getWebkitThresholds()` (or the correct browser/platform) and set `macos: ceil2(actual) / 100, // %`. If no entry exists (the failure was against the browser-family default), add one in key order. +5. Files that currently carry webkit macOS thresholds: `bpmn.rendering.test.ts`, `bpmn.rendering.ignore.options.test.ts`, `bpmn.colors.test.ts`, `diagram.navigation.fit.test.ts`, `style.api.test.ts` (a test file may hold several threshold subclasses, one per configurator). +6. Reporting: flag any entry where the increase (`new threshold - old threshold`) exceeds 0.3 percentage points. A large jump usually signals a real rendering change or regression worth a human look, not just pixel noise. + +Note (from the class JSDoc): prefer NOT adding a threshold for a new test until it actually fails on CI. Discrepancies mostly come from labels; if labels are not part of what the test verifies, remove labels from the BPMN diagram instead of raising a threshold. + ## Key Architectural Patterns ### Converter Pattern diff --git a/package-lock.json b/package-lock.json index b733701a08..e213a1dd95 100644 --- a/package-lock.json +++ b/package-lock.json @@ -56,7 +56,7 @@ "minimist": "~1.2.8", "npm-run-all": "~4.1.5", "pinst": "~3.0.0", - "playwright": "~1.58.2", + "playwright": "~1.61.1", "postcss": "~8.5.14", "postcss-cli": "~11.0.1", "prettier": "~3.8.1", @@ -2902,9 +2902,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2922,9 +2919,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -2942,9 +2936,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2962,9 +2953,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2982,9 +2970,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3002,9 +2987,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -13986,12 +13968,13 @@ } }, "node_modules/playwright": { - "version": "1.58.2", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.2.tgz", - "integrity": "sha512-vA30H8Nvkq/cPBnNw4Q8TWz1EJyqgpuinBcHET0YVJVFldr8JDNiU9LaWAE1KqSkRYazuaBhTpB5ZzShOezQ6A==", + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz", + "integrity": "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==", "dev": true, + "license": "Apache-2.0", "dependencies": { - "playwright-core": "1.58.2" + "playwright-core": "1.61.1" }, "bin": { "playwright": "cli.js" @@ -14004,10 +13987,11 @@ } }, "node_modules/playwright-core": { - "version": "1.58.2", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.58.2.tgz", - "integrity": "sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==", + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz", + "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==", "dev": true, + "license": "Apache-2.0", "bin": { "playwright-core": "cli.js" }, @@ -17326,9 +17310,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -17350,9 +17331,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -17374,9 +17352,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -17398,9 +17373,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -27511,19 +27483,19 @@ } }, "playwright": { - "version": "1.58.2", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.2.tgz", - "integrity": "sha512-vA30H8Nvkq/cPBnNw4Q8TWz1EJyqgpuinBcHET0YVJVFldr8JDNiU9LaWAE1KqSkRYazuaBhTpB5ZzShOezQ6A==", + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz", + "integrity": "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==", "dev": true, "requires": { "fsevents": "2.3.2", - "playwright-core": "1.58.2" + "playwright-core": "1.61.1" } }, "playwright-core": { - "version": "1.58.2", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.58.2.tgz", - "integrity": "sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==", + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz", + "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==", "dev": true }, "pluralize": { diff --git a/package.json b/package.json index aad06b4fd6..401cbfb0d8 100644 --- a/package.json +++ b/package.json @@ -151,7 +151,7 @@ "minimist": "~1.2.8", "npm-run-all": "~4.1.5", "pinst": "~3.0.0", - "playwright": "~1.58.2", + "playwright": "~1.61.1", "postcss": "~8.5.14", "postcss-cli": "~11.0.1", "prettier": "~3.8.1", diff --git a/test/e2e/bpmn.colors.test.ts b/test/e2e/bpmn.colors.test.ts index 58f01124f4..29f41627dc 100644 --- a/test/e2e/bpmn.colors.test.ts +++ b/test/e2e/bpmn.colors.test.ts @@ -77,7 +77,7 @@ class ImageSnapshotThresholdsModelColors extends MultiBrowserImageSnapshotThresh [ 'elements.colors.02.labels', { - macos: 0.41 / 100, // 0.40831372531949794% + macos: 1.04 / 100, // 1.033599854269418% }, ], ]); @@ -140,7 +140,7 @@ class ImageSnapshotThresholdsIgnoreBpmnColors extends MultiBrowserImageSnapshotT [ 'elements.colors.02.labels', { - macos: 0.49 / 100, // 0.483122009334358% + macos: 1.13 / 100, // 1.1249364408232543% }, ], ]); diff --git a/test/e2e/bpmn.rendering.ignore.options.test.ts b/test/e2e/bpmn.rendering.ignore.options.test.ts index d27437d03f..6d28c80740 100644 --- a/test/e2e/bpmn.rendering.ignore.options.test.ts +++ b/test/e2e/bpmn.rendering.ignore.options.test.ts @@ -70,13 +70,13 @@ class ImageSnapshotThresholdsActivityLabelBounds extends MultiBrowserImageSnapsh [ 'activities.with.wrongly.positioned.labels.not-ignored', { - macos: 0.44 / 100, // 0.4382175377357411% + macos: 1.17 / 100, // 1.1642294732319813% }, ], [ 'activities.with.wrongly.positioned.labels.ignored', { - macos: 1.5 / 100, // 1.4951298719464878% + macos: 2.01 / 100, // 2.0057344642194885% }, ], ]); @@ -135,13 +135,13 @@ class ImageSnapshotThresholdsLabelStyles extends MultiBrowserImageSnapshotThresh [ 'labels.with.font.styles.not-ignored', { - macos: 0.31 / 100, // 0.30621380597637415% + macos: 0.59 / 100, // 0.584031268762264% }, ], [ 'labels.with.font.styles.ignored', { - macos: 0.38 / 100, // 0.37988509633168904% + macos: 0.84 / 100, // 0.8385178209979638% }, ], ]); diff --git a/test/e2e/bpmn.rendering.test.ts b/test/e2e/bpmn.rendering.test.ts index c195068557..dfcb16b948 100644 --- a/test/e2e/bpmn.rendering.test.ts +++ b/test/e2e/bpmn.rendering.test.ts @@ -207,36 +207,42 @@ class ImageSnapshotThresholds extends MultiBrowserImageSnapshotThresholds { macos: 0.41 / 100, // 0.40487233108913445% }, ], + [ + 'group.02.in.collaboration.with.label', + { + macos: 0.16 / 100, // 0.15458864441817433% + }, + ], [ 'labels.01.general', { // high value due to font rendering discrepancies with chromium rendering - macos: 1.79 / 100, // 1.7833883910028492% + macos: 4.65 / 100, // 4.644599369259983% }, ], [ 'labels.02.position.and.line.breaks', { // TODO possible rendering issue so high threshold value - macos: 6.11 / 100, // 6.105183205727094% + macos: 6.65 / 100, // 6.6443181311426125% }, ], [ 'labels.03.default.position', { - macos: 0.64 / 100, // 0.6346061558805904% + macos: 1.05 / 100, // 1.0442324603631814% }, ], [ 'labels.04.fonts', { - macos: 0.71 / 100, // 0.703880504764276% + macos: 1.11 / 100, // 1.1087684739990888% }, ], [ 'labels.05.default.position.activities', { - macos: 1.2 / 100, // 1.192492604936246% + macos: 2.31 / 100, // 2.303108822951705% }, ], [ @@ -248,13 +254,13 @@ class ImageSnapshotThresholds extends MultiBrowserImageSnapshotThresholds { [ 'pools.02.vertical.with.lanes', { - macos: 0.24 / 100, // 0.23336351480325318% + macos: 0.47 / 100, // 0.4618019396829043% }, ], [ 'pools.03.black.box', { - macos: 0.36 / 100, // 0.3576987596416892% + macos: 0.53 / 100, // 0.5228413635451014% }, ], ]); diff --git a/test/e2e/diagram.navigation.fit.test.ts b/test/e2e/diagram.navigation.fit.test.ts index 038613e93a..5d9a3420d9 100644 --- a/test/e2e/diagram.navigation.fit.test.ts +++ b/test/e2e/diagram.navigation.fit.test.ts @@ -121,7 +121,7 @@ class ImageSnapshotThresholds extends MultiBrowserImageSnapshotThresholds { [ 'with.outside.labels', { - macos: 0.39 / 100, // max 0.38104004012843307% + macos: 0.51 / 100, // max 0.5058234845605303% }, ], ]); diff --git a/test/e2e/style.api.test.ts b/test/e2e/style.api.test.ts index 9da4241081..0e3ec6e462 100644 --- a/test/e2e/style.api.test.ts +++ b/test/e2e/style.api.test.ts @@ -117,7 +117,7 @@ class StyleImageSnapshotThresholds extends MultiBrowserImageSnapshotThresholds { [ 'font.color.opacity', { - macos: 0.2 / 100, // 0.18895676780704695% + macos: 0.33 / 100, // 0.324776594957199% }, ], [