From 3806b1c8ab5219af1b97bbe08df6dfbcaea11f28 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Tue, 18 Aug 2026 18:32:43 +0400 Subject: [PATCH 01/10] Add corpus-diff CI: regenerate a pinned WordPress corpus and diff it --- .github/workflows/corpus-diff.yml | 99 +++++++++++++++++++++++++++++++ tools/export-corpus.php | 48 +++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 .github/workflows/corpus-diff.yml create mode 100644 tools/export-corpus.php diff --git a/.github/workflows/corpus-diff.yml b/.github/workflows/corpus-diff.yml new file mode 100644 index 0000000..cacf74e --- /dev/null +++ b/.github/workflows/corpus-diff.yml @@ -0,0 +1,99 @@ +# Parses a pinned corpus of WordPress core source with the parser at the PR's +# merge base and at its head, normalizes both JSON outputs with prep-diff.php, +# and diffs them. Anything in the diff is a behavior change this PR makes: +# every hunk must be either intended (and explained in the PR) or a regression. +# +# Policy decisions, deliberate: +# - The corpus is pinned to one WordPress tag so diffs are reproducible. +# - The head checkout's tools/export-corpus.php and prep-diff.php drive both +# sides, so tooling changes never masquerade as parser changes. When +# prep-diff.php itself changes, its effect on normalization shows up in the +# diff and is reviewed like any other change. +# - Non-blocking: the job succeeds even when the diff is non-empty. The diff +# is published as an artifact and summarized. Make it blocking only after +# the signal has proven trustworthy. +# - PHP 7.4, the supported floor, keeps parity with the oldest runtime. + +name: Corpus Diff + +on: + pull_request: + +jobs: + corpus-diff: + name: WordPress corpus regeneration diff + runs-on: ubuntu-latest + + env: + WP_CORPUS_TAG: "6.8" + LC_ALL: C + + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Set up PHP + uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # 2.37.2 + with: + php-version: "7.4" + coverage: none + + - name: Cache corpus + id: cache-corpus + uses: actions/cache@v4 + with: + path: corpus + key: wp-corpus-${{ env.WP_CORPUS_TAG }} + + - name: Download corpus + if: steps.cache-corpus.outputs.cache-hit != 'true' + run: | + curl -sSfL -o wordpress.zip "https://github.com/WordPress/WordPress/archive/refs/tags/${WP_CORPUS_TAG}.zip" + unzip -q wordpress.zip "WordPress-${WP_CORPUS_TAG}/wp-includes/*" + mkdir -p corpus + mv "WordPress-${WP_CORPUS_TAG}/wp-includes" corpus/wp-includes + rm -rf wordpress.zip "WordPress-${WP_CORPUS_TAG}" + + - name: Check out merge base + run: git worktree add base "$(git merge-base "origin/${{ github.base_ref }}" HEAD)" + + - name: Install Composer dependencies (head) + run: composer install --no-interaction --no-security-blocking + + - name: Install Composer dependencies (base) + run: composer --working-dir=base install --no-interaction --no-security-blocking + + - name: Export corpus (base) + run: php -d memory_limit=4G tools/export-corpus.php base corpus/wp-includes > base.json + + - name: Export corpus (head) + run: php -d memory_limit=4G tools/export-corpus.php . corpus/wp-includes > head.json + + - name: Normalize and diff + run: | + php -d memory_limit=4G prep-diff.php < base.json > base.norm.json + php -d memory_limit=4G prep-diff.php < head.json > head.norm.json + # diff exits 1 on differences (expected) and 2 on trouble (fail). + diff -u --label base --label head base.norm.json head.norm.json > corpus.diff || [ $? -eq 1 ] + if [ -s corpus.diff ]; then + hunks=$(grep -c '^@@' corpus.diff) + lines=$(wc -l < corpus.diff) + { + echo "### Corpus diff: ${hunks} hunks, ${lines} lines" + echo + echo "The parser's output over wp-includes@${WP_CORPUS_TAG} changed." + echo "Review the \`corpus.diff\` artifact: every hunk must be intended and explained in the PR." + } >> "$GITHUB_STEP_SUMMARY" + else + echo "### Corpus diff: 0 hunks (no behavior change)" >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload diff + if: always() + uses: actions/upload-artifact@v4 + with: + name: corpus.diff + path: corpus.diff + if-no-files-found: ignore diff --git a/tools/export-corpus.php b/tools/export-corpus.php new file mode 100644 index 0000000..8bf062e --- /dev/null +++ b/tools/export-corpus.php @@ -0,0 +1,48 @@ + > corpus.json + */ + +// Keep stdout pure JSON: vendored code emits deprecation notices on newer +// PHP, and display_errors otherwise interleaves them with the output. +ini_set( 'display_errors', 'stderr' ); + +if ( $argc < 3 ) { + fwrite( STDERR, 'Usage: php tools/export-corpus.php ' . PHP_EOL ); + exit( 1 ); +} + +$parser_root = rtrim( $argv[1], '/' ); +$corpus_dir = rtrim( $argv[2], '/' ); + +if ( ! is_file( $parser_root . '/vendor/autoload.php' ) ) { + fwrite( STDERR, 'No Composer autoloader in ' . $parser_root . '; run composer install there first.' . PHP_EOL ); + exit( 1 ); +} + +if ( ! is_dir( $corpus_dir ) ) { + fwrite( STDERR, 'Corpus directory not found: ' . $corpus_dir . PHP_EOL ); + exit( 1 ); +} + +require $parser_root . '/vendor/autoload.php'; + +$files = \WP_Parser\get_wp_files( $corpus_dir ); + +if ( ! is_array( $files ) ) { + fwrite( STDERR, 'Could not list the PHP files in ' . $corpus_dir . '.' . PHP_EOL ); + exit( 1 ); +} + +echo json_encode( \WP_Parser\parse_files( $files, $corpus_dir ), JSON_PRETTY_PRINT ), PHP_EOL; From 09fa3552c4d2a68c9a507da61c6b1390bbf9fb79 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Tue, 18 Aug 2026 18:45:54 +0400 Subject: [PATCH 02/10] Guard the corpus-diff job against a silently empty corpus --- .github/workflows/corpus-diff.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/corpus-diff.yml b/.github/workflows/corpus-diff.yml index cacf74e..c7245f0 100644 --- a/.github/workflows/corpus-diff.yml +++ b/.github/workflows/corpus-diff.yml @@ -56,6 +56,15 @@ jobs: mv "WordPress-${WP_CORPUS_TAG}/wp-includes" corpus/wp-includes rm -rf wordpress.zip "WordPress-${WP_CORPUS_TAG}" + # An empty or miscached corpus would make both exports emit `[]` and the + # diff trivially empty — a green job that checked nothing. wp-includes + # has well over 500 PHP files on any supported tag. + - name: Verify corpus + run: | + count=$(find corpus/wp-includes -name '*.php' | wc -l) + echo "Corpus PHP files: ${count}" + [ "$count" -ge 500 ] + - name: Check out merge base run: git worktree add base "$(git merge-base "origin/${{ github.base_ref }}" HEAD)" @@ -65,11 +74,19 @@ jobs: - name: Install Composer dependencies (base) run: composer --working-dir=base install --no-interaction --no-security-blocking + # The size floor guards the same failure mode as Verify corpus: a real + # wp-includes export is tens of megabytes of JSON. - name: Export corpus (base) - run: php -d memory_limit=4G tools/export-corpus.php base corpus/wp-includes > base.json + run: | + php -d memory_limit=4G tools/export-corpus.php base corpus/wp-includes > base.json + ls -l base.json + [ "$(wc -c < base.json)" -ge 1000000 ] - name: Export corpus (head) - run: php -d memory_limit=4G tools/export-corpus.php . corpus/wp-includes > head.json + run: | + php -d memory_limit=4G tools/export-corpus.php . corpus/wp-includes > head.json + ls -l head.json + [ "$(wc -c < head.json)" -ge 1000000 ] - name: Normalize and diff run: | From 66ca1eb6a6e9876d92c3b3aabc5dfb6db84f55dc Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Tue, 18 Aug 2026 21:45:41 +0400 Subject: [PATCH 03/10] Pin the corpus to WordPress 7.0.4 7.0.4 is the latest WordPress release. Pinning the point release rather than the 7.0 major tag tracks the patches to wp-includes, so the corpus matches what is shipped. WordPress 7.0 raised core's minimum PHP to 7.4, which is exactly this job's PHP floor. Verified against wp-includes@7.0.4: 1039 PHP files, all of them present in the export, no parse errors, ~49 MB of JSON. Both of the job's guards (>= 500 files, >= 1 MB of JSON) still hold. --- .github/workflows/corpus-diff.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/corpus-diff.yml b/.github/workflows/corpus-diff.yml index c7245f0..0217269 100644 --- a/.github/workflows/corpus-diff.yml +++ b/.github/workflows/corpus-diff.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest env: - WP_CORPUS_TAG: "6.8" + WP_CORPUS_TAG: "7.0.4" LC_ALL: C steps: From 1c23bb71a6a052169a0c2aaa551970a1dc8b451d Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Tue, 18 Aug 2026 21:46:29 +0400 Subject: [PATCH 04/10] Document the corpus diff procedure Records what the corpus-diff check is, how to run it by hand with the repo's own tools/export-corpus.php and prep-diff.php, and the policy the workflow already states: the head checkout's tooling drives both sides. --- README.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/README.md b/README.md index ab35ee4..0057f8b 100644 --- a/README.md +++ b/README.md @@ -50,3 +50,37 @@ In your site's directory: ```bash wp parser create /path/to/source/code --user= ``` + +## Corpus diff + +Unit tests do not cover every shape of real-world documentation, so changes to the parser are also checked against a corpus of WordPress core source. The same corpus is parsed with the parser at two refs — the merge base of a pull request and its head — both JSON outputs are normalized with `prep-diff.php`, and the two are diffed. Everything in that diff is a behavior change the pull request makes: every hunk must be either intended and explained, or it is a regression. + +`.github/workflows/corpus-diff.yml` runs this on every pull request. The corpus is `wp-includes` from a pinned WordPress tag (`WP_CORPUS_TAG` in the workflow), so diffs are reproducible. The job is non-blocking: it uploads the diff as a `corpus.diff` artifact and reports the hunk count in the job summary. The head checkout's `tools/export-corpus.php` and `prep-diff.php` drive both sides, so tooling changes never masquerade as parser changes; when `prep-diff.php` itself changes, its effect on normalization shows up in the diff and is reviewed like any other change. + +To run it locally, get the pinned corpus: + +```bash +curl -sSfL -o wordpress.zip https://github.com/WordPress/WordPress/archive/refs/tags/7.0.4.zip +unzip -q wordpress.zip 'WordPress-7.0.4/wp-includes/*' +``` + +Check out the other side of the comparison and install its dependencies. The exporter runs under plain PHP — it does not load WordPress — but it needs a Composer autoloader in each parser root: + +```bash +git worktree add base "$(git merge-base origin/master HEAD)" +composer --working-dir=base install +composer install +``` + +Export both sides over the same corpus, normalize, and diff. `export-corpus.php` takes the parser root and the corpus directory, and writes JSON to stdout: + +```bash +export LC_ALL=C +php -d memory_limit=4G tools/export-corpus.php base WordPress-7.0.4/wp-includes > base.json +php -d memory_limit=4G tools/export-corpus.php . WordPress-7.0.4/wp-includes > head.json +php -d memory_limit=4G prep-diff.php < base.json > base.norm.json +php -d memory_limit=4G prep-diff.php < head.json > head.norm.json +diff -u base.norm.json head.norm.json > corpus.diff +``` + +An empty `corpus.diff` means the change has no effect on parser output. From ac7f6b7a3bdd0304d518c7550bd97e216ec155bc Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Thu, 20 Aug 2026 18:16:27 +0400 Subject: [PATCH 05/10] Address corpus-diff review notes - Name the comparison for what it is: on pull_request HEAD is the PR merged into the base branch, so the merge base resolves to the base branch tip. Rename the step, explain it in the workflow, and say the same in the README, including how to reproduce that comparison locally. - Drop the unreachable is_array() guard in tools/export-corpus.php: get_wp_files() returns a WP_Error on failure, which does not exist under plain PHP, so that path fatals with a non-zero exit before it can return. - Add a concurrency group so a new push cancels the in-flight run for the same PR. --- .github/workflows/corpus-diff.yml | 21 ++++++++++++++++----- README.md | 4 +++- tools/export-corpus.php | 8 +++----- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/.github/workflows/corpus-diff.yml b/.github/workflows/corpus-diff.yml index 0217269..fdf0635 100644 --- a/.github/workflows/corpus-diff.yml +++ b/.github/workflows/corpus-diff.yml @@ -1,7 +1,8 @@ -# Parses a pinned corpus of WordPress core source with the parser at the PR's -# merge base and at its head, normalizes both JSON outputs with prep-diff.php, -# and diffs them. Anything in the diff is a behavior change this PR makes: -# every hunk must be either intended (and explained in the PR) or a regression. +# Parses a pinned corpus of WordPress core source with the parser at the base +# branch and with the PR merged into it, normalizes both JSON outputs with +# prep-diff.php, and diffs them. Anything in the diff is a behavior change this +# PR makes: every hunk must be either intended (and explained in the PR) or a +# regression. # # Policy decisions, deliberate: # - The corpus is pinned to one WordPress tag so diffs are reproducible. @@ -19,6 +20,11 @@ name: Corpus Diff on: pull_request: +# A new push to the PR supersedes any run still in flight for it. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + jobs: corpus-diff: name: WordPress corpus regeneration diff @@ -65,7 +71,12 @@ jobs: echo "Corpus PHP files: ${count}" [ "$count" -ge 500 ] - - name: Check out merge base + # On pull_request, HEAD is the PR merged into the base branch, so its + # merge base with the base branch is the base branch tip, not the commit + # the PR branched from. That is the intended comparison: base as it is + # now against base plus exactly this PR, with no hunks from other work + # that landed since the branch point. + - name: Check out base run: git worktree add base "$(git merge-base "origin/${{ github.base_ref }}" HEAD)" - name: Install Composer dependencies (head) diff --git a/README.md b/README.md index 0057f8b..173d6be 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ wp parser create /path/to/source/code --user= ## Corpus diff -Unit tests do not cover every shape of real-world documentation, so changes to the parser are also checked against a corpus of WordPress core source. The same corpus is parsed with the parser at two refs — the merge base of a pull request and its head — both JSON outputs are normalized with `prep-diff.php`, and the two are diffed. Everything in that diff is a behavior change the pull request makes: every hunk must be either intended and explained, or it is a regression. +Unit tests do not cover every shape of real-world documentation, so changes to the parser are also checked against a corpus of WordPress core source. The same corpus is parsed with the parser at two refs — the base branch and the pull request merged into it — both JSON outputs are normalized with `prep-diff.php`, and the two are diffed. Everything in that diff is a behavior change the pull request makes: every hunk must be either intended and explained, or it is a regression. `.github/workflows/corpus-diff.yml` runs this on every pull request. The corpus is `wp-includes` from a pinned WordPress tag (`WP_CORPUS_TAG` in the workflow), so diffs are reproducible. The job is non-blocking: it uploads the diff as a `corpus.diff` artifact and reports the hunk count in the job summary. The head checkout's `tools/export-corpus.php` and `prep-diff.php` drive both sides, so tooling changes never masquerade as parser changes; when `prep-diff.php` itself changes, its effect on normalization shows up in the diff and is reviewed like any other change. @@ -72,6 +72,8 @@ composer --working-dir=base install composer install ``` +On a branch this compares against the commit the branch left `master` at. CI runs on the pull request merged into `master`, so it compares against the `master` tip; merge `master` into the branch first to get the same comparison. + Export both sides over the same corpus, normalize, and diff. `export-corpus.php` takes the parser root and the corpus directory, and writes JSON to stdout: ```bash diff --git a/tools/export-corpus.php b/tools/export-corpus.php index 8bf062e..bb771f9 100644 --- a/tools/export-corpus.php +++ b/tools/export-corpus.php @@ -38,11 +38,9 @@ require $parser_root . '/vendor/autoload.php'; +// get_wp_files() reports an unreadable subdirectory with a WP_Error, a class +// that only exists inside WordPress: under plain PHP that path is a fatal +// error with a non-zero exit, so there is no return value to check here. $files = \WP_Parser\get_wp_files( $corpus_dir ); -if ( ! is_array( $files ) ) { - fwrite( STDERR, 'Could not list the PHP files in ' . $corpus_dir . '.' . PHP_EOL ); - exit( 1 ); -} - echo json_encode( \WP_Parser\parse_files( $files, $corpus_dir ), JSON_PRETTY_PRINT ), PHP_EOL; From 2df6c27b6b53200bd588a507c7865466b070277a Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Thu, 20 Aug 2026 18:17:31 +0400 Subject: [PATCH 06/10] Run the corpus diff on PHP 8.4 Both sides run under the same binary, so the PHP version is not part of the comparison; use the newest runtime in the unit-test matrix. --- .github/workflows/corpus-diff.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/corpus-diff.yml b/.github/workflows/corpus-diff.yml index fdf0635..265ae88 100644 --- a/.github/workflows/corpus-diff.yml +++ b/.github/workflows/corpus-diff.yml @@ -13,7 +13,8 @@ # - Non-blocking: the job succeeds even when the diff is non-empty. The diff # is published as an artifact and summarized. Make it blocking only after # the signal has proven trustworthy. -# - PHP 7.4, the supported floor, keeps parity with the oldest runtime. +# - PHP 8.4, the newest runtime in the unit-test matrix. Both sides run under +# the same binary, so the PHP version never shows up as a parser change. name: Corpus Diff @@ -43,7 +44,7 @@ jobs: - name: Set up PHP uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # 2.37.2 with: - php-version: "7.4" + php-version: "8.4" coverage: none - name: Cache corpus From 8afb662f1ddf0f487acbb55cecd20e2f4422f397 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Fri, 21 Aug 2026 13:50:40 +0400 Subject: [PATCH 07/10] Diff the corpus exports without prep-diff.php Both sides share the corpus, the PHP binary, and the exporter, so the export is already deterministic. prep-diff.php reconciles exports from different environments: it zeroes line numbers, strips global-namespace prefixes, and sorts collections. Measured against three open PRs, the raw diff was equal or smaller in every case and kept each change at its source location, while the collection sort relocated changed records and the erasures hid classes of real change. Diff the exports as emitted. --- .github/workflows/corpus-diff.yml | 24 ++++++++++++------------ README.md | 10 ++++------ 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/.github/workflows/corpus-diff.yml b/.github/workflows/corpus-diff.yml index 265ae88..f14ee21 100644 --- a/.github/workflows/corpus-diff.yml +++ b/.github/workflows/corpus-diff.yml @@ -1,15 +1,17 @@ # Parses a pinned corpus of WordPress core source with the parser at the base -# branch and with the PR merged into it, normalizes both JSON outputs with -# prep-diff.php, and diffs them. Anything in the diff is a behavior change this -# PR makes: every hunk must be either intended (and explained in the PR) or a -# regression. +# branch and with the PR merged into it, and diffs the two JSON exports. +# Anything in the diff is a behavior change this PR makes: every hunk must be +# either intended (and explained in the PR) or a regression. # # Policy decisions, deliberate: # - The corpus is pinned to one WordPress tag so diffs are reproducible. -# - The head checkout's tools/export-corpus.php and prep-diff.php drive both -# sides, so tooling changes never masquerade as parser changes. When -# prep-diff.php itself changes, its effect on normalization shows up in the -# diff and is reviewed like any other change. +# - The head checkout's tools/export-corpus.php drives both sides, so tooling +# changes never masquerade as parser changes. +# - The exports are diffed as emitted, without prep-diff.php. Both sides share +# the corpus, the PHP binary, and the exporter, so the export is already +# deterministic; prep-diff.php exists to reconcile exports from different +# environments, and its erasures (line numbers, global-namespace prefixes) +# and collection sorting would hide or scatter real changes here. # - Non-blocking: the job succeeds even when the diff is non-empty. The diff # is published as an artifact and summarized. Make it blocking only after # the signal has proven trustworthy. @@ -100,12 +102,10 @@ jobs: ls -l head.json [ "$(wc -c < head.json)" -ge 1000000 ] - - name: Normalize and diff + - name: Diff run: | - php -d memory_limit=4G prep-diff.php < base.json > base.norm.json - php -d memory_limit=4G prep-diff.php < head.json > head.norm.json # diff exits 1 on differences (expected) and 2 on trouble (fail). - diff -u --label base --label head base.norm.json head.norm.json > corpus.diff || [ $? -eq 1 ] + diff -u --label base --label head base.json head.json > corpus.diff || [ $? -eq 1 ] if [ -s corpus.diff ]; then hunks=$(grep -c '^@@' corpus.diff) lines=$(wc -l < corpus.diff) diff --git a/README.md b/README.md index 173d6be..eb4087c 100644 --- a/README.md +++ b/README.md @@ -53,9 +53,9 @@ wp parser create /path/to/source/code --user= ## Corpus diff -Unit tests do not cover every shape of real-world documentation, so changes to the parser are also checked against a corpus of WordPress core source. The same corpus is parsed with the parser at two refs — the base branch and the pull request merged into it — both JSON outputs are normalized with `prep-diff.php`, and the two are diffed. Everything in that diff is a behavior change the pull request makes: every hunk must be either intended and explained, or it is a regression. +Unit tests do not cover every shape of real-world documentation, so changes to the parser are also checked against a corpus of WordPress core source. The same corpus is parsed with the parser at two refs, the base branch and the pull request merged into it, and the two JSON exports are diffed. Everything in that diff is a behavior change the pull request makes: every hunk must be either intended and explained, or it is a regression. -`.github/workflows/corpus-diff.yml` runs this on every pull request. The corpus is `wp-includes` from a pinned WordPress tag (`WP_CORPUS_TAG` in the workflow), so diffs are reproducible. The job is non-blocking: it uploads the diff as a `corpus.diff` artifact and reports the hunk count in the job summary. The head checkout's `tools/export-corpus.php` and `prep-diff.php` drive both sides, so tooling changes never masquerade as parser changes; when `prep-diff.php` itself changes, its effect on normalization shows up in the diff and is reviewed like any other change. +`.github/workflows/corpus-diff.yml` runs this on every pull request. The corpus is `wp-includes` from a pinned WordPress tag (`WP_CORPUS_TAG` in the workflow), so diffs are reproducible. The job is non-blocking: it uploads the diff as a `corpus.diff` artifact and reports the hunk count in the job summary. The head checkout's `tools/export-corpus.php` drives both sides, so tooling changes never masquerade as parser changes. The exports are diffed as emitted: both sides share the corpus, the PHP binary, and the exporter, so the output is already deterministic. `prep-diff.php` is for comparing exports from different environments; its line-number and namespace-prefix erasure and its collection sorting would hide or scatter real changes here. To run it locally, get the pinned corpus: @@ -74,15 +74,13 @@ composer install On a branch this compares against the commit the branch left `master` at. CI runs on the pull request merged into `master`, so it compares against the `master` tip; merge `master` into the branch first to get the same comparison. -Export both sides over the same corpus, normalize, and diff. `export-corpus.php` takes the parser root and the corpus directory, and writes JSON to stdout: +Export both sides over the same corpus and diff. `export-corpus.php` takes the parser root and the corpus directory, and writes JSON to stdout: ```bash export LC_ALL=C php -d memory_limit=4G tools/export-corpus.php base WordPress-7.0.4/wp-includes > base.json php -d memory_limit=4G tools/export-corpus.php . WordPress-7.0.4/wp-includes > head.json -php -d memory_limit=4G prep-diff.php < base.json > base.norm.json -php -d memory_limit=4G prep-diff.php < head.json > head.norm.json -diff -u base.norm.json head.norm.json > corpus.diff +diff -u base.json head.json > corpus.diff ``` An empty `corpus.diff` means the change has no effect on parser output. From 878dadb96c65c2372a7c291c5e5090ec9409714e Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Fri, 21 Aug 2026 18:32:58 +0400 Subject: [PATCH 08/10] Post the corpus diff as a pull request comment tools/corpus-diff-comment.sh renders the diff as Markdown: hunk and line counts, the compared commits, a download link, and the diff in a collapsed diff block when the body fits under GitHub's comment limit; otherwise the counts and the link alone. The workflow writes the same report to the job summary and creates or updates a single PR comment, found by a marker on its first line. pull_request runs for forks and Dependabot get a read-only token, so the comment step is skipped for them. --- .github/workflows/corpus-diff.yml | 53 +++++++++++++++++++------- README.md | 2 +- tools/corpus-diff-comment.sh | 62 +++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 15 deletions(-) create mode 100755 tools/corpus-diff-comment.sh diff --git a/.github/workflows/corpus-diff.yml b/.github/workflows/corpus-diff.yml index f14ee21..5469489 100644 --- a/.github/workflows/corpus-diff.yml +++ b/.github/workflows/corpus-diff.yml @@ -13,8 +13,12 @@ # environments, and its erasures (line numbers, global-namespace prefixes) # and collection sorting would hide or scatter real changes here. # - Non-blocking: the job succeeds even when the diff is non-empty. The diff -# is published as an artifact and summarized. Make it blocking only after -# the signal has proven trustworthy. +# is published as an artifact, summarized, and posted as a PR comment that +# is updated in place on every run. Make it blocking only after the signal +# has proven trustworthy. +# - The comment needs a token that can write to the PR. pull_request runs for +# forks and for Dependabot get a read-only token, so those PRs get the +# summary and the artifact only. # - PHP 8.4, the newest runtime in the unit-test matrix. Both sides run under # the same binary, so the PHP version never shows up as a parser change. @@ -28,6 +32,10 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} cancel-in-progress: true +permissions: + contents: read + pull-requests: write + jobs: corpus-diff: name: WordPress corpus regeneration diff @@ -106,23 +114,40 @@ jobs: run: | # diff exits 1 on differences (expected) and 2 on trouble (fail). diff -u --label base --label head base.json head.json > corpus.diff || [ $? -eq 1 ] - if [ -s corpus.diff ]; then - hunks=$(grep -c '^@@' corpus.diff) - lines=$(wc -l < corpus.diff) - { - echo "### Corpus diff: ${hunks} hunks, ${lines} lines" - echo - echo "The parser's output over wp-includes@${WP_CORPUS_TAG} changed." - echo "Review the \`corpus.diff\` artifact: every hunk must be intended and explained in the PR." - } >> "$GITHUB_STEP_SUMMARY" - else - echo "### Corpus diff: 0 hunks (no behavior change)" >> "$GITHUB_STEP_SUMMARY" - fi + echo "Corpus diff: $(grep -c '^@@' corpus.diff || true) hunks, $(wc -l < corpus.diff) lines" - name: Upload diff + id: upload if: always() uses: actions/upload-artifact@v4 with: name: corpus.diff path: corpus.diff if-no-files-found: ignore + + - name: Render report + env: + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + ARTIFACT_URL: ${{ steps.upload.outputs.artifact-url }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + BASE_SHA="$(git -C base rev-parse HEAD)" tools/corpus-diff-comment.sh corpus.diff > comment.md + tail -n +2 comment.md >> "$GITHUB_STEP_SUMMARY" + + # Create the comment on the first run and update it on every later one, + # so the PR carries one report that reflects the latest push. + - name: Comment on the pull request + if: github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + PR: ${{ github.event.pull_request.number }} + run: | + ids=$(gh api "repos/${REPO}/issues/${PR}/comments" --paginate \ + --jq '.[] | select(.body | startswith("")) | .id') + id=${ids%%$'\n'*} + if [ -n "$id" ]; then + gh api --silent -X PATCH "repos/${REPO}/issues/comments/${id}" -F body=@comment.md + else + gh api --silent -X POST "repos/${REPO}/issues/${PR}/comments" -F body=@comment.md + fi diff --git a/README.md b/README.md index eb4087c..68efd39 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ wp parser create /path/to/source/code --user= Unit tests do not cover every shape of real-world documentation, so changes to the parser are also checked against a corpus of WordPress core source. The same corpus is parsed with the parser at two refs, the base branch and the pull request merged into it, and the two JSON exports are diffed. Everything in that diff is a behavior change the pull request makes: every hunk must be either intended and explained, or it is a regression. -`.github/workflows/corpus-diff.yml` runs this on every pull request. The corpus is `wp-includes` from a pinned WordPress tag (`WP_CORPUS_TAG` in the workflow), so diffs are reproducible. The job is non-blocking: it uploads the diff as a `corpus.diff` artifact and reports the hunk count in the job summary. The head checkout's `tools/export-corpus.php` drives both sides, so tooling changes never masquerade as parser changes. The exports are diffed as emitted: both sides share the corpus, the PHP binary, and the exporter, so the output is already deterministic. `prep-diff.php` is for comparing exports from different environments; its line-number and namespace-prefix erasure and its collection sorting would hide or scatter real changes here. +`.github/workflows/corpus-diff.yml` runs this on every pull request. The corpus is `wp-includes` from a pinned WordPress tag (`WP_CORPUS_TAG` in the workflow), so diffs are reproducible. The job is non-blocking: it uploads the diff as a `corpus.diff` artifact, reports it in the job summary, and posts one comment on the pull request, updated on every push, with the hunk and line counts and the diff itself when it fits in a comment (`tools/corpus-diff-comment.sh` renders it). Pull requests from forks and from Dependabot run with a read-only token, so they get the summary and the artifact only. The head checkout's `tools/export-corpus.php` drives both sides, so tooling changes never masquerade as parser changes. The exports are diffed as emitted: both sides share the corpus, the PHP binary, and the exporter, so the output is already deterministic. `prep-diff.php` is for comparing exports from different environments; its line-number and namespace-prefix erasure and its collection sorting would hide or scatter real changes here. To run it locally, get the pinned corpus: diff --git a/tools/corpus-diff-comment.sh b/tools/corpus-diff-comment.sh new file mode 100755 index 0000000..dd075d6 --- /dev/null +++ b/tools/corpus-diff-comment.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# +# Renders the corpus diff as GitHub-flavored Markdown for a pull request +# comment or a job summary. +# +# Usage: +# +# WP_CORPUS_TAG=7.0.4 BASE_SHA=... HEAD_SHA=... tools/corpus-diff-comment.sh corpus.diff > comment.md +# +# Optional: ARTIFACT_URL (download link for corpus.diff), RUN_URL (fallback +# link), MAX_BYTES (largest body that inlines the diff; GitHub rejects +# comments over 65536 characters, default 60000). +# +# The first line is an HTML comment that marks the output as this tool's, so +# the workflow can find and update its own comment instead of adding one per +# run. + +set -euo pipefail + +diff_file=$1 +max_bytes=${MAX_BYTES:-60000} +link=${ARTIFACT_URL:-${RUN_URL:-}} + +marker='' +heading='### Corpus diff' +compared="\`wp-includes@${WP_CORPUS_TAG}\`, parser at \`${BASE_SHA:0:7}\` (base) vs \`${HEAD_SHA:0:7}\` (this PR merged)" + +if [ ! -s "$diff_file" ]; then + printf '%s\n%s\n\n**0 hunks.** No behavior change over %s.\n' "$marker" "$heading" "$compared" + exit 0 +fi + +hunks=$(grep -c '^@@' "$diff_file" || true) +added=$(grep -c '^+[^+]' "$diff_file" || true) +removed=$(grep -c '^-[^-]' "$diff_file" || true) +lines=$(wc -l < "$diff_file" | tr -d ' ') +bytes=$(wc -c < "$diff_file" | tr -d ' ') + +stat="**${hunks} hunks**, \`+${added}\` \`-${removed}\` lines, over ${compared}. Every hunk must be intended and explained in the PR." +download='' +if [ -n "$link" ]; then + download=" [Download corpus.diff](${link})." +fi + +# A fence must be longer than any backtick run inside the diff. +ticks=$( { grep -o '`\{3,\}' "$diff_file" || true; } | awk '{ if ( length( $0 ) > m ) m = length( $0 ) } END { print m + 0 }') +fence=$(printf '%*s' $(( ticks > 2 ? ticks + 1 : 3 )) '' | tr ' ' '`') + +# Everything except the diff itself, to decide whether the diff fits. +frame=$(printf '%s\n%s\n\n%s%s\n\n
\ncorpus.diff (%s lines)\n\n%sdiff\n%s\n
\n' \ + "$marker" "$heading" "$stat" "$download" "$lines" "$fence" "$fence") + +if [ $(( ${#frame} + bytes )) -gt "$max_bytes" ]; then + printf '%s\n%s\n\n%s The diff (%s lines, %s bytes) is too large for a comment.%s\n' \ + "$marker" "$heading" "$stat" "$lines" "$bytes" "$download" + exit 0 +fi + +printf '%s\n%s\n\n%s%s\n\n
\ncorpus.diff (%s lines)\n\n%sdiff\n' \ + "$marker" "$heading" "$stat" "$download" "$lines" "$fence" +cat "$diff_file" +printf '%s\n
\n' "$fence" From 3a285597a48f617d433c0178fe18b5d063adadcd Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Fri, 21 Aug 2026 19:08:18 +0400 Subject: [PATCH 09/10] Cut an oversize corpus diff to the hunks that fit Instead of dropping the diff from the comment when it exceeds the size budget, keep the leading whole hunks that fit, and say how many of the total are shown. The download link still has the full diff. --- tools/corpus-diff-comment.sh | 48 +++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/tools/corpus-diff-comment.sh b/tools/corpus-diff-comment.sh index dd075d6..0beaa4a 100755 --- a/tools/corpus-diff-comment.sh +++ b/tools/corpus-diff-comment.sh @@ -8,8 +8,9 @@ # WP_CORPUS_TAG=7.0.4 BASE_SHA=... HEAD_SHA=... tools/corpus-diff-comment.sh corpus.diff > comment.md # # Optional: ARTIFACT_URL (download link for corpus.diff), RUN_URL (fallback -# link), MAX_BYTES (largest body that inlines the diff; GitHub rejects -# comments over 65536 characters, default 60000). +# link), MAX_BYTES (largest body to emit; GitHub rejects comments over 65536 +# characters, default 60000). A diff that does not fit is cut to the leading +# whole hunks that do, and the comment says so. # # The first line is an HTML comment that marks the output as this tool's, so # the workflow can find and update its own comment instead of adding one per @@ -17,6 +18,9 @@ set -euo pipefail +# Count bytes, not characters: GitHub's limit is characters, so bytes are safe. +export LC_ALL=C + diff_file=$1 max_bytes=${MAX_BYTES:-60000} link=${ARTIFACT_URL:-${RUN_URL:-}} @@ -36,7 +40,11 @@ removed=$(grep -c '^-[^-]' "$diff_file" || true) lines=$(wc -l < "$diff_file" | tr -d ' ') bytes=$(wc -c < "$diff_file" | tr -d ' ') -stat="**${hunks} hunks**, \`+${added}\` \`-${removed}\` lines, over ${compared}. Every hunk must be intended and explained in the PR." +hunk_word=hunks +if [ "$hunks" -eq 1 ]; then + hunk_word=hunk +fi +stat="**${hunks} ${hunk_word}**, \`+${added}\` \`-${removed}\` lines, over ${compared}. Every hunk must be intended and explained in the PR." download='' if [ -n "$link" ]; then download=" [Download corpus.diff](${link})." @@ -46,17 +54,35 @@ fi ticks=$( { grep -o '`\{3,\}' "$diff_file" || true; } | awk '{ if ( length( $0 ) > m ) m = length( $0 ) } END { print m + 0 }') fence=$(printf '%*s' $(( ticks > 2 ? ticks + 1 : 3 )) '' | tr ' ' '`') -# Everything except the diff itself, to decide whether the diff fits. -frame=$(printf '%s\n%s\n\n%s%s\n\n
\ncorpus.diff (%s lines)\n\n%sdiff\n%s\n
\n' \ - "$marker" "$heading" "$stat" "$download" "$lines" "$fence" "$fence") +# Everything except the diff itself, sized with the longer, truncated wording, +# decides how many bytes of diff fit. +note=" Only the first ${hunks} of ${hunks} hunks are shown below; the download has them all." +frame=$(printf '%s\n%s\n\n%s%s%s\n\n
\ncorpus.diff (first %s of %s hunks)\n\n%sdiff\n%s\n
\n' \ + "$marker" "$heading" "$stat" "$note" "$download" "$hunks" "$hunks" "$fence" "$fence") +budget=$(( max_bytes - ${#frame} )) + +# Last line and count of the leading whole hunks that fit the budget. +read -r keep_line keep_hunks < <(awk -v budget="$budget" ' + /^@@/ { if ( total <= budget ) { keep_line = NR - 1; keep_hunks = seen } seen++ } + { total += length( $0 ) + 1 } + END { if ( total <= budget ) { keep_line = NR; keep_hunks = seen } print keep_line + 0, keep_hunks + 0 } +' "$diff_file") -if [ $(( ${#frame} + bytes )) -gt "$max_bytes" ]; then - printf '%s\n%s\n\n%s The diff (%s lines, %s bytes) is too large for a comment.%s\n' \ +if [ "$keep_hunks" -eq 0 ]; then + printf '%s\n%s\n\n%s No hunk fits in a comment (%s lines, %s bytes).%s\n' \ "$marker" "$heading" "$stat" "$lines" "$bytes" "$download" exit 0 fi -printf '%s\n%s\n\n%s%s\n\n
\ncorpus.diff (%s lines)\n\n%sdiff\n' \ - "$marker" "$heading" "$stat" "$download" "$lines" "$fence" -cat "$diff_file" +if [ "$keep_hunks" -eq "$hunks" ]; then + note='' + summary="corpus.diff (${lines} lines)" +else + note=" Only the first ${keep_hunks} of ${hunks} hunks are shown below; the download has them all." + summary="corpus.diff (first ${keep_hunks} of ${hunks} hunks)" +fi + +printf '%s\n%s\n\n%s%s%s\n\n
\n%s\n\n%sdiff\n' \ + "$marker" "$heading" "$stat" "$note" "$download" "$summary" "$fence" +head -n "$keep_line" "$diff_file" printf '%s\n
\n' "$fence" From e74611e93f86db8f0086478d9f9eb943ccf5efeb Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Fri, 21 Aug 2026 19:11:58 +0400 Subject: [PATCH 10/10] Parse the whole WordPress release as the corpus wp-admin, the root files, and the bundled themes and plugins join wp-includes: 1880 PHP files instead of 1039, 64 MB of JSON instead of 49 MB, about 1.4 seconds more per export locally. More shapes of real code mean more regressions caught, and every hunk is still a parser behavior change. Only PHP files are kept in the cache; the cache key carries a layout suffix so the old wp-includes-only entry is not reused. --- .github/workflows/corpus-diff.yml | 29 +++++++++++++++++------------ README.md | 8 ++++---- tools/corpus-diff-comment.sh | 2 +- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/.github/workflows/corpus-diff.yml b/.github/workflows/corpus-diff.yml index 5469489..af0cfe3 100644 --- a/.github/workflows/corpus-diff.yml +++ b/.github/workflows/corpus-diff.yml @@ -57,30 +57,35 @@ jobs: php-version: "8.4" coverage: none + # The key carries the corpus layout: bump the suffix when the download + # step changes what it keeps. - name: Cache corpus id: cache-corpus uses: actions/cache@v4 with: path: corpus - key: wp-corpus-${{ env.WP_CORPUS_TAG }} + key: wp-corpus-${{ env.WP_CORPUS_TAG }}-php-all + # The whole release: wp-admin, wp-includes, the root files, and + # wp-content with the bundled themes and plugins. Only PHP files are + # kept; the parser reads nothing else. - name: Download corpus if: steps.cache-corpus.outputs.cache-hit != 'true' run: | curl -sSfL -o wordpress.zip "https://github.com/WordPress/WordPress/archive/refs/tags/${WP_CORPUS_TAG}.zip" - unzip -q wordpress.zip "WordPress-${WP_CORPUS_TAG}/wp-includes/*" - mkdir -p corpus - mv "WordPress-${WP_CORPUS_TAG}/wp-includes" corpus/wp-includes - rm -rf wordpress.zip "WordPress-${WP_CORPUS_TAG}" + unzip -q wordpress.zip + mv "WordPress-${WP_CORPUS_TAG}" corpus + find corpus -type f ! -name '*.php' -delete + rm wordpress.zip # An empty or miscached corpus would make both exports emit `[]` and the - # diff trivially empty — a green job that checked nothing. wp-includes - # has well over 500 PHP files on any supported tag. + # diff trivially empty, a green job that checked nothing. A release has + # well over 1500 PHP files on any supported tag. - name: Verify corpus run: | - count=$(find corpus/wp-includes -name '*.php' | wc -l) + count=$(find corpus -name '*.php' | wc -l) echo "Corpus PHP files: ${count}" - [ "$count" -ge 500 ] + [ "$count" -ge 1500 ] # On pull_request, HEAD is the PR merged into the base branch, so its # merge base with the base branch is the base branch tip, not the commit @@ -97,16 +102,16 @@ jobs: run: composer --working-dir=base install --no-interaction --no-security-blocking # The size floor guards the same failure mode as Verify corpus: a real - # wp-includes export is tens of megabytes of JSON. + # export is tens of megabytes of JSON. - name: Export corpus (base) run: | - php -d memory_limit=4G tools/export-corpus.php base corpus/wp-includes > base.json + php -d memory_limit=4G tools/export-corpus.php base corpus > base.json ls -l base.json [ "$(wc -c < base.json)" -ge 1000000 ] - name: Export corpus (head) run: | - php -d memory_limit=4G tools/export-corpus.php . corpus/wp-includes > head.json + php -d memory_limit=4G tools/export-corpus.php . corpus > head.json ls -l head.json [ "$(wc -c < head.json)" -ge 1000000 ] diff --git a/README.md b/README.md index 68efd39..aa8d26c 100644 --- a/README.md +++ b/README.md @@ -55,13 +55,13 @@ wp parser create /path/to/source/code --user= Unit tests do not cover every shape of real-world documentation, so changes to the parser are also checked against a corpus of WordPress core source. The same corpus is parsed with the parser at two refs, the base branch and the pull request merged into it, and the two JSON exports are diffed. Everything in that diff is a behavior change the pull request makes: every hunk must be either intended and explained, or it is a regression. -`.github/workflows/corpus-diff.yml` runs this on every pull request. The corpus is `wp-includes` from a pinned WordPress tag (`WP_CORPUS_TAG` in the workflow), so diffs are reproducible. The job is non-blocking: it uploads the diff as a `corpus.diff` artifact, reports it in the job summary, and posts one comment on the pull request, updated on every push, with the hunk and line counts and the diff itself when it fits in a comment (`tools/corpus-diff-comment.sh` renders it). Pull requests from forks and from Dependabot run with a read-only token, so they get the summary and the artifact only. The head checkout's `tools/export-corpus.php` drives both sides, so tooling changes never masquerade as parser changes. The exports are diffed as emitted: both sides share the corpus, the PHP binary, and the exporter, so the output is already deterministic. `prep-diff.php` is for comparing exports from different environments; its line-number and namespace-prefix erasure and its collection sorting would hide or scatter real changes here. +`.github/workflows/corpus-diff.yml` runs this on every pull request. The corpus is the PHP of a whole WordPress release at a pinned tag (`WP_CORPUS_TAG` in the workflow): wp-admin, wp-includes, the root files, and the bundled themes and plugins. Pinning keeps diffs reproducible. The job is non-blocking: it uploads the diff as a `corpus.diff` artifact, reports it in the job summary, and posts one comment on the pull request, updated on every push, with the hunk and line counts and the diff itself when it fits in a comment (`tools/corpus-diff-comment.sh` renders it). Pull requests from forks and from Dependabot run with a read-only token, so they get the summary and the artifact only. The head checkout's `tools/export-corpus.php` drives both sides, so tooling changes never masquerade as parser changes. The exports are diffed as emitted: both sides share the corpus, the PHP binary, and the exporter, so the output is already deterministic. `prep-diff.php` is for comparing exports from different environments; its line-number and namespace-prefix erasure and its collection sorting would hide or scatter real changes here. To run it locally, get the pinned corpus: ```bash curl -sSfL -o wordpress.zip https://github.com/WordPress/WordPress/archive/refs/tags/7.0.4.zip -unzip -q wordpress.zip 'WordPress-7.0.4/wp-includes/*' +unzip -q wordpress.zip ``` Check out the other side of the comparison and install its dependencies. The exporter runs under plain PHP — it does not load WordPress — but it needs a Composer autoloader in each parser root: @@ -78,8 +78,8 @@ Export both sides over the same corpus and diff. `export-corpus.php` takes the p ```bash export LC_ALL=C -php -d memory_limit=4G tools/export-corpus.php base WordPress-7.0.4/wp-includes > base.json -php -d memory_limit=4G tools/export-corpus.php . WordPress-7.0.4/wp-includes > head.json +php -d memory_limit=4G tools/export-corpus.php base WordPress-7.0.4 > base.json +php -d memory_limit=4G tools/export-corpus.php . WordPress-7.0.4 > head.json diff -u base.json head.json > corpus.diff ``` diff --git a/tools/corpus-diff-comment.sh b/tools/corpus-diff-comment.sh index 0beaa4a..1dc6bf8 100755 --- a/tools/corpus-diff-comment.sh +++ b/tools/corpus-diff-comment.sh @@ -27,7 +27,7 @@ link=${ARTIFACT_URL:-${RUN_URL:-}} marker='' heading='### Corpus diff' -compared="\`wp-includes@${WP_CORPUS_TAG}\`, parser at \`${BASE_SHA:0:7}\` (base) vs \`${HEAD_SHA:0:7}\` (this PR merged)" +compared="WordPress ${WP_CORPUS_TAG}, parser at \`${BASE_SHA:0:7}\` (base) vs \`${HEAD_SHA:0:7}\` (this PR merged)" if [ ! -s "$diff_file" ]; then printf '%s\n%s\n\n**0 hunks.** No behavior change over %s.\n' "$marker" "$heading" "$compared"