From 3806b1c8ab5219af1b97bbe08df6dfbcaea11f28 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Tue, 18 Aug 2026 18:32:43 +0400 Subject: [PATCH 01/14] Add corpus-diff CI: regenerate a pinned WordPress corpus and diff it --- .github/workflows/corpus-diff.yml | 99 +++++++++++++++++++++++++++++++ tools/export-corpus.php | 48 +++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 .github/workflows/corpus-diff.yml create mode 100644 tools/export-corpus.php diff --git a/.github/workflows/corpus-diff.yml b/.github/workflows/corpus-diff.yml new file mode 100644 index 0000000..cacf74e --- /dev/null +++ b/.github/workflows/corpus-diff.yml @@ -0,0 +1,99 @@ +# Parses a pinned corpus of WordPress core source with the parser at the PR's +# merge base and at its head, normalizes both JSON outputs with prep-diff.php, +# and diffs them. Anything in the diff is a behavior change this PR makes: +# every hunk must be either intended (and explained in the PR) or a regression. +# +# Policy decisions, deliberate: +# - The corpus is pinned to one WordPress tag so diffs are reproducible. +# - The head checkout's tools/export-corpus.php and prep-diff.php drive both +# sides, so tooling changes never masquerade as parser changes. When +# prep-diff.php itself changes, its effect on normalization shows up in the +# diff and is reviewed like any other change. +# - Non-blocking: the job succeeds even when the diff is non-empty. The diff +# is published as an artifact and summarized. Make it blocking only after +# the signal has proven trustworthy. +# - PHP 7.4, the supported floor, keeps parity with the oldest runtime. + +name: Corpus Diff + +on: + pull_request: + +jobs: + corpus-diff: + name: WordPress corpus regeneration diff + runs-on: ubuntu-latest + + env: + WP_CORPUS_TAG: "6.8" + LC_ALL: C + + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Set up PHP + uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # 2.37.2 + with: + php-version: "7.4" + coverage: none + + - name: Cache corpus + id: cache-corpus + uses: actions/cache@v4 + with: + path: corpus + key: wp-corpus-${{ env.WP_CORPUS_TAG }} + + - name: Download corpus + if: steps.cache-corpus.outputs.cache-hit != 'true' + run: | + curl -sSfL -o wordpress.zip "https://github.com/WordPress/WordPress/archive/refs/tags/${WP_CORPUS_TAG}.zip" + unzip -q wordpress.zip "WordPress-${WP_CORPUS_TAG}/wp-includes/*" + mkdir -p corpus + mv "WordPress-${WP_CORPUS_TAG}/wp-includes" corpus/wp-includes + rm -rf wordpress.zip "WordPress-${WP_CORPUS_TAG}" + + - name: Check out merge base + run: git worktree add base "$(git merge-base "origin/${{ github.base_ref }}" HEAD)" + + - name: Install Composer dependencies (head) + run: composer install --no-interaction --no-security-blocking + + - name: Install Composer dependencies (base) + run: composer --working-dir=base install --no-interaction --no-security-blocking + + - name: Export corpus (base) + run: php -d memory_limit=4G tools/export-corpus.php base corpus/wp-includes > base.json + + - name: Export corpus (head) + run: php -d memory_limit=4G tools/export-corpus.php . corpus/wp-includes > head.json + + - name: Normalize and diff + run: | + php -d memory_limit=4G prep-diff.php < base.json > base.norm.json + php -d memory_limit=4G prep-diff.php < head.json > head.norm.json + # diff exits 1 on differences (expected) and 2 on trouble (fail). + diff -u --label base --label head base.norm.json head.norm.json > corpus.diff || [ $? -eq 1 ] + if [ -s corpus.diff ]; then + hunks=$(grep -c '^@@' corpus.diff) + lines=$(wc -l < corpus.diff) + { + echo "### Corpus diff: ${hunks} hunks, ${lines} lines" + echo + echo "The parser's output over wp-includes@${WP_CORPUS_TAG} changed." + echo "Review the \`corpus.diff\` artifact: every hunk must be intended and explained in the PR." + } >> "$GITHUB_STEP_SUMMARY" + else + echo "### Corpus diff: 0 hunks (no behavior change)" >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload diff + if: always() + uses: actions/upload-artifact@v4 + with: + name: corpus.diff + path: corpus.diff + if-no-files-found: ignore diff --git a/tools/export-corpus.php b/tools/export-corpus.php new file mode 100644 index 0000000..8bf062e --- /dev/null +++ b/tools/export-corpus.php @@ -0,0 +1,48 @@ + > corpus.json + */ + +// Keep stdout pure JSON: vendored code emits deprecation notices on newer +// PHP, and display_errors otherwise interleaves them with the output. +ini_set( 'display_errors', 'stderr' ); + +if ( $argc < 3 ) { + fwrite( STDERR, 'Usage: php tools/export-corpus.php ' . PHP_EOL ); + exit( 1 ); +} + +$parser_root = rtrim( $argv[1], '/' ); +$corpus_dir = rtrim( $argv[2], '/' ); + +if ( ! is_file( $parser_root . '/vendor/autoload.php' ) ) { + fwrite( STDERR, 'No Composer autoloader in ' . $parser_root . '; run composer install there first.' . PHP_EOL ); + exit( 1 ); +} + +if ( ! is_dir( $corpus_dir ) ) { + fwrite( STDERR, 'Corpus directory not found: ' . $corpus_dir . PHP_EOL ); + exit( 1 ); +} + +require $parser_root . '/vendor/autoload.php'; + +$files = \WP_Parser\get_wp_files( $corpus_dir ); + +if ( ! is_array( $files ) ) { + fwrite( STDERR, 'Could not list the PHP files in ' . $corpus_dir . '.' . PHP_EOL ); + exit( 1 ); +} + +echo json_encode( \WP_Parser\parse_files( $files, $corpus_dir ), JSON_PRETTY_PRINT ), PHP_EOL; From 09fa3552c4d2a68c9a507da61c6b1390bbf9fb79 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Tue, 18 Aug 2026 18:45:54 +0400 Subject: [PATCH 02/14] Guard the corpus-diff job against a silently empty corpus --- .github/workflows/corpus-diff.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/corpus-diff.yml b/.github/workflows/corpus-diff.yml index cacf74e..c7245f0 100644 --- a/.github/workflows/corpus-diff.yml +++ b/.github/workflows/corpus-diff.yml @@ -56,6 +56,15 @@ jobs: mv "WordPress-${WP_CORPUS_TAG}/wp-includes" corpus/wp-includes rm -rf wordpress.zip "WordPress-${WP_CORPUS_TAG}" + # An empty or miscached corpus would make both exports emit `[]` and the + # diff trivially empty — a green job that checked nothing. wp-includes + # has well over 500 PHP files on any supported tag. + - name: Verify corpus + run: | + count=$(find corpus/wp-includes -name '*.php' | wc -l) + echo "Corpus PHP files: ${count}" + [ "$count" -ge 500 ] + - name: Check out merge base run: git worktree add base "$(git merge-base "origin/${{ github.base_ref }}" HEAD)" @@ -65,11 +74,19 @@ jobs: - name: Install Composer dependencies (base) run: composer --working-dir=base install --no-interaction --no-security-blocking + # The size floor guards the same failure mode as Verify corpus: a real + # wp-includes export is tens of megabytes of JSON. - name: Export corpus (base) - run: php -d memory_limit=4G tools/export-corpus.php base corpus/wp-includes > base.json + run: | + php -d memory_limit=4G tools/export-corpus.php base corpus/wp-includes > base.json + ls -l base.json + [ "$(wc -c < base.json)" -ge 1000000 ] - name: Export corpus (head) - run: php -d memory_limit=4G tools/export-corpus.php . corpus/wp-includes > head.json + run: | + php -d memory_limit=4G tools/export-corpus.php . corpus/wp-includes > head.json + ls -l head.json + [ "$(wc -c < head.json)" -ge 1000000 ] - name: Normalize and diff run: | From 66ca1eb6a6e9876d92c3b3aabc5dfb6db84f55dc Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Tue, 18 Aug 2026 21:45:41 +0400 Subject: [PATCH 03/14] Pin the corpus to WordPress 7.0.4 7.0.4 is the latest WordPress release. Pinning the point release rather than the 7.0 major tag tracks the patches to wp-includes, so the corpus matches what is shipped. WordPress 7.0 raised core's minimum PHP to 7.4, which is exactly this job's PHP floor. Verified against wp-includes@7.0.4: 1039 PHP files, all of them present in the export, no parse errors, ~49 MB of JSON. Both of the job's guards (>= 500 files, >= 1 MB of JSON) still hold. --- .github/workflows/corpus-diff.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/corpus-diff.yml b/.github/workflows/corpus-diff.yml index c7245f0..0217269 100644 --- a/.github/workflows/corpus-diff.yml +++ b/.github/workflows/corpus-diff.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest env: - WP_CORPUS_TAG: "6.8" + WP_CORPUS_TAG: "7.0.4" LC_ALL: C steps: From 1c23bb71a6a052169a0c2aaa551970a1dc8b451d Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Tue, 18 Aug 2026 21:46:29 +0400 Subject: [PATCH 04/14] Document the corpus diff procedure Records what the corpus-diff check is, how to run it by hand with the repo's own tools/export-corpus.php and prep-diff.php, and the policy the workflow already states: the head checkout's tooling drives both sides. --- README.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/README.md b/README.md index ab35ee4..0057f8b 100644 --- a/README.md +++ b/README.md @@ -50,3 +50,37 @@ In your site's directory: ```bash wp parser create /path/to/source/code --user= ``` + +## Corpus diff + +Unit tests do not cover every shape of real-world documentation, so changes to the parser are also checked against a corpus of WordPress core source. The same corpus is parsed with the parser at two refs — the merge base of a pull request and its head — both JSON outputs are normalized with `prep-diff.php`, and the two are diffed. Everything in that diff is a behavior change the pull request makes: every hunk must be either intended and explained, or it is a regression. + +`.github/workflows/corpus-diff.yml` runs this on every pull request. The corpus is `wp-includes` from a pinned WordPress tag (`WP_CORPUS_TAG` in the workflow), so diffs are reproducible. The job is non-blocking: it uploads the diff as a `corpus.diff` artifact and reports the hunk count in the job summary. The head checkout's `tools/export-corpus.php` and `prep-diff.php` drive both sides, so tooling changes never masquerade as parser changes; when `prep-diff.php` itself changes, its effect on normalization shows up in the diff and is reviewed like any other change. + +To run it locally, get the pinned corpus: + +```bash +curl -sSfL -o wordpress.zip https://github.com/WordPress/WordPress/archive/refs/tags/7.0.4.zip +unzip -q wordpress.zip 'WordPress-7.0.4/wp-includes/*' +``` + +Check out the other side of the comparison and install its dependencies. The exporter runs under plain PHP — it does not load WordPress — but it needs a Composer autoloader in each parser root: + +```bash +git worktree add base "$(git merge-base origin/master HEAD)" +composer --working-dir=base install +composer install +``` + +Export both sides over the same corpus, normalize, and diff. `export-corpus.php` takes the parser root and the corpus directory, and writes JSON to stdout: + +```bash +export LC_ALL=C +php -d memory_limit=4G tools/export-corpus.php base WordPress-7.0.4/wp-includes > base.json +php -d memory_limit=4G tools/export-corpus.php . WordPress-7.0.4/wp-includes > head.json +php -d memory_limit=4G prep-diff.php < base.json > base.norm.json +php -d memory_limit=4G prep-diff.php < head.json > head.norm.json +diff -u base.norm.json head.norm.json > corpus.diff +``` + +An empty `corpus.diff` means the change has no effect on parser output. From 931fcc392b693585a1b80327606b05d3d65c2a17 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Tue, 18 Aug 2026 21:48:59 +0400 Subject: [PATCH 05/14] Add a scheduled workflow that keeps the corpus pin current --- .github/workflows/corpus-pin-update.yml | 169 ++++++++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 .github/workflows/corpus-pin-update.yml diff --git a/.github/workflows/corpus-pin-update.yml b/.github/workflows/corpus-pin-update.yml new file mode 100644 index 0000000..a51e79e --- /dev/null +++ b/.github/workflows/corpus-pin-update.yml @@ -0,0 +1,169 @@ +# Keeps `WP_CORPUS_TAG` in corpus-diff.yml pinned to the current WordPress +# stable. Dependabot cannot track WordPress core releases, so this does it: +# weekly, ask api.wordpress.org for the latest stable, and open a PR that +# rewrites the one pin line when it has moved. +# +# Why a bump PR is cheap to review, deliberately: it changes only the corpus +# input, never the parser. Its own corpus-diff run therefore parses the new +# corpus with an identical parser on both sides — 0 hunks by construction. +# Reviewing a bump PR is checking that the corpus guards (file count, export +# size) still pass on the new tag. A non-zero diff on one of these PRs would +# mean corpus-diff is not comparing what it claims to. +# +# Dependency, and the likely failure: creating the PR needs "Allow GitHub +# Actions to create and approve pull requests" (Settings -> Actions -> General +# -> Workflow permissions). Many organizations disable it. If it is off, the +# "Open the bump PR" step fails loudly with that setting named; enable it, or +# give the step a PAT as GH_TOKEN. The failure is never swallowed: a silent +# no-op would leave the corpus quietly rotting on an old tag. + +name: Corpus Pin Update + +on: + schedule: + # Weekly, Mondays. Off the hour to dodge the top-of-hour scheduling queue. + - cron: "43 5 * * 1" + workflow_dispatch: + +concurrency: + group: corpus-pin-update + cancel-in-progress: false + +jobs: + corpus-pin-update: + name: Bump the pinned WordPress corpus + # Forks inherit schedules; only the canonical repo should open these PRs. + if: github.repository == 'WordPress/phpdoc-parser' + runs-on: ubuntu-latest + + permissions: + contents: write + pull-requests: write + + env: + WORKFLOW_FILE: .github/workflows/corpus-diff.yml + # Canonical, and authoritative about what "stable" means: it excludes + # betas and RCs, which a raw tag listing of WordPress/WordPress does not. + VERSION_CHECK_API: https://api.wordpress.org/core/version-check/1.7/ + CORPUS_MIRROR: https://github.com/WordPress/WordPress.git + LC_ALL: C + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Resolve the latest WordPress stable + id: resolve + run: | + set -euo pipefail + latest="$(curl -sSfL --retry 3 --retry-delay 5 "${VERSION_CHECK_API}" | jq -r '.offers[0].current // ""')" + # An API hiccup must not become a garbage commit. + if ! printf '%s' "${latest}" | grep -Eq '^[0-9]+\.[0-9]+(\.[0-9]+)?$'; then + echo "::error::${VERSION_CHECK_API} did not yield a usable version (got: '${latest}')." + exit 1 + fi + echo "Latest WordPress stable: ${latest}" + echo "latest=${latest}" >> "${GITHUB_OUTPUT}" + + - name: Compare against the pinned tag + id: pin + env: + LATEST: ${{ steps.resolve.outputs.latest }} + run: | + set -euo pipefail + current="$(sed -n 's/^ *WP_CORPUS_TAG: *"\([^"]*\)".*/\1/p' "${WORKFLOW_FILE}")" + if ! printf '%s' "${current}" | grep -Eq '^[0-9]+\.[0-9]+(\.[0-9]+)?$'; then + echo "::error::Could not read a single WP_CORPUS_TAG version out of ${WORKFLOW_FILE} (got: '${current}')." + exit 1 + fi + echo "Pinned corpus tag: ${current}" + + # Three reasons not to bump, in order: nothing moved; the pin is + # ahead of the published stable, which is deliberate (someone pinned + # a prerelease) and must not be quietly walked back; or the + # WordPress/WordPress mirror corpus-diff downloads from has not + # tagged the release yet, in which case the next run picks it up + # rather than this one opening a PR whose corpus cannot be built. + bump=false + if [ "${current}" = "${LATEST}" ]; then + echo "Already pinned to the latest stable; nothing to do." + elif [ "$(printf '%s\n%s\n' "${current}" "${LATEST}" | sort -V | tail -n 1)" != "${LATEST}" ]; then + echo "Pinned ${current} is newer than the published stable ${LATEST}; leaving it alone." + elif ! git ls-remote --exit-code --tags "${CORPUS_MIRROR}" "refs/tags/${LATEST}" > /dev/null; then + echo "${CORPUS_MIRROR} has no ${LATEST} tag yet; will retry on the next run." + else + echo "Bumping the corpus pin from ${current} to ${LATEST}." + bump=true + fi + + { + echo "bump=${bump}" + echo "current=${current}" + } >> "${GITHUB_OUTPUT}" + + - name: Rewrite the pin + if: steps.pin.outputs.bump == 'true' + env: + LATEST: ${{ steps.resolve.outputs.latest }} + run: | + set -euo pipefail + sed "s/^\( *WP_CORPUS_TAG: *\).*/\1\"${LATEST}\"/" "${WORKFLOW_FILE}" > "${WORKFLOW_FILE}.tmp" + mv "${WORKFLOW_FILE}.tmp" "${WORKFLOW_FILE}" + + # The rewrite must be exactly the pin line and nothing else. + check="$(sed -n 's/^ *WP_CORPUS_TAG: *"\([^"]*\)".*/\1/p' "${WORKFLOW_FILE}")" + added="$(git diff --numstat -- "${WORKFLOW_FILE}" | awk '{print $1}')" + removed="$(git diff --numstat -- "${WORKFLOW_FILE}" | awk '{print $2}')" + if [ "${check}" != "${LATEST}" ] || [ "${added}" != "1" ] || [ "${removed}" != "1" ]; then + echo "::error::Pin rewrite did not produce a single-line change to ${LATEST} (read back '${check}', +${added:-0}/-${removed:-0})." + git --no-pager diff -- "${WORKFLOW_FILE}" + exit 1 + fi + git --no-pager diff -- "${WORKFLOW_FILE}" + + - name: Open the bump PR + if: steps.pin.outputs.bump == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BASE_BRANCH: ${{ github.event.repository.default_branch }} + CURRENT: ${{ steps.pin.outputs.current }} + LATEST: ${{ steps.resolve.outputs.latest }} + run: | + set -euo pipefail + branch="bump-corpus-pin-${LATEST}" + + # Idempotent: any prior PR for this version, open or closed, means the + # bump has already been proposed and possibly declined. Leave it be. + existing="$(gh pr list --repo "${GITHUB_REPOSITORY}" --head "${branch}" --state all --json url --jq '.[0].url // ""')" + if [ -n "${existing}" ]; then + echo "A bump PR for ${LATEST} already exists: ${existing}" + exit 0 + fi + + title="Bump the pinned WordPress corpus to ${LATEST}" + cat > pr-body.md <&1)"; then + printf '%s\n' "${url}" >&2 + echo "::error::Pushed ${branch} but could not open the PR. The usual cause is 'Allow GitHub Actions to create and approve pull requests' being disabled for this repository or organization (Settings -> Actions -> General -> Workflow permissions). Enable it, or supply a PAT with 'repo' scope as GH_TOKEN for this step, then re-run this workflow." + exit 1 + fi + + echo "Opened ${url}" + echo "Opened [${title}](${url})" >> "${GITHUB_STEP_SUMMARY}" From ac7f6b7a3bdd0304d518c7550bd97e216ec155bc Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Thu, 20 Aug 2026 18:16:27 +0400 Subject: [PATCH 06/14] Address corpus-diff review notes - Name the comparison for what it is: on pull_request HEAD is the PR merged into the base branch, so the merge base resolves to the base branch tip. Rename the step, explain it in the workflow, and say the same in the README, including how to reproduce that comparison locally. - Drop the unreachable is_array() guard in tools/export-corpus.php: get_wp_files() returns a WP_Error on failure, which does not exist under plain PHP, so that path fatals with a non-zero exit before it can return. - Add a concurrency group so a new push cancels the in-flight run for the same PR. --- .github/workflows/corpus-diff.yml | 21 ++++++++++++++++----- README.md | 4 +++- tools/export-corpus.php | 8 +++----- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/.github/workflows/corpus-diff.yml b/.github/workflows/corpus-diff.yml index 0217269..fdf0635 100644 --- a/.github/workflows/corpus-diff.yml +++ b/.github/workflows/corpus-diff.yml @@ -1,7 +1,8 @@ -# Parses a pinned corpus of WordPress core source with the parser at the PR's -# merge base and at its head, normalizes both JSON outputs with prep-diff.php, -# and diffs them. Anything in the diff is a behavior change this PR makes: -# every hunk must be either intended (and explained in the PR) or a regression. +# Parses a pinned corpus of WordPress core source with the parser at the base +# branch and with the PR merged into it, normalizes both JSON outputs with +# prep-diff.php, and diffs them. Anything in the diff is a behavior change this +# PR makes: every hunk must be either intended (and explained in the PR) or a +# regression. # # Policy decisions, deliberate: # - The corpus is pinned to one WordPress tag so diffs are reproducible. @@ -19,6 +20,11 @@ name: Corpus Diff on: pull_request: +# A new push to the PR supersedes any run still in flight for it. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + jobs: corpus-diff: name: WordPress corpus regeneration diff @@ -65,7 +71,12 @@ jobs: echo "Corpus PHP files: ${count}" [ "$count" -ge 500 ] - - name: Check out merge base + # On pull_request, HEAD is the PR merged into the base branch, so its + # merge base with the base branch is the base branch tip, not the commit + # the PR branched from. That is the intended comparison: base as it is + # now against base plus exactly this PR, with no hunks from other work + # that landed since the branch point. + - name: Check out base run: git worktree add base "$(git merge-base "origin/${{ github.base_ref }}" HEAD)" - name: Install Composer dependencies (head) diff --git a/README.md b/README.md index 0057f8b..173d6be 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ wp parser create /path/to/source/code --user= ## Corpus diff -Unit tests do not cover every shape of real-world documentation, so changes to the parser are also checked against a corpus of WordPress core source. The same corpus is parsed with the parser at two refs — the merge base of a pull request and its head — both JSON outputs are normalized with `prep-diff.php`, and the two are diffed. Everything in that diff is a behavior change the pull request makes: every hunk must be either intended and explained, or it is a regression. +Unit tests do not cover every shape of real-world documentation, so changes to the parser are also checked against a corpus of WordPress core source. The same corpus is parsed with the parser at two refs — the base branch and the pull request merged into it — both JSON outputs are normalized with `prep-diff.php`, and the two are diffed. Everything in that diff is a behavior change the pull request makes: every hunk must be either intended and explained, or it is a regression. `.github/workflows/corpus-diff.yml` runs this on every pull request. The corpus is `wp-includes` from a pinned WordPress tag (`WP_CORPUS_TAG` in the workflow), so diffs are reproducible. The job is non-blocking: it uploads the diff as a `corpus.diff` artifact and reports the hunk count in the job summary. The head checkout's `tools/export-corpus.php` and `prep-diff.php` drive both sides, so tooling changes never masquerade as parser changes; when `prep-diff.php` itself changes, its effect on normalization shows up in the diff and is reviewed like any other change. @@ -72,6 +72,8 @@ composer --working-dir=base install composer install ``` +On a branch this compares against the commit the branch left `master` at. CI runs on the pull request merged into `master`, so it compares against the `master` tip; merge `master` into the branch first to get the same comparison. + Export both sides over the same corpus, normalize, and diff. `export-corpus.php` takes the parser root and the corpus directory, and writes JSON to stdout: ```bash diff --git a/tools/export-corpus.php b/tools/export-corpus.php index 8bf062e..bb771f9 100644 --- a/tools/export-corpus.php +++ b/tools/export-corpus.php @@ -38,11 +38,9 @@ require $parser_root . '/vendor/autoload.php'; +// get_wp_files() reports an unreadable subdirectory with a WP_Error, a class +// that only exists inside WordPress: under plain PHP that path is a fatal +// error with a non-zero exit, so there is no return value to check here. $files = \WP_Parser\get_wp_files( $corpus_dir ); -if ( ! is_array( $files ) ) { - fwrite( STDERR, 'Could not list the PHP files in ' . $corpus_dir . '.' . PHP_EOL ); - exit( 1 ); -} - echo json_encode( \WP_Parser\parse_files( $files, $corpus_dir ), JSON_PRETTY_PRINT ), PHP_EOL; From 2df6c27b6b53200bd588a507c7865466b070277a Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Thu, 20 Aug 2026 18:17:31 +0400 Subject: [PATCH 07/14] Run the corpus diff on PHP 8.4 Both sides run under the same binary, so the PHP version is not part of the comparison; use the newest runtime in the unit-test matrix. --- .github/workflows/corpus-diff.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/corpus-diff.yml b/.github/workflows/corpus-diff.yml index fdf0635..265ae88 100644 --- a/.github/workflows/corpus-diff.yml +++ b/.github/workflows/corpus-diff.yml @@ -13,7 +13,8 @@ # - Non-blocking: the job succeeds even when the diff is non-empty. The diff # is published as an artifact and summarized. Make it blocking only after # the signal has proven trustworthy. -# - PHP 7.4, the supported floor, keeps parity with the oldest runtime. +# - PHP 8.4, the newest runtime in the unit-test matrix. Both sides run under +# the same binary, so the PHP version never shows up as a parser change. name: Corpus Diff @@ -43,7 +44,7 @@ jobs: - name: Set up PHP uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # 2.37.2 with: - php-version: "7.4" + php-version: "8.4" coverage: none - name: Cache corpus From 8afb662f1ddf0f487acbb55cecd20e2f4422f397 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Fri, 21 Aug 2026 13:50:40 +0400 Subject: [PATCH 08/14] Diff the corpus exports without prep-diff.php Both sides share the corpus, the PHP binary, and the exporter, so the export is already deterministic. prep-diff.php reconciles exports from different environments: it zeroes line numbers, strips global-namespace prefixes, and sorts collections. Measured against three open PRs, the raw diff was equal or smaller in every case and kept each change at its source location, while the collection sort relocated changed records and the erasures hid classes of real change. Diff the exports as emitted. --- .github/workflows/corpus-diff.yml | 24 ++++++++++++------------ README.md | 10 ++++------ 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/.github/workflows/corpus-diff.yml b/.github/workflows/corpus-diff.yml index 265ae88..f14ee21 100644 --- a/.github/workflows/corpus-diff.yml +++ b/.github/workflows/corpus-diff.yml @@ -1,15 +1,17 @@ # Parses a pinned corpus of WordPress core source with the parser at the base -# branch and with the PR merged into it, normalizes both JSON outputs with -# prep-diff.php, and diffs them. Anything in the diff is a behavior change this -# PR makes: every hunk must be either intended (and explained in the PR) or a -# regression. +# branch and with the PR merged into it, and diffs the two JSON exports. +# Anything in the diff is a behavior change this PR makes: every hunk must be +# either intended (and explained in the PR) or a regression. # # Policy decisions, deliberate: # - The corpus is pinned to one WordPress tag so diffs are reproducible. -# - The head checkout's tools/export-corpus.php and prep-diff.php drive both -# sides, so tooling changes never masquerade as parser changes. When -# prep-diff.php itself changes, its effect on normalization shows up in the -# diff and is reviewed like any other change. +# - The head checkout's tools/export-corpus.php drives both sides, so tooling +# changes never masquerade as parser changes. +# - The exports are diffed as emitted, without prep-diff.php. Both sides share +# the corpus, the PHP binary, and the exporter, so the export is already +# deterministic; prep-diff.php exists to reconcile exports from different +# environments, and its erasures (line numbers, global-namespace prefixes) +# and collection sorting would hide or scatter real changes here. # - Non-blocking: the job succeeds even when the diff is non-empty. The diff # is published as an artifact and summarized. Make it blocking only after # the signal has proven trustworthy. @@ -100,12 +102,10 @@ jobs: ls -l head.json [ "$(wc -c < head.json)" -ge 1000000 ] - - name: Normalize and diff + - name: Diff run: | - php -d memory_limit=4G prep-diff.php < base.json > base.norm.json - php -d memory_limit=4G prep-diff.php < head.json > head.norm.json # diff exits 1 on differences (expected) and 2 on trouble (fail). - diff -u --label base --label head base.norm.json head.norm.json > corpus.diff || [ $? -eq 1 ] + diff -u --label base --label head base.json head.json > corpus.diff || [ $? -eq 1 ] if [ -s corpus.diff ]; then hunks=$(grep -c '^@@' corpus.diff) lines=$(wc -l < corpus.diff) diff --git a/README.md b/README.md index 173d6be..eb4087c 100644 --- a/README.md +++ b/README.md @@ -53,9 +53,9 @@ wp parser create /path/to/source/code --user= ## Corpus diff -Unit tests do not cover every shape of real-world documentation, so changes to the parser are also checked against a corpus of WordPress core source. The same corpus is parsed with the parser at two refs — the base branch and the pull request merged into it — both JSON outputs are normalized with `prep-diff.php`, and the two are diffed. Everything in that diff is a behavior change the pull request makes: every hunk must be either intended and explained, or it is a regression. +Unit tests do not cover every shape of real-world documentation, so changes to the parser are also checked against a corpus of WordPress core source. The same corpus is parsed with the parser at two refs, the base branch and the pull request merged into it, and the two JSON exports are diffed. Everything in that diff is a behavior change the pull request makes: every hunk must be either intended and explained, or it is a regression. -`.github/workflows/corpus-diff.yml` runs this on every pull request. The corpus is `wp-includes` from a pinned WordPress tag (`WP_CORPUS_TAG` in the workflow), so diffs are reproducible. The job is non-blocking: it uploads the diff as a `corpus.diff` artifact and reports the hunk count in the job summary. The head checkout's `tools/export-corpus.php` and `prep-diff.php` drive both sides, so tooling changes never masquerade as parser changes; when `prep-diff.php` itself changes, its effect on normalization shows up in the diff and is reviewed like any other change. +`.github/workflows/corpus-diff.yml` runs this on every pull request. The corpus is `wp-includes` from a pinned WordPress tag (`WP_CORPUS_TAG` in the workflow), so diffs are reproducible. The job is non-blocking: it uploads the diff as a `corpus.diff` artifact and reports the hunk count in the job summary. The head checkout's `tools/export-corpus.php` drives both sides, so tooling changes never masquerade as parser changes. The exports are diffed as emitted: both sides share the corpus, the PHP binary, and the exporter, so the output is already deterministic. `prep-diff.php` is for comparing exports from different environments; its line-number and namespace-prefix erasure and its collection sorting would hide or scatter real changes here. To run it locally, get the pinned corpus: @@ -74,15 +74,13 @@ composer install On a branch this compares against the commit the branch left `master` at. CI runs on the pull request merged into `master`, so it compares against the `master` tip; merge `master` into the branch first to get the same comparison. -Export both sides over the same corpus, normalize, and diff. `export-corpus.php` takes the parser root and the corpus directory, and writes JSON to stdout: +Export both sides over the same corpus and diff. `export-corpus.php` takes the parser root and the corpus directory, and writes JSON to stdout: ```bash export LC_ALL=C php -d memory_limit=4G tools/export-corpus.php base WordPress-7.0.4/wp-includes > base.json php -d memory_limit=4G tools/export-corpus.php . WordPress-7.0.4/wp-includes > head.json -php -d memory_limit=4G prep-diff.php < base.json > base.norm.json -php -d memory_limit=4G prep-diff.php < head.json > head.norm.json -diff -u base.norm.json head.norm.json > corpus.diff +diff -u base.json head.json > corpus.diff ``` An empty `corpus.diff` means the change has no effect on parser output. From 878dadb96c65c2372a7c291c5e5090ec9409714e Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Fri, 21 Aug 2026 18:32:58 +0400 Subject: [PATCH 09/14] Post the corpus diff as a pull request comment tools/corpus-diff-comment.sh renders the diff as Markdown: hunk and line counts, the compared commits, a download link, and the diff in a collapsed diff block when the body fits under GitHub's comment limit; otherwise the counts and the link alone. The workflow writes the same report to the job summary and creates or updates a single PR comment, found by a marker on its first line. pull_request runs for forks and Dependabot get a read-only token, so the comment step is skipped for them. --- .github/workflows/corpus-diff.yml | 53 +++++++++++++++++++------- README.md | 2 +- tools/corpus-diff-comment.sh | 62 +++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 15 deletions(-) create mode 100755 tools/corpus-diff-comment.sh diff --git a/.github/workflows/corpus-diff.yml b/.github/workflows/corpus-diff.yml index f14ee21..5469489 100644 --- a/.github/workflows/corpus-diff.yml +++ b/.github/workflows/corpus-diff.yml @@ -13,8 +13,12 @@ # environments, and its erasures (line numbers, global-namespace prefixes) # and collection sorting would hide or scatter real changes here. # - Non-blocking: the job succeeds even when the diff is non-empty. The diff -# is published as an artifact and summarized. Make it blocking only after -# the signal has proven trustworthy. +# is published as an artifact, summarized, and posted as a PR comment that +# is updated in place on every run. Make it blocking only after the signal +# has proven trustworthy. +# - The comment needs a token that can write to the PR. pull_request runs for +# forks and for Dependabot get a read-only token, so those PRs get the +# summary and the artifact only. # - PHP 8.4, the newest runtime in the unit-test matrix. Both sides run under # the same binary, so the PHP version never shows up as a parser change. @@ -28,6 +32,10 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} cancel-in-progress: true +permissions: + contents: read + pull-requests: write + jobs: corpus-diff: name: WordPress corpus regeneration diff @@ -106,23 +114,40 @@ jobs: run: | # diff exits 1 on differences (expected) and 2 on trouble (fail). diff -u --label base --label head base.json head.json > corpus.diff || [ $? -eq 1 ] - if [ -s corpus.diff ]; then - hunks=$(grep -c '^@@' corpus.diff) - lines=$(wc -l < corpus.diff) - { - echo "### Corpus diff: ${hunks} hunks, ${lines} lines" - echo - echo "The parser's output over wp-includes@${WP_CORPUS_TAG} changed." - echo "Review the \`corpus.diff\` artifact: every hunk must be intended and explained in the PR." - } >> "$GITHUB_STEP_SUMMARY" - else - echo "### Corpus diff: 0 hunks (no behavior change)" >> "$GITHUB_STEP_SUMMARY" - fi + echo "Corpus diff: $(grep -c '^@@' corpus.diff || true) hunks, $(wc -l < corpus.diff) lines" - name: Upload diff + id: upload if: always() uses: actions/upload-artifact@v4 with: name: corpus.diff path: corpus.diff if-no-files-found: ignore + + - name: Render report + env: + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + ARTIFACT_URL: ${{ steps.upload.outputs.artifact-url }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + BASE_SHA="$(git -C base rev-parse HEAD)" tools/corpus-diff-comment.sh corpus.diff > comment.md + tail -n +2 comment.md >> "$GITHUB_STEP_SUMMARY" + + # Create the comment on the first run and update it on every later one, + # so the PR carries one report that reflects the latest push. + - name: Comment on the pull request + if: github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + PR: ${{ github.event.pull_request.number }} + run: | + ids=$(gh api "repos/${REPO}/issues/${PR}/comments" --paginate \ + --jq '.[] | select(.body | startswith("")) | .id') + id=${ids%%$'\n'*} + if [ -n "$id" ]; then + gh api --silent -X PATCH "repos/${REPO}/issues/comments/${id}" -F body=@comment.md + else + gh api --silent -X POST "repos/${REPO}/issues/${PR}/comments" -F body=@comment.md + fi diff --git a/README.md b/README.md index eb4087c..68efd39 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ wp parser create /path/to/source/code --user= Unit tests do not cover every shape of real-world documentation, so changes to the parser are also checked against a corpus of WordPress core source. The same corpus is parsed with the parser at two refs, the base branch and the pull request merged into it, and the two JSON exports are diffed. Everything in that diff is a behavior change the pull request makes: every hunk must be either intended and explained, or it is a regression. -`.github/workflows/corpus-diff.yml` runs this on every pull request. The corpus is `wp-includes` from a pinned WordPress tag (`WP_CORPUS_TAG` in the workflow), so diffs are reproducible. The job is non-blocking: it uploads the diff as a `corpus.diff` artifact and reports the hunk count in the job summary. The head checkout's `tools/export-corpus.php` drives both sides, so tooling changes never masquerade as parser changes. The exports are diffed as emitted: both sides share the corpus, the PHP binary, and the exporter, so the output is already deterministic. `prep-diff.php` is for comparing exports from different environments; its line-number and namespace-prefix erasure and its collection sorting would hide or scatter real changes here. +`.github/workflows/corpus-diff.yml` runs this on every pull request. The corpus is `wp-includes` from a pinned WordPress tag (`WP_CORPUS_TAG` in the workflow), so diffs are reproducible. The job is non-blocking: it uploads the diff as a `corpus.diff` artifact, reports it in the job summary, and posts one comment on the pull request, updated on every push, with the hunk and line counts and the diff itself when it fits in a comment (`tools/corpus-diff-comment.sh` renders it). Pull requests from forks and from Dependabot run with a read-only token, so they get the summary and the artifact only. The head checkout's `tools/export-corpus.php` drives both sides, so tooling changes never masquerade as parser changes. The exports are diffed as emitted: both sides share the corpus, the PHP binary, and the exporter, so the output is already deterministic. `prep-diff.php` is for comparing exports from different environments; its line-number and namespace-prefix erasure and its collection sorting would hide or scatter real changes here. To run it locally, get the pinned corpus: diff --git a/tools/corpus-diff-comment.sh b/tools/corpus-diff-comment.sh new file mode 100755 index 0000000..dd075d6 --- /dev/null +++ b/tools/corpus-diff-comment.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# +# Renders the corpus diff as GitHub-flavored Markdown for a pull request +# comment or a job summary. +# +# Usage: +# +# WP_CORPUS_TAG=7.0.4 BASE_SHA=... HEAD_SHA=... tools/corpus-diff-comment.sh corpus.diff > comment.md +# +# Optional: ARTIFACT_URL (download link for corpus.diff), RUN_URL (fallback +# link), MAX_BYTES (largest body that inlines the diff; GitHub rejects +# comments over 65536 characters, default 60000). +# +# The first line is an HTML comment that marks the output as this tool's, so +# the workflow can find and update its own comment instead of adding one per +# run. + +set -euo pipefail + +diff_file=$1 +max_bytes=${MAX_BYTES:-60000} +link=${ARTIFACT_URL:-${RUN_URL:-}} + +marker='' +heading='### Corpus diff' +compared="\`wp-includes@${WP_CORPUS_TAG}\`, parser at \`${BASE_SHA:0:7}\` (base) vs \`${HEAD_SHA:0:7}\` (this PR merged)" + +if [ ! -s "$diff_file" ]; then + printf '%s\n%s\n\n**0 hunks.** No behavior change over %s.\n' "$marker" "$heading" "$compared" + exit 0 +fi + +hunks=$(grep -c '^@@' "$diff_file" || true) +added=$(grep -c '^+[^+]' "$diff_file" || true) +removed=$(grep -c '^-[^-]' "$diff_file" || true) +lines=$(wc -l < "$diff_file" | tr -d ' ') +bytes=$(wc -c < "$diff_file" | tr -d ' ') + +stat="**${hunks} hunks**, \`+${added}\` \`-${removed}\` lines, over ${compared}. Every hunk must be intended and explained in the PR." +download='' +if [ -n "$link" ]; then + download=" [Download corpus.diff](${link})." +fi + +# A fence must be longer than any backtick run inside the diff. +ticks=$( { grep -o '`\{3,\}' "$diff_file" || true; } | awk '{ if ( length( $0 ) > m ) m = length( $0 ) } END { print m + 0 }') +fence=$(printf '%*s' $(( ticks > 2 ? ticks + 1 : 3 )) '' | tr ' ' '`') + +# Everything except the diff itself, to decide whether the diff fits. +frame=$(printf '%s\n%s\n\n%s%s\n\n
\ncorpus.diff (%s lines)\n\n%sdiff\n%s\n
\n' \ + "$marker" "$heading" "$stat" "$download" "$lines" "$fence" "$fence") + +if [ $(( ${#frame} + bytes )) -gt "$max_bytes" ]; then + printf '%s\n%s\n\n%s The diff (%s lines, %s bytes) is too large for a comment.%s\n' \ + "$marker" "$heading" "$stat" "$lines" "$bytes" "$download" + exit 0 +fi + +printf '%s\n%s\n\n%s%s\n\n
\ncorpus.diff (%s lines)\n\n%sdiff\n' \ + "$marker" "$heading" "$stat" "$download" "$lines" "$fence" +cat "$diff_file" +printf '%s\n
\n' "$fence" From 3a285597a48f617d433c0178fe18b5d063adadcd Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Fri, 21 Aug 2026 19:08:18 +0400 Subject: [PATCH 10/14] Cut an oversize corpus diff to the hunks that fit Instead of dropping the diff from the comment when it exceeds the size budget, keep the leading whole hunks that fit, and say how many of the total are shown. The download link still has the full diff. --- tools/corpus-diff-comment.sh | 48 +++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/tools/corpus-diff-comment.sh b/tools/corpus-diff-comment.sh index dd075d6..0beaa4a 100755 --- a/tools/corpus-diff-comment.sh +++ b/tools/corpus-diff-comment.sh @@ -8,8 +8,9 @@ # WP_CORPUS_TAG=7.0.4 BASE_SHA=... HEAD_SHA=... tools/corpus-diff-comment.sh corpus.diff > comment.md # # Optional: ARTIFACT_URL (download link for corpus.diff), RUN_URL (fallback -# link), MAX_BYTES (largest body that inlines the diff; GitHub rejects -# comments over 65536 characters, default 60000). +# link), MAX_BYTES (largest body to emit; GitHub rejects comments over 65536 +# characters, default 60000). A diff that does not fit is cut to the leading +# whole hunks that do, and the comment says so. # # The first line is an HTML comment that marks the output as this tool's, so # the workflow can find and update its own comment instead of adding one per @@ -17,6 +18,9 @@ set -euo pipefail +# Count bytes, not characters: GitHub's limit is characters, so bytes are safe. +export LC_ALL=C + diff_file=$1 max_bytes=${MAX_BYTES:-60000} link=${ARTIFACT_URL:-${RUN_URL:-}} @@ -36,7 +40,11 @@ removed=$(grep -c '^-[^-]' "$diff_file" || true) lines=$(wc -l < "$diff_file" | tr -d ' ') bytes=$(wc -c < "$diff_file" | tr -d ' ') -stat="**${hunks} hunks**, \`+${added}\` \`-${removed}\` lines, over ${compared}. Every hunk must be intended and explained in the PR." +hunk_word=hunks +if [ "$hunks" -eq 1 ]; then + hunk_word=hunk +fi +stat="**${hunks} ${hunk_word}**, \`+${added}\` \`-${removed}\` lines, over ${compared}. Every hunk must be intended and explained in the PR." download='' if [ -n "$link" ]; then download=" [Download corpus.diff](${link})." @@ -46,17 +54,35 @@ fi ticks=$( { grep -o '`\{3,\}' "$diff_file" || true; } | awk '{ if ( length( $0 ) > m ) m = length( $0 ) } END { print m + 0 }') fence=$(printf '%*s' $(( ticks > 2 ? ticks + 1 : 3 )) '' | tr ' ' '`') -# Everything except the diff itself, to decide whether the diff fits. -frame=$(printf '%s\n%s\n\n%s%s\n\n
\ncorpus.diff (%s lines)\n\n%sdiff\n%s\n
\n' \ - "$marker" "$heading" "$stat" "$download" "$lines" "$fence" "$fence") +# Everything except the diff itself, sized with the longer, truncated wording, +# decides how many bytes of diff fit. +note=" Only the first ${hunks} of ${hunks} hunks are shown below; the download has them all." +frame=$(printf '%s\n%s\n\n%s%s%s\n\n
\ncorpus.diff (first %s of %s hunks)\n\n%sdiff\n%s\n
\n' \ + "$marker" "$heading" "$stat" "$note" "$download" "$hunks" "$hunks" "$fence" "$fence") +budget=$(( max_bytes - ${#frame} )) + +# Last line and count of the leading whole hunks that fit the budget. +read -r keep_line keep_hunks < <(awk -v budget="$budget" ' + /^@@/ { if ( total <= budget ) { keep_line = NR - 1; keep_hunks = seen } seen++ } + { total += length( $0 ) + 1 } + END { if ( total <= budget ) { keep_line = NR; keep_hunks = seen } print keep_line + 0, keep_hunks + 0 } +' "$diff_file") -if [ $(( ${#frame} + bytes )) -gt "$max_bytes" ]; then - printf '%s\n%s\n\n%s The diff (%s lines, %s bytes) is too large for a comment.%s\n' \ +if [ "$keep_hunks" -eq 0 ]; then + printf '%s\n%s\n\n%s No hunk fits in a comment (%s lines, %s bytes).%s\n' \ "$marker" "$heading" "$stat" "$lines" "$bytes" "$download" exit 0 fi -printf '%s\n%s\n\n%s%s\n\n
\ncorpus.diff (%s lines)\n\n%sdiff\n' \ - "$marker" "$heading" "$stat" "$download" "$lines" "$fence" -cat "$diff_file" +if [ "$keep_hunks" -eq "$hunks" ]; then + note='' + summary="corpus.diff (${lines} lines)" +else + note=" Only the first ${keep_hunks} of ${hunks} hunks are shown below; the download has them all." + summary="corpus.diff (first ${keep_hunks} of ${hunks} hunks)" +fi + +printf '%s\n%s\n\n%s%s%s\n\n
\n%s\n\n%sdiff\n' \ + "$marker" "$heading" "$stat" "$note" "$download" "$summary" "$fence" +head -n "$keep_line" "$diff_file" printf '%s\n
\n' "$fence" From e74611e93f86db8f0086478d9f9eb943ccf5efeb Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Fri, 21 Aug 2026 19:11:58 +0400 Subject: [PATCH 11/14] Parse the whole WordPress release as the corpus wp-admin, the root files, and the bundled themes and plugins join wp-includes: 1880 PHP files instead of 1039, 64 MB of JSON instead of 49 MB, about 1.4 seconds more per export locally. More shapes of real code mean more regressions caught, and every hunk is still a parser behavior change. Only PHP files are kept in the cache; the cache key carries a layout suffix so the old wp-includes-only entry is not reused. --- .github/workflows/corpus-diff.yml | 29 +++++++++++++++++------------ README.md | 8 ++++---- tools/corpus-diff-comment.sh | 2 +- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/.github/workflows/corpus-diff.yml b/.github/workflows/corpus-diff.yml index 5469489..af0cfe3 100644 --- a/.github/workflows/corpus-diff.yml +++ b/.github/workflows/corpus-diff.yml @@ -57,30 +57,35 @@ jobs: php-version: "8.4" coverage: none + # The key carries the corpus layout: bump the suffix when the download + # step changes what it keeps. - name: Cache corpus id: cache-corpus uses: actions/cache@v4 with: path: corpus - key: wp-corpus-${{ env.WP_CORPUS_TAG }} + key: wp-corpus-${{ env.WP_CORPUS_TAG }}-php-all + # The whole release: wp-admin, wp-includes, the root files, and + # wp-content with the bundled themes and plugins. Only PHP files are + # kept; the parser reads nothing else. - name: Download corpus if: steps.cache-corpus.outputs.cache-hit != 'true' run: | curl -sSfL -o wordpress.zip "https://github.com/WordPress/WordPress/archive/refs/tags/${WP_CORPUS_TAG}.zip" - unzip -q wordpress.zip "WordPress-${WP_CORPUS_TAG}/wp-includes/*" - mkdir -p corpus - mv "WordPress-${WP_CORPUS_TAG}/wp-includes" corpus/wp-includes - rm -rf wordpress.zip "WordPress-${WP_CORPUS_TAG}" + unzip -q wordpress.zip + mv "WordPress-${WP_CORPUS_TAG}" corpus + find corpus -type f ! -name '*.php' -delete + rm wordpress.zip # An empty or miscached corpus would make both exports emit `[]` and the - # diff trivially empty — a green job that checked nothing. wp-includes - # has well over 500 PHP files on any supported tag. + # diff trivially empty, a green job that checked nothing. A release has + # well over 1500 PHP files on any supported tag. - name: Verify corpus run: | - count=$(find corpus/wp-includes -name '*.php' | wc -l) + count=$(find corpus -name '*.php' | wc -l) echo "Corpus PHP files: ${count}" - [ "$count" -ge 500 ] + [ "$count" -ge 1500 ] # On pull_request, HEAD is the PR merged into the base branch, so its # merge base with the base branch is the base branch tip, not the commit @@ -97,16 +102,16 @@ jobs: run: composer --working-dir=base install --no-interaction --no-security-blocking # The size floor guards the same failure mode as Verify corpus: a real - # wp-includes export is tens of megabytes of JSON. + # export is tens of megabytes of JSON. - name: Export corpus (base) run: | - php -d memory_limit=4G tools/export-corpus.php base corpus/wp-includes > base.json + php -d memory_limit=4G tools/export-corpus.php base corpus > base.json ls -l base.json [ "$(wc -c < base.json)" -ge 1000000 ] - name: Export corpus (head) run: | - php -d memory_limit=4G tools/export-corpus.php . corpus/wp-includes > head.json + php -d memory_limit=4G tools/export-corpus.php . corpus > head.json ls -l head.json [ "$(wc -c < head.json)" -ge 1000000 ] diff --git a/README.md b/README.md index 68efd39..aa8d26c 100644 --- a/README.md +++ b/README.md @@ -55,13 +55,13 @@ wp parser create /path/to/source/code --user= Unit tests do not cover every shape of real-world documentation, so changes to the parser are also checked against a corpus of WordPress core source. The same corpus is parsed with the parser at two refs, the base branch and the pull request merged into it, and the two JSON exports are diffed. Everything in that diff is a behavior change the pull request makes: every hunk must be either intended and explained, or it is a regression. -`.github/workflows/corpus-diff.yml` runs this on every pull request. The corpus is `wp-includes` from a pinned WordPress tag (`WP_CORPUS_TAG` in the workflow), so diffs are reproducible. The job is non-blocking: it uploads the diff as a `corpus.diff` artifact, reports it in the job summary, and posts one comment on the pull request, updated on every push, with the hunk and line counts and the diff itself when it fits in a comment (`tools/corpus-diff-comment.sh` renders it). Pull requests from forks and from Dependabot run with a read-only token, so they get the summary and the artifact only. The head checkout's `tools/export-corpus.php` drives both sides, so tooling changes never masquerade as parser changes. The exports are diffed as emitted: both sides share the corpus, the PHP binary, and the exporter, so the output is already deterministic. `prep-diff.php` is for comparing exports from different environments; its line-number and namespace-prefix erasure and its collection sorting would hide or scatter real changes here. +`.github/workflows/corpus-diff.yml` runs this on every pull request. The corpus is the PHP of a whole WordPress release at a pinned tag (`WP_CORPUS_TAG` in the workflow): wp-admin, wp-includes, the root files, and the bundled themes and plugins. Pinning keeps diffs reproducible. The job is non-blocking: it uploads the diff as a `corpus.diff` artifact, reports it in the job summary, and posts one comment on the pull request, updated on every push, with the hunk and line counts and the diff itself when it fits in a comment (`tools/corpus-diff-comment.sh` renders it). Pull requests from forks and from Dependabot run with a read-only token, so they get the summary and the artifact only. The head checkout's `tools/export-corpus.php` drives both sides, so tooling changes never masquerade as parser changes. The exports are diffed as emitted: both sides share the corpus, the PHP binary, and the exporter, so the output is already deterministic. `prep-diff.php` is for comparing exports from different environments; its line-number and namespace-prefix erasure and its collection sorting would hide or scatter real changes here. To run it locally, get the pinned corpus: ```bash curl -sSfL -o wordpress.zip https://github.com/WordPress/WordPress/archive/refs/tags/7.0.4.zip -unzip -q wordpress.zip 'WordPress-7.0.4/wp-includes/*' +unzip -q wordpress.zip ``` Check out the other side of the comparison and install its dependencies. The exporter runs under plain PHP — it does not load WordPress — but it needs a Composer autoloader in each parser root: @@ -78,8 +78,8 @@ Export both sides over the same corpus and diff. `export-corpus.php` takes the p ```bash export LC_ALL=C -php -d memory_limit=4G tools/export-corpus.php base WordPress-7.0.4/wp-includes > base.json -php -d memory_limit=4G tools/export-corpus.php . WordPress-7.0.4/wp-includes > head.json +php -d memory_limit=4G tools/export-corpus.php base WordPress-7.0.4 > base.json +php -d memory_limit=4G tools/export-corpus.php . WordPress-7.0.4 > head.json diff -u base.json head.json > corpus.diff ``` diff --git a/tools/corpus-diff-comment.sh b/tools/corpus-diff-comment.sh index 0beaa4a..1dc6bf8 100755 --- a/tools/corpus-diff-comment.sh +++ b/tools/corpus-diff-comment.sh @@ -27,7 +27,7 @@ link=${ARTIFACT_URL:-${RUN_URL:-}} marker='' heading='### Corpus diff' -compared="\`wp-includes@${WP_CORPUS_TAG}\`, parser at \`${BASE_SHA:0:7}\` (base) vs \`${HEAD_SHA:0:7}\` (this PR merged)" +compared="WordPress ${WP_CORPUS_TAG}, parser at \`${BASE_SHA:0:7}\` (base) vs \`${HEAD_SHA:0:7}\` (this PR merged)" if [ ! -s "$diff_file" ]; then printf '%s\n%s\n\n**0 hunks.** No behavior change over %s.\n' "$marker" "$heading" "$compared" From 806ed6d7f40f3c128cfe846c9d44e20c8a16e64a Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Mon, 24 Aug 2026 09:54:20 +0400 Subject: [PATCH 12/14] Scope corpus bump PR lookup to this repository --- .github/workflows/corpus-pin-update.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/corpus-pin-update.yml b/.github/workflows/corpus-pin-update.yml index a51e79e..e8e0f5a 100644 --- a/.github/workflows/corpus-pin-update.yml +++ b/.github/workflows/corpus-pin-update.yml @@ -132,9 +132,16 @@ jobs: set -euo pipefail branch="bump-corpus-pin-${LATEST}" - # Idempotent: any prior PR for this version, open or closed, means the - # bump has already been proposed and possibly declined. Leave it be. - existing="$(gh pr list --repo "${GITHUB_REPOSITORY}" --head "${branch}" --state all --json url --jq '.[0].url // ""')" + # Idempotent: any prior PR from this repository for this version, open + # or closed, means the bump has already been proposed and possibly + # declined. Leave it be. + existing="$( + gh api --method GET \ + "repos/${GITHUB_REPOSITORY}/pulls" \ + -f state=all \ + -f head="${GITHUB_REPOSITORY_OWNER}:${branch}" \ + --jq '.[0].html_url // ""' + )" if [ -n "${existing}" ]; then echo "A bump PR for ${LATEST} already exists: ${existing}" exit 0 From 17f91531a93e48b10b6e6501c899d29f19b63ea2 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Mon, 24 Aug 2026 10:02:41 +0400 Subject: [PATCH 13/14] Address zizmor feedback for corpus pin workflow --- .github/workflows/corpus-pin-update.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/corpus-pin-update.yml b/.github/workflows/corpus-pin-update.yml index e8e0f5a..b9beac8 100644 --- a/.github/workflows/corpus-pin-update.yml +++ b/.github/workflows/corpus-pin-update.yml @@ -50,7 +50,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # The later git push deliberately uses checkout's persisted token. + persist-credentials: true - name: Resolve the latest WordPress stable id: resolve From 134c8fa7002ab7f07720de146e9e1b399713fb22 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Mon, 24 Aug 2026 10:12:04 +0400 Subject: [PATCH 14/14] simplify comments --- .github/workflows/corpus-pin-update.yml | 52 ++++++++----------------- 1 file changed, 16 insertions(+), 36 deletions(-) diff --git a/.github/workflows/corpus-pin-update.yml b/.github/workflows/corpus-pin-update.yml index b9beac8..7fe8148 100644 --- a/.github/workflows/corpus-pin-update.yml +++ b/.github/workflows/corpus-pin-update.yml @@ -1,28 +1,19 @@ -# Keeps `WP_CORPUS_TAG` in corpus-diff.yml pinned to the current WordPress -# stable. Dependabot cannot track WordPress core releases, so this does it: -# weekly, ask api.wordpress.org for the latest stable, and open a PR that -# rewrites the one pin line when it has moved. +# Dependabot cannot track WordPress core releases, so this workflow keeps +# `WP_CORPUS_TAG` current. # -# Why a bump PR is cheap to review, deliberately: it changes only the corpus -# input, never the parser. Its own corpus-diff run therefore parses the new -# corpus with an identical parser on both sides — 0 hunks by construction. -# Reviewing a bump PR is checking that the corpus guards (file count, export -# size) still pass on the new tag. A non-zero diff on one of these PRs would -# mean corpus-diff is not comparing what it claims to. +# A pin-only PR does not change the parser, and corpus-diff uses the new corpus +# on both sides. The expected diff is therefore empty. # -# Dependency, and the likely failure: creating the PR needs "Allow GitHub -# Actions to create and approve pull requests" (Settings -> Actions -> General -# -> Workflow permissions). Many organizations disable it. If it is off, the -# "Open the bump PR" step fails loudly with that setting named; enable it, or -# give the step a PAT as GH_TOKEN. The failure is never swallowed: a silent -# no-op would leave the corpus quietly rotting on an old tag. +# Opening the PR also requires "Allow GitHub Actions to create and approve pull +# requests" in the repository's Actions settings. name: Corpus Pin Update on: schedule: - # Weekly, Mondays. Off the hour to dodge the top-of-hour scheduling queue. - - cron: "43 5 * * 1" + # Every Sunday at 03:17 UTC. Running at 17 minutes past the hour avoids + # the busiest scheduling window. + - cron: "17 3 * * 0" workflow_dispatch: concurrency: @@ -32,7 +23,7 @@ concurrency: jobs: corpus-pin-update: name: Bump the pinned WordPress corpus - # Forks inherit schedules; only the canonical repo should open these PRs. + # A copied or forked workflow must not open update PRs. if: github.repository == 'WordPress/phpdoc-parser' runs-on: ubuntu-latest @@ -42,8 +33,7 @@ jobs: env: WORKFLOW_FILE: .github/workflows/corpus-diff.yml - # Canonical, and authoritative about what "stable" means: it excludes - # betas and RCs, which a raw tag listing of WordPress/WordPress does not. + # This API excludes beta and RC versions; repository tags do not. VERSION_CHECK_API: https://api.wordpress.org/core/version-check/1.7/ CORPUS_MIRROR: https://github.com/WordPress/WordPress.git LC_ALL: C @@ -52,7 +42,7 @@ jobs: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - # The later git push deliberately uses checkout's persisted token. + # The push below uses the token saved by checkout. persist-credentials: true - name: Resolve the latest WordPress stable @@ -60,7 +50,6 @@ jobs: run: | set -euo pipefail latest="$(curl -sSfL --retry 3 --retry-delay 5 "${VERSION_CHECK_API}" | jq -r '.offers[0].current // ""')" - # An API hiccup must not become a garbage commit. if ! printf '%s' "${latest}" | grep -Eq '^[0-9]+\.[0-9]+(\.[0-9]+)?$'; then echo "::error::${VERSION_CHECK_API} did not yield a usable version (got: '${latest}')." exit 1 @@ -81,12 +70,6 @@ jobs: fi echo "Pinned corpus tag: ${current}" - # Three reasons not to bump, in order: nothing moved; the pin is - # ahead of the published stable, which is deliberate (someone pinned - # a prerelease) and must not be quietly walked back; or the - # WordPress/WordPress mirror corpus-diff downloads from has not - # tagged the release yet, in which case the next run picks it up - # rather than this one opening a PR whose corpus cannot be built. bump=false if [ "${current}" = "${LATEST}" ]; then echo "Already pinned to the latest stable; nothing to do." @@ -113,7 +96,7 @@ jobs: sed "s/^\( *WP_CORPUS_TAG: *\).*/\1\"${LATEST}\"/" "${WORKFLOW_FILE}" > "${WORKFLOW_FILE}.tmp" mv "${WORKFLOW_FILE}.tmp" "${WORKFLOW_FILE}" - # The rewrite must be exactly the pin line and nothing else. + # Keep the automated edit narrow enough to review safely. check="$(sed -n 's/^ *WP_CORPUS_TAG: *"\([^"]*\)".*/\1/p' "${WORKFLOW_FILE}")" added="$(git diff --numstat -- "${WORKFLOW_FILE}" | awk '{print $1}')" removed="$(git diff --numstat -- "${WORKFLOW_FILE}" | awk '{print $2}')" @@ -135,9 +118,7 @@ jobs: set -euo pipefail branch="bump-corpus-pin-${LATEST}" - # Idempotent: any prior PR from this repository for this version, open - # or closed, means the bump has already been proposed and possibly - # declined. Leave it be. + # Treat closed PRs as handled so a declined update is not reopened. existing="$( gh api --method GET \ "repos/${GITHUB_REPOSITORY}/pulls" \ @@ -164,9 +145,8 @@ jobs: git checkout -b "${branch}" git add -- "${WORKFLOW_FILE}" git commit -m "${title}" - # Force is safe and wanted: no PR exists for this branch, so any - # branch of this name is debris from a run that pushed and then - # failed to open the PR. Overwriting it lets that run self-heal. + # No PR exists at this point. Reusing the branch repairs a run that + # pushed but failed before opening its PR. git push --force origin "HEAD:refs/heads/${branch}" if ! url="$(gh pr create --repo "${GITHUB_REPOSITORY}" --base "${BASE_BRANCH}" --head "${branch}" --title "${title}" --body-file pr-body.md 2>&1)"; then