Add corpus regeneration diff CI

.github/workflows/corpus-diff.yml

@@ -0,0 +1,158 @@
+# Parses a pinned corpus of WordPress core source with the parser at the base
+# branch and with the PR merged into it, and diffs the two JSON exports.
+# Anything in the diff is a behavior change this PR makes: every hunk must be
+# either intended (and explained in the PR) or a regression.
+#
+# Policy decisions, deliberate:
+# - The corpus is pinned to one WordPress tag so diffs are reproducible.
+# - The head checkout's tools/export-corpus.php drives both sides, so tooling
+#   changes never masquerade as parser changes.
+# - The exports are diffed as emitted, without prep-diff.php. Both sides share
+#   the corpus, the PHP binary, and the exporter, so the export is already
+#   deterministic; prep-diff.php exists to reconcile exports from different
+#   environments, and its erasures (line numbers, global-namespace prefixes)
+#   and collection sorting would hide or scatter real changes here.
+# - Non-blocking: the job succeeds even when the diff is non-empty. The diff
+#   is published as an artifact, summarized, and posted as a PR comment that
+#   is updated in place on every run. Make it blocking only after the signal
+#   has proven trustworthy.
+# - The comment needs a token that can write to the PR. pull_request runs for
+#   forks and for Dependabot get a read-only token, so those PRs get the
+#   summary and the artifact only.
+# - PHP 8.4, the newest runtime in the unit-test matrix. Both sides run under
+#   the same binary, so the PHP version never shows up as a parser change.
+
+name: Corpus Diff
+
+on:
+  pull_request:
+
+# A new push to the PR supersedes any run still in flight for it.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  corpus-diff:
+    name: WordPress corpus regeneration diff
+    runs-on: ubuntu-latest
+
+    env:
+      WP_CORPUS_TAG: "7.0.4"
+      LC_ALL: C
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+
+      - name: Set up PHP
+        uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # 2.37.2
+        with:
+          php-version: "8.4"
+          coverage: none
+
+      # The key carries the corpus layout: bump the suffix when the download
+      # step changes what it keeps.
+      - name: Cache corpus
+        id: cache-corpus
+        uses: actions/cache@v4
+        with:
+          path: corpus
+          key: wp-corpus-${{ env.WP_CORPUS_TAG }}-php-all
+
+      # The whole release: wp-admin, wp-includes, the root files, and
+      # wp-content with the bundled themes and plugins. Only PHP files are
+      # kept; the parser reads nothing else.
+      - name: Download corpus
+        if: steps.cache-corpus.outputs.cache-hit != 'true'
+        run: |
+          curl -sSfL -o wordpress.zip "https://github.com/WordPress/WordPress/archive/refs/tags/${WP_CORPUS_TAG}.zip"
+          unzip -q wordpress.zip
+          mv "WordPress-${WP_CORPUS_TAG}" corpus
+          find corpus -type f ! -name '*.php' -delete
+          rm wordpress.zip
+
+      # An empty or miscached corpus would make both exports emit `[]` and the
+      # diff trivially empty, a green job that checked nothing. A release has
+      # well over 1500 PHP files on any supported tag.
+      - name: Verify corpus
+        run: |
+          count=$(find corpus -name '*.php' | wc -l)
+          echo "Corpus PHP files: ${count}"
+          [ "$count" -ge 1500 ]
+
+      # On pull_request, HEAD is the PR merged into the base branch, so its
+      # merge base with the base branch is the base branch tip, not the commit
+      # the PR branched from. That is the intended comparison: base as it is
+      # now against base plus exactly this PR, with no hunks from other work
+      # that landed since the branch point.
+      - name: Check out base
+        run: git worktree add base "$(git merge-base "origin/${{ github.base_ref }}" HEAD)"
+
+      - name: Install Composer dependencies (head)
+        run: composer install --no-interaction --no-security-blocking
+
+      - name: Install Composer dependencies (base)
+        run: composer --working-dir=base install --no-interaction --no-security-blocking
+
+      # The size floor guards the same failure mode as Verify corpus: a real
+      # export is tens of megabytes of JSON.
+      - name: Export corpus (base)
+        run: |
+          php -d memory_limit=4G tools/export-corpus.php base corpus > base.json
+          ls -l base.json
+          [ "$(wc -c < base.json)" -ge 1000000 ]
+
+      - name: Export corpus (head)
+        run: |
+          php -d memory_limit=4G tools/export-corpus.php . corpus > head.json
+          ls -l head.json
+          [ "$(wc -c < head.json)" -ge 1000000 ]
+
+      - name: Diff
+        run: |
+          # diff exits 1 on differences (expected) and 2 on trouble (fail).
+          diff -u --label base --label head base.json head.json > corpus.diff || [ $? -eq 1 ]
+          echo "Corpus diff: $(grep -c '^@@' corpus.diff || true) hunks, $(wc -l < corpus.diff) lines"
+
+      - name: Upload diff
+        id: upload
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: corpus.diff
+          path: corpus.diff
+          if-no-files-found: ignore
+
+      - name: Render report
+        env:
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+          ARTIFACT_URL: ${{ steps.upload.outputs.artifact-url }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          BASE_SHA="$(git -C base rev-parse HEAD)" tools/corpus-diff-comment.sh corpus.diff > comment.md
+          tail -n +2 comment.md >> "$GITHUB_STEP_SUMMARY"
+
+      # Create the comment on the first run and update it on every later one,
+      # so the PR carries one report that reflects the latest push.
+      - name: Comment on the pull request
+        if: github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+          PR: ${{ github.event.pull_request.number }}
+        run: |
+          ids=$(gh api "repos/${REPO}/issues/${PR}/comments" --paginate \
+            --jq '.[] | select(.body | startswith("<!-- corpus-diff -->")) | .id')
+          id=${ids%%$'\n'*}
+          if [ -n "$id" ]; then
+            gh api --silent -X PATCH "repos/${REPO}/issues/comments/${id}" -F body=@comment.md
+          else
+            gh api --silent -X POST "repos/${REPO}/issues/${PR}/comments" -F body=@comment.md
+          fi

README.md

@@ -50,3 +50,37 @@ In your site's directory:
 ```bash
 wp parser create /path/to/source/code --user=<id|login>
 ```
+
+## Corpus diff
+
+Unit tests do not cover every shape of real-world documentation, so changes to the parser are also checked against a corpus of WordPress core source. The same corpus is parsed with the parser at two refs, the base branch and the pull request merged into it, and the two JSON exports are diffed. Everything in that diff is a behavior change the pull request makes: every hunk must be either intended and explained, or it is a regression.
+
+`.github/workflows/corpus-diff.yml` runs this on every pull request. The corpus is the PHP of a whole WordPress release at a pinned tag (`WP_CORPUS_TAG` in the workflow): wp-admin, wp-includes, the root files, and the bundled themes and plugins. Pinning keeps diffs reproducible. The job is non-blocking: it uploads the diff as a `corpus.diff` artifact, reports it in the job summary, and posts one comment on the pull request, updated on every push, with the hunk and line counts and the diff itself when it fits in a comment (`tools/corpus-diff-comment.sh` renders it). Pull requests from forks and from Dependabot run with a read-only token, so they get the summary and the artifact only. The head checkout's `tools/export-corpus.php` drives both sides, so tooling changes never masquerade as parser changes. The exports are diffed as emitted: both sides share the corpus, the PHP binary, and the exporter, so the output is already deterministic. `prep-diff.php` is for comparing exports from different environments; its line-number and namespace-prefix erasure and its collection sorting would hide or scatter real changes here.
+
+To run it locally, get the pinned corpus:
+
+```bash
+curl -sSfL -o wordpress.zip https://github.com/WordPress/WordPress/archive/refs/tags/7.0.4.zip
+unzip -q wordpress.zip
+```
+
+Check out the other side of the comparison and install its dependencies. The exporter runs under plain PHP — it does not load WordPress — but it needs a Composer autoloader in each parser root:
+
+```bash
+git worktree add base "$(git merge-base origin/master HEAD)"
+composer --working-dir=base install
+composer install
+```
+
+On a branch this compares against the commit the branch left `master` at. CI runs on the pull request merged into `master`, so it compares against the `master` tip; merge `master` into the branch first to get the same comparison.
+
+Export both sides over the same corpus and diff. `export-corpus.php` takes the parser root and the corpus directory, and writes JSON to stdout:
+
+```bash
+export LC_ALL=C
+php -d memory_limit=4G tools/export-corpus.php base WordPress-7.0.4 > base.json
+php -d memory_limit=4G tools/export-corpus.php . WordPress-7.0.4 > head.json
+diff -u base.json head.json > corpus.diff
+```
+
+An empty `corpus.diff` means the change has no effect on parser output.

tools/corpus-diff-comment.sh

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+#
+# Renders the corpus diff as GitHub-flavored Markdown for a pull request
+# comment or a job summary.
+#
+# Usage:
+#
+#     WP_CORPUS_TAG=7.0.4 BASE_SHA=... HEAD_SHA=... tools/corpus-diff-comment.sh corpus.diff > comment.md
+#
+# Optional: ARTIFACT_URL (download link for corpus.diff), RUN_URL (fallback
+# link), MAX_BYTES (largest body to emit; GitHub rejects comments over 65536
+# characters, default 60000). A diff that does not fit is cut to the leading
+# whole hunks that do, and the comment says so.
+#
+# The first line is an HTML comment that marks the output as this tool's, so
+# the workflow can find and update its own comment instead of adding one per
+# run.
+
+set -euo pipefail
+
+# Count bytes, not characters: GitHub's limit is characters, so bytes are safe.
+export LC_ALL=C
+
+diff_file=$1
+max_bytes=${MAX_BYTES:-60000}
+link=${ARTIFACT_URL:-${RUN_URL:-}}
+
+marker='<!-- corpus-diff -->'
+heading='### Corpus diff'
+compared="WordPress ${WP_CORPUS_TAG}, parser at \`${BASE_SHA:0:7}\` (base) vs \`${HEAD_SHA:0:7}\` (this PR merged)"
+
+if [ ! -s "$diff_file" ]; then
+	printf '%s\n%s\n\n**0 hunks.** No behavior change over %s.\n' "$marker" "$heading" "$compared"
+	exit 0
+fi
+
+hunks=$(grep -c '^@@' "$diff_file" || true)
+added=$(grep -c '^+[^+]' "$diff_file" || true)
+removed=$(grep -c '^-[^-]' "$diff_file" || true)
+lines=$(wc -l < "$diff_file" | tr -d ' ')
+bytes=$(wc -c < "$diff_file" | tr -d ' ')
+
+hunk_word=hunks
+if [ "$hunks" -eq 1 ]; then
+	hunk_word=hunk
+fi
+stat="**${hunks} ${hunk_word}**, \`+${added}\` \`-${removed}\` lines, over ${compared}. Every hunk must be intended and explained in the PR."
+download=''
+if [ -n "$link" ]; then
+	download=" [Download corpus.diff](${link})."
+fi
+
+# A fence must be longer than any backtick run inside the diff.
+ticks=$( { grep -o '`\{3,\}' "$diff_file" || true; } | awk '{ if ( length( $0 ) > m ) m = length( $0 ) } END { print m + 0 }')
+fence=$(printf '%*s' $(( ticks > 2 ? ticks + 1 : 3 )) '' | tr ' ' '`')
+
+# Everything except the diff itself, sized with the longer, truncated wording,
+# decides how many bytes of diff fit.
+note=" Only the first ${hunks} of ${hunks} hunks are shown below; the download has them all."
+frame=$(printf '%s\n%s\n\n%s%s%s\n\n<details>\n<summary>corpus.diff (first %s of %s hunks)</summary>\n\n%sdiff\n%s\n</details>\n' \
+	"$marker" "$heading" "$stat" "$note" "$download" "$hunks" "$hunks" "$fence" "$fence")
+budget=$(( max_bytes - ${#frame} ))
+
+# Last line and count of the leading whole hunks that fit the budget.
+read -r keep_line keep_hunks < <(awk -v budget="$budget" '
+	/^@@/ { if ( total <= budget ) { keep_line = NR - 1; keep_hunks = seen } seen++ }
+	{ total += length( $0 ) + 1 }
+	END { if ( total <= budget ) { keep_line = NR; keep_hunks = seen } print keep_line + 0, keep_hunks + 0 }
+' "$diff_file")
+
+if [ "$keep_hunks" -eq 0 ]; then
+	printf '%s\n%s\n\n%s No hunk fits in a comment (%s lines, %s bytes).%s\n' \
+		"$marker" "$heading" "$stat" "$lines" "$bytes" "$download"
+	exit 0
+fi
+
+if [ "$keep_hunks" -eq "$hunks" ]; then
+	note=''
+	summary="corpus.diff (${lines} lines)"
+else
+	note=" Only the first ${keep_hunks} of ${hunks} hunks are shown below; the download has them all."
+	summary="corpus.diff (first ${keep_hunks} of ${hunks} hunks)"
+fi
+
+printf '%s\n%s\n\n%s%s%s\n\n<details>\n<summary>%s</summary>\n\n%sdiff\n' \
+	"$marker" "$heading" "$stat" "$note" "$download" "$summary" "$fence"
+head -n "$keep_line" "$diff_file"
+printf '%s\n</details>\n' "$fence"

tools/export-corpus.php

@@ -0,0 +1,46 @@
+<?php
+
+/**
+ * Exports parsed documentation JSON for a corpus of PHP files.
+ *
+ * Runs under plain PHP, without WordPress: the exporter in lib/runner.php is a
+ * side-effect-free function library loaded by the Composer autoloader.
+ *
+ * The parser root is a parameter so one copy of this script can drive two
+ * checkouts of the parser (base and head) over the same corpus.
+ *
+ * Usage:
+ *
+ *     php -d memory_limit=4G tools/export-corpus.php <parser-root> <corpus-dir> > corpus.json
+ */
+
+// Keep stdout pure JSON: vendored code emits deprecation notices on newer
+// PHP, and display_errors otherwise interleaves them with the output.
+ini_set( 'display_errors', 'stderr' );
+
+if ( $argc < 3 ) {
+	fwrite( STDERR, 'Usage: php tools/export-corpus.php <parser-root> <corpus-dir>' . PHP_EOL );
+	exit( 1 );
+}
+
+$parser_root = rtrim( $argv[1], '/' );
+$corpus_dir  = rtrim( $argv[2], '/' );
+
+if ( ! is_file( $parser_root . '/vendor/autoload.php' ) ) {
+	fwrite( STDERR, 'No Composer autoloader in ' . $parser_root . '; run composer install there first.' . PHP_EOL );
+	exit( 1 );
+}
+
+if ( ! is_dir( $corpus_dir ) ) {
+	fwrite( STDERR, 'Corpus directory not found: ' . $corpus_dir . PHP_EOL );
+	exit( 1 );
+}
+
+require $parser_root . '/vendor/autoload.php';
+
+// get_wp_files() reports an unreadable subdirectory with a WP_Error, a class
+// that only exists inside WordPress: under plain PHP that path is a fatal
+// error with a non-zero exit, so there is no return value to check here.
+$files = \WP_Parser\get_wp_files( $corpus_dir );
+
+echo json_encode( \WP_Parser\parse_files( $files, $corpus_dir ), JSON_PRETTY_PRINT ), PHP_EOL;