From 355c7fef490bc20d315f9caed9da037c87a3aaa0 Mon Sep 17 00:00:00 2001 From: Serhii Pylypchuk Date: Mon, 31 Aug 2026 10:02:53 +0300 Subject: [PATCH] [add] eval suite for the article-grammar skill - 30 fixtures across three tiers with 443 gold records: per-category micro cases, negative controls and composite pages - headless runner over the real CLI, prose-zone scorer, grading workflow - behavioral scenarios for the skill contract: target selection, no file modification, argument robustness - two baselines committed for comparison: v1.1.0 and v1.2.0 - raw runs stay out of the repo, only summary baselines are tracked --- .claude/evals/article-grammar/README.md | 66 ++ .../evals/article-grammar/corpus/sample.txt | 25 + .../fixtures/behavioral/target.md | 16 + .../tier-a/01-passive-voice.gold.json | 184 +++++ .../fixtures/tier-a/01-passive-voice.md | 38 + .../fixtures/tier-a/02-gerunds.gold.json | 151 ++++ .../fixtures/tier-a/02-gerunds.md | 33 + .../fixtures/tier-a/03-weak-verbs.gold.json | 162 ++++ .../fixtures/tier-a/03-weak-verbs.md | 45 ++ .../tier-a/04-misplaced-modifiers.gold.json | 157 ++++ .../fixtures/tier-a/04-misplaced-modifiers.md | 30 + .../fixtures/tier-a/05-pronouns.gold.json | 183 +++++ .../fixtures/tier-a/05-pronouns.md | 32 + .../fixtures/tier-a/06-inversion.gold.json | 170 +++++ .../fixtures/tier-a/06-inversion.md | 33 + .../tier-a/07-overused-words.gold.json | 184 +++++ .../fixtures/tier-a/07-overused-words.md | 49 ++ .../tier-a/08-abbreviations.gold.json | 181 +++++ .../fixtures/tier-a/08-abbreviations.md | 32 + .../fixtures/tier-a/09-wordiness.gold.json | 166 +++++ .../fixtures/tier-a/09-wordiness.md | 32 + .../fixtures/tier-a/10-possessives.gold.json | 171 +++++ .../fixtures/tier-a/10-possessives.md | 30 + .../fixtures/tier-a/11-punctuation.gold.json | 171 +++++ .../fixtures/tier-a/11-punctuation.md | 46 ++ .../fixtures/tier-a/12-tenses.gold.json | 151 ++++ .../fixtures/tier-a/12-tenses.md | 33 + .../fixtures/tier-a/13-articles.gold.json | 182 +++++ .../fixtures/tier-a/13-articles.md | 27 + .../tier-a/14-misused-words.gold.json | 194 +++++ .../fixtures/tier-a/14-misused-words.md | 38 + .../tier-a/15-api-formatting.gold.json | 171 +++++ .../fixtures/tier-a/15-api-formatting.md | 25 + .../tier-a/16-articles-boundary.gold.json | 182 +++++ .../fixtures/tier-a/16-articles-boundary.md | 30 + .../17-long-sentences-repetition.gold.json | 159 ++++ .../tier-a/17-long-sentences-repetition.md | 35 + .../tier-a/18-vague-word-choice.gold.json | 217 ++++++ .../fixtures/tier-a/18-vague-word-choice.md | 33 + .../tier-a/19-obvious-generic.gold.json | 161 ++++ .../fixtures/tier-a/19-obvious-generic.md | 41 + .../tier-a/20-prose-should-be-code.gold.json | 155 ++++ .../tier-a/20-prose-should-be-code.md | 36 + .../fixtures/tier-a/21-info-mapping.gold.json | 148 ++++ .../fixtures/tier-a/21-info-mapping.md | 44 ++ .../fixtures/tier-b/30-clean-prose.gold.json | 87 +++ .../fixtures/tier-b/30-clean-prose.md | 67 ++ .../tier-b/31-api-reference-page.gold.json | 104 +++ .../fixtures/tier-b/31-api-reference-page.md | 34 + .../fixtures/tier-b/32-code-heavy.gold.json | 245 ++++++ .../fixtures/tier-b/32-code-heavy.md | 77 ++ .../tier-b/33-exception-clauses.gold.json | 229 ++++++ .../fixtures/tier-b/33-exception-clauses.md | 61 ++ .../fixtures/tier-b/34-lookalikes.gold.json | 218 ++++++ .../fixtures/tier-b/34-lookalikes.md | 44 ++ .../tier-c/40-getting-started.gold.json | 329 +++++++++ .../fixtures/tier-c/40-getting-started.md | 107 +++ .../tier-c/41-guide-overlap.gold.json | 278 +++++++ .../fixtures/tier-c/41-guide-overlap.md | 75 ++ .../fixtures/tier-c/42-api-mixed.gold.json | 390 ++++++++++ .../fixtures/tier-c/42-api-mixed.md | 48 ++ .../fixtures/tier-c/43-long-form.gold.json | 699 ++++++++++++++++++ .../fixtures/tier-c/43-long-form.md | 196 +++++ .claude/evals/article-grammar/manifest.json | 52 ++ .../evals/article-grammar/runner/MATCHING.md | 70 ++ .../article-grammar/runner/behavioral.sh | 61 ++ .../evals/article-grammar/runner/compare.mjs | 70 ++ .../article-grammar/runner/extract-reports.sh | 20 + .../article-grammar/runner/grade-workflow.js | 224 ++++++ .../evals/article-grammar/runner/invoke.sh | 136 ++++ .../article-grammar/runner/parse-stream.mjs | 135 ++++ .../article-grammar/runner/run-corpus.sh | 18 + .../article-grammar/runner/run-fixtures.sh | 25 + .../evals/article-grammar/runner/runsum.mjs | 16 + .../evals/article-grammar/runner/score.mjs | 413 +++++++++++ .../runs/baseline-1.1.0-opus/report.md | 208 ++++++ .../runs/baseline-1.1.0-opus/results.json | 44 ++ .../runs/baseline-1.2.0-opus/report.md | 104 +++ .../runs/baseline-1.2.0-opus/results.json | 95 +++ .claude/evals/article-grammar/thresholds.json | 152 ++++ .gitignore | 9 + 81 files changed, 9589 insertions(+) create mode 100644 .claude/evals/article-grammar/README.md create mode 100644 .claude/evals/article-grammar/corpus/sample.txt create mode 100644 .claude/evals/article-grammar/fixtures/behavioral/target.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/01-passive-voice.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/01-passive-voice.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/02-gerunds.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/02-gerunds.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/03-weak-verbs.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/03-weak-verbs.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/04-misplaced-modifiers.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/04-misplaced-modifiers.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/05-pronouns.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/05-pronouns.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/06-inversion.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/06-inversion.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/07-overused-words.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/07-overused-words.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/08-abbreviations.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/08-abbreviations.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/09-wordiness.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/09-wordiness.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/10-possessives.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/10-possessives.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/11-punctuation.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/11-punctuation.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/12-tenses.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/12-tenses.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/13-articles.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/13-articles.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/14-misused-words.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/14-misused-words.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/15-api-formatting.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/15-api-formatting.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/16-articles-boundary.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/16-articles-boundary.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/17-long-sentences-repetition.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/17-long-sentences-repetition.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/18-vague-word-choice.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/18-vague-word-choice.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/19-obvious-generic.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/19-obvious-generic.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/20-prose-should-be-code.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/20-prose-should-be-code.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/21-info-mapping.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-a/21-info-mapping.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-b/30-clean-prose.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-b/30-clean-prose.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-b/31-api-reference-page.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-b/31-api-reference-page.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-b/32-code-heavy.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-b/32-code-heavy.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-b/33-exception-clauses.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-b/33-exception-clauses.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-b/34-lookalikes.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-b/34-lookalikes.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-c/40-getting-started.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-c/40-getting-started.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-c/41-guide-overlap.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-c/41-guide-overlap.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-c/42-api-mixed.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-c/42-api-mixed.md create mode 100644 .claude/evals/article-grammar/fixtures/tier-c/43-long-form.gold.json create mode 100644 .claude/evals/article-grammar/fixtures/tier-c/43-long-form.md create mode 100644 .claude/evals/article-grammar/manifest.json create mode 100644 .claude/evals/article-grammar/runner/MATCHING.md create mode 100755 .claude/evals/article-grammar/runner/behavioral.sh create mode 100644 .claude/evals/article-grammar/runner/compare.mjs create mode 100755 .claude/evals/article-grammar/runner/extract-reports.sh create mode 100644 .claude/evals/article-grammar/runner/grade-workflow.js create mode 100755 .claude/evals/article-grammar/runner/invoke.sh create mode 100644 .claude/evals/article-grammar/runner/parse-stream.mjs create mode 100755 .claude/evals/article-grammar/runner/run-corpus.sh create mode 100755 .claude/evals/article-grammar/runner/run-fixtures.sh create mode 100644 .claude/evals/article-grammar/runner/runsum.mjs create mode 100644 .claude/evals/article-grammar/runner/score.mjs create mode 100644 .claude/evals/article-grammar/runs/baseline-1.1.0-opus/report.md create mode 100644 .claude/evals/article-grammar/runs/baseline-1.1.0-opus/results.json create mode 100644 .claude/evals/article-grammar/runs/baseline-1.2.0-opus/report.md create mode 100644 .claude/evals/article-grammar/runs/baseline-1.2.0-opus/results.json create mode 100644 .claude/evals/article-grammar/thresholds.json diff --git a/.claude/evals/article-grammar/README.md b/.claude/evals/article-grammar/README.md new file mode 100644 index 00000000..4aa240de --- /dev/null +++ b/.claude/evals/article-grammar/README.md @@ -0,0 +1,66 @@ +# Eval-сьюта скилла `article-grammar` + +Измеряет, сколько реальных нарушений скилл находит, сколько выдумывает, стабилен ли между +прогонами и не проверяет ли он код вместо прозы. + +Система под тестом: `/Users/DHX-XB/DOCS/skills/.claude/skills/article-grammar` (v1.1.0). +Версия скилла зафиксирована в [manifest.json](manifest.json) через sha256 `SKILL.md` и всех +15 файлов правил — по нему видно, против какой версии снят базлайн. + +## Как запустить + +~~~bash +# один файл через настоящий headless CLI +./runner/invoke.sh