diff --git a/.gitleaks.toml b/.gitleaks.toml index d8e8449..be40874 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -8,4 +8,5 @@ description = "Docs that intentionally contain example or archived sample creden paths = [ '''(?:^|/)docs/audits/artifacts/''', '''(?:^|/)docs/bootstrap-templates/security-patterns\.md$''', + '''(?:^|/)scripts/__tests__/golden/bootstrap-generate\.sample-cli\.md$''', ] diff --git a/docs/audits/anvil-audit-2026-08-09.md b/docs/audits/anvil-audit-2026-08-09.md index 5084797..1769ee5 100644 --- a/docs/audits/anvil-audit-2026-08-09.md +++ b/docs/audits/anvil-audit-2026-08-09.md @@ -217,7 +217,7 @@ Remaining score delta is near-max: enforcement layer (0.8/1). Treat it as option | `docs/bootstrap-templates/pnpm-commands.md` | anvil-bootstrap-template | governance | ✅ | 39 | ✅ | ✅ | ✅ | ✅ | | `docs/bootstrap-templates/prisma-migrations.md` | anvil-bootstrap-template | governance | ✅ | 43 | ✅ | ✅ | ✅ | ✅ | | `docs/bootstrap-templates/react-hooks-deps.md` | anvil-bootstrap-template | governance | ✅ | 41 | ✅ | ✅ | ✅ | ✅ | -| `docs/bootstrap-templates/scope-boundaries.md` | anvil-bootstrap-template | governance | ✅ | 161 | ✅ | ✅ | ✅ | ✅ | +| `docs/bootstrap-templates/scope-boundaries.md` | anvil-bootstrap-template | governance | ✅ | 162 | ✅ | ✅ | ✅ | ✅ | | `docs/bootstrap-templates/security-patterns.md` | anvil-bootstrap-template | governance | ✅ | 82 | ✅ | ✅ | ✅ | ✅ | | `docs/bootstrap-templates/tailwind-no-inline-styles.md` | anvil-bootstrap-template | governance | ✅ | 38 | ✅ | ✅ | ✅ | ✅ | | `docs/bootstrap-templates/testing-patterns.md` | anvil-bootstrap-template | governance | ✅ | 136 | ✅ | ✅ | ✅ | ✅ | @@ -237,7 +237,7 @@ Canonical governance surface: - `docs/bootstrap-templates/pnpm-commands.md` (39 lines) - `docs/bootstrap-templates/prisma-migrations.md` (43 lines) - `docs/bootstrap-templates/react-hooks-deps.md` (41 lines) -- `docs/bootstrap-templates/scope-boundaries.md` (161 lines) +- `docs/bootstrap-templates/scope-boundaries.md` (162 lines) - `docs/bootstrap-templates/security-patterns.md` (82 lines) - `docs/bootstrap-templates/tailwind-no-inline-styles.md` (38 lines) - `docs/bootstrap-templates/testing-patterns.md` (136 lines) @@ -341,11 +341,11 @@ PRs analyzed: 43 · Comments reviewed: 37 · Substantive comments: 37 · Candida | Theme | Frequency | PR Spread | Severity | Rule Signal Match | Comment Alignment | |-------|-----------|-----------|----------|-------------------|------------------| -| Naming | 12 comments | 6 PRs (high) | medium | 🟡 signal match | 100% strong | -| Testing | 7 comments | 5 PRs (high) | medium | 🟡 signal match | 100% strong | -| Error Handling | 3 comments | 3 PRs (medium) | medium | 🟡 signal match | 100% strong | -| Documentation | 5 comments | 2 PRs (medium) | low | 🟡 signal match | 100% strong | -| Types | 3 comments | 1 PRs (low) | medium | 🟡 signal match | 100% strong | +| Naming | 12 comments | 6 PRs (high) | medium | 🟢 signal match | 100% strong | +| Testing | 7 comments | 5 PRs (high) | medium | 🟢 signal match | 100% strong | +| Error Handling | 3 comments | 3 PRs (medium) | medium | 🟢 signal match | 100% strong | +| Documentation | 5 comments | 2 PRs (medium) | low | 🟢 signal match | 100% strong | +| Types | 3 comments | 1 PRs (low) | medium | 🟢 signal match | 100% strong | ## Rule Portfolio Actions diff --git a/docs/bootstrap-templates/scope-boundaries.md b/docs/bootstrap-templates/scope-boundaries.md index df19379..0d5a197 100644 --- a/docs/bootstrap-templates/scope-boundaries.md +++ b/docs/bootstrap-templates/scope-boundaries.md @@ -1,5 +1,6 @@ # Bootstrap Template: Scope Boundaries +*Signal: general · Tier: alwaysApply · Glob: —* *Last validated: 2026-05-27 · Author: Scout/Anvil · Status: Active* *Sources: Concentrix "12 Failure Patterns" (Nov 2025), 12-Factor Agents Factor 12, Gartner 2027 cancellation data* diff --git a/scripts/__tests__/golden/bootstrap-generate.sample-cli.md b/scripts/__tests__/golden/bootstrap-generate.sample-cli.md index 591b465..2d2ee3f 100644 --- a/scripts/__tests__/golden/bootstrap-generate.sample-cli.md +++ b/scripts/__tests__/golden/bootstrap-generate.sample-cli.md @@ -17,7 +17,7 @@ --- -## Suggested AGENTS.md Additions (5 rules) +## Suggested AGENTS.md Additions (11 rules) Copy the rules you want to adopt into the appropriate section of `AGENTS.md`. Validate each one against your project's actual behavior before committing. @@ -39,6 +39,364 @@ Never use `npm`, `npx` (prefer `bunx`), or `yarn` in this project. --- +### Rule: Handle Errors Explicitly — No Silent Failures +*Signal: `language:typescript` · Tier: glob* + +**Why (failure mode):** +Agents swallow errors, use empty catch blocks, or log-and-continue without proper error propagation. The result is silent failures in production that are impossible to debug — the code appears to work but quietly discards error state, leaving users with broken behavior and developers with no signal. + +**The rule:** +Use typed error handling. Catch specific error types. Either handle the error with recovery logic or re-throw it. Never use empty catch blocks. + +- Catch blocks must do one of: (a) recover with explicit logic, or (b) re-throw the error +- Use `instanceof` guards to distinguish error types before handling +- Prefer discriminated union Result types (`{ ok: true, data } | { ok: false, error }`) for functions that can fail predictably +- Never use `catch (e) {}` — empty catch blocks are always wrong +- Never use `catch (e) { console.log(e) }` as a substitute for handling — log-and-swallow is a silent failure +- If an error is truly ignorable, document why with an explicit comment + +``` +// Typed recovery with re-throw for unexpected errors +try { + await doThing(); +} catch (e) { + if (e instanceof NetworkError) { + await retry(); + } else { + throw e; // propagate unexpected errors + } +} + +// Result type pattern for predictable failures +type Result = { ok: true; data: T } | { ok: false; error: string }; + +async function fetchUser(id: string): Promise> { + try { + const user = await db.users.findById(id); + return { ok: true, data: user }; + } catch (e) { + return { ok: false, error: e instanceof Error ? e.message : "Unknown error" }; + } +} + +// Caller checks result explicitly +const result = await fetchUser(userId); +if (!result.ok) { + showError(result.error); + return; +} +processUser(result.data); +``` +``` +// Silent failure — swallowed error, broken state, no signal +try { + await doThing(); +} catch (e) {} + +// Log-and-swallow — looks like handling but isn't; execution continues +try { + await saveRecord(data); +} catch (e) { + console.log(e); // logged but not propagated — caller thinks it succeeded +} + +// Untyped catch with no discrimination — can't handle correctly +try { + await fetchUser(id); +} catch (e) { + handleError(e); // what kind of error? network? auth? not-found? unknown +} +``` + +*See also: `docs/rubric.md` — rule sizing and format standards, TypeScript Handbook: [Error Handling](https://www.typescriptlang.org/docs/handbook/), neverthrow library — ergonomic Result types for TypeScript* + +--- + +### Rule: Naming Conventions — Consistent, Readable Identifiers +*Signal: `language:typescript` · Tier: glob* + +**Why (failure mode):** +Agents use inconsistent naming — mixing `camelCase` and `snake_case` in the same file, using abbreviated names that lose meaning (`res`, `usr`, `cfg`), or using generic placeholder names like `data`, `result`, `temp`, `item` that require surrounding context to understand. This degrades readability, increases cognitive load, and makes code harder for both humans and AI to reason about correctly. + +**The rule:** +Follow these naming conventions consistently across all TypeScript files: + +- **Variables and functions:** `camelCase` — `userProfile`, `fetchOrder`, `handleSubmit` +- **Types, interfaces, and classes:** `PascalCase` — `UserProfile`, `OrderSummary`, `AuthService` +- **Module-level constants:** `SCREAMING_SNAKE_CASE` — `MAX_RETRY_COUNT`, `DEFAULT_TIMEOUT_MS` +- **Files:** `kebab-case.ts` for modules and utilities; `PascalCase.tsx` for React components +- **Booleans:** prefix with `is`, `has`, `can`, or `should` — `isLoading`, `hasError`, `canSubmit`, `shouldRetry` +- **Functions:** start with a verb that describes the action — `fetchUser`, `validateInput`, `handleClick`, `buildQuery` +- **Avoid generic names:** never use `data`, `result`, `temp`, `item`, `obj`, `val`, `res` as variable names — use the domain-specific term + +``` +// Variables and functions: camelCase, verb-first functions, domain names +const user = await fetchUser(userId); +const orderTotal = calculateTotal(lineItems); +const isAuthenticated = checkAuthStatus(session); + +// Types and interfaces: PascalCase +interface UserProfile { + id: string; + displayName: string; + emailAddress: string; +} + +// Module-level constants: SCREAMING_SNAKE_CASE +const MAX_RETRY_COUNT = 3; +const DEFAULT_TIMEOUT_MS = 5000; + +// Booleans: is/has/can/should prefix +const isLoading = true; +const hasValidationErrors = errors.length > 0; +const canSubmitForm = isValid && !isSubmitting; + +// Files +// fetch-user.ts — utility module +// UserProfile.tsx — React component +// order-service.ts — service module +``` +``` +// snake_case in TypeScript code +const fetch_user_data = async (id) => { ... }; +const user_profile = await fetch_user_data(userId); + +// Abbreviated names — meaning is lost +const usr = await getUsr(id); +const cfg = loadCfg(); +const res = await req.json(); + +// Generic placeholder names — what does "data" refer to? +let temp = calculateTotal(items); +const data = await fetchUser(id); +const result = validateForm(input); +// ^^ all require reading the RHS to understand what the variable holds + +// Missing verb prefix — function name doesn't describe action +function userById(id: string) { ... } // fetch? find? get? validate? +function loginCheck(session: Session) {} // ambiguous direction +``` + +*See also: `docs/rubric.md` — rule sizing and format standards, [Google TypeScript Style Guide](https://google.github.io/styleguide/tsguide.html) — naming section, [TypeScript Deep Dive — Naming Conventions](https://basarat.gitbook.io/typescript/styleguide)* + +--- + +### Rule: Performance work must start with measurement +*Signal: `general` · Tier: on-demand* + +**Why (failure mode):** +AI coding agents often "optimize" from vibes: they add caches, memoization, batching, or query rewrites before anyone has measured the real bottleneck. That creates a second system to maintain without proving the original problem. The common failure mode is permanent complexity added for no user-visible gain, followed by a slower debugging loop when the real bottleneck shows up elsewhere. + +The opposite failure also happens: agents notice an obviously hot path, but they change it without capturing a before/after signal, so the team cannot tell whether the change helped or regressed the system. + +**The rule:** +Treat performance work as evidence-backed maintenance, not speculative cleanup. + +- Measure before optimizing — capture one concrete baseline first (latency, query count, bundle size, memory, CPU time, or build time) +- Optimize the dominant bottleneck, not every suspicious line +- Keep the first change reversible — prefer the smallest change that can prove or disprove the hypothesis +- Re-measure after the change with the same signal and record the delta +- Remove or avoid "just in case" caches, memoization, or concurrency if no measurement shows they help + +``` +Performance investigation: +- Baseline: checkout endpoint p95 = 840 ms over the last 200 requests +- Suspected bottleneck: duplicate product queries inside cart enrichment +- Change: collapse N+1 fetches into one batched query +- Recheck: checkout endpoint p95 = 430 ms with identical payload size +``` +``` +// Added from instinct, not evidence +const expensiveValue = useMemo(() => computeDashboard(data), [data]); + +// No baseline, no measured hotspot, no proof this helps +``` + +*See also: `docs/rubric.md` — scoring standards for evidence-backed rules, `docs/bootstrap-templates/testing-patterns.md` — pair performance changes with regression tests when the bottleneck sits in business logic, Research Digest #14 — token and context costs are measurable performance constraints, not vibes* + +--- + +### Rule: Bootstrap Template: Scope Boundaries +*Signal: `general` · Tier: alwaysApply* + +**Why (failure mode):** + + +**The rule:** +**Title:** Agent Scope Boundaries +**Loading tier:** `alwaysApply: true` (this must load on every session — scope failures happen on any task) +**Size:** ~30–50 lines (lean; this is foundational) + +```markdown + +*See also: `docs/rubric.md` §Part 6 — Scope Boundary Declarations, `docs/rubric.md` §Part 9 — Reliability Lens, Add a companion high-stakes operations rule in the target repo when migrations, billing, production deploys, or destructive admin actions need explicit confirmation, Research Digest #13 — Agentic Workflow Reliability, Concentrix "12 Failure Patterns of Agentic AI Systems" (Nov 2025)* + +--- + +### Rule: Security Basics — Input Validation and Secret Handling +*Signal: `general` · Tier: alwaysApply* + +**Why (failure mode):** +Agents trust external input without validation, hardcode secrets in source code, or log sensitive data. These are the most common AI-introduced security vulnerabilities per research — agents have seen countless examples of secrets in code and tend to reproduce the pattern without flagging it as dangerous. A single leaked key or unvalidated input can compromise an entire system. + +**The rule:** +- **Never hardcode secrets, API keys, or credentials** — use environment variables loaded at runtime; store secrets in `.env` files that are `.gitignore`d +- **Validate and sanitize all external input before use** — use zod or equivalent schema validation on every request body, query param, and external API response +- **Never log tokens, passwords, or PII** — mask or omit sensitive fields in logs; if a field might be sensitive, omit it +- **Treat all user input as untrusted**, regardless of source — validate server-side even when client-side validation exists + +``` +// Secrets via environment variables +const apiKey = process.env.API_KEY; +if (!apiKey) throw new Error("API_KEY environment variable is required"); + +// .env file — always in .gitignore +// API_KEY=sk-proj-abc123... + +// Schema validation before processing external input +import { z } from "zod"; + +const CreateUserSchema = z.object({ + email: z.string().email(), + name: z.string().min(1).max(100), + role: z.enum(["user", "admin"]), +}); + +export async function createUser(rawInput: unknown) { + const input = CreateUserSchema.parse(rawInput); // throws on invalid input + return db.users.create(input); +} + +// Safe logging — omit sensitive fields +console.log("Auth request", { userId: user.id, email: user.email }); +// NOT: { userId, token, passwordHash } +``` +``` +// Hardcoded secret — will be committed to git, visible in history forever +const apiKey = "sk-proj-abc123xyzDEFGHIJKLMNOP"; +const db = new Client({ password: "hunter2" }); + +// Logging sensitive data — token in log = token in log aggregation = token leaked +console.log("Auth token:", token); +console.log("User login:", { email, password }); // password in plaintext log + +// Trusting external input without validation +app.post("/users", async (req) => { + await db.users.create(req.body); // req.body is untrusted, unvalidated +}); + +// Client-side only validation — bypassed trivially +function submitForm(data) { + if (!data.email) return alert("Email required"); // client guard only + fetch("/api/users", { method: "POST", body: JSON.stringify(data) }); + // server endpoint accepts anything +} +``` + +*See also: `docs/rubric.md` — rule sizing and format standards, [OWASP Top 10](https://owasp.org/www-project-top-ten/) — authoritative web security risk list, [zod](https://zod.dev/) — TypeScript-first schema validation, OWASP: [Secrets Management Cheat Sheet](https://cheatsheetseries.owasp.org/cheatsheets/Secrets_Management_Cheat_Sheet.html)* + +--- + +### Rule: Test Behavior, Not Implementation +*Signal: `testing` · Tier: glob* + +**Why (failure mode):** +AI coding agents default to "make tests pass" mode, not "validate requirements" mode. Left without explicit guidance, agents produce self-validating tests — tests that mirror the implementation rather than asserting business behavior. These tests always pass, yield high coverage numbers, and are worthless: they don't catch bugs, they don't survive refactors, and they create false confidence. The failure is invisible until production. + +The second failure mode is equally common: when a test fails, the agent modifies the test to match the new behavior rather than investigating whether the behavior regression is intentional. This inverts the purpose of testing entirely. + +**The rule:** +Tests assert requirements, not implementations. Tests are specifications; they define how code *should* behave, not how it currently *does* behave. + +**Core rules:** + +1. **Assert concrete values** — Use literal expected values (`31.49`, `"unauthorized"`, `[]`), not re-derived expressions that mirror implementation logic +2. **Name tests as behavioral specs** — `"should reject login after 5 failed attempts"`, not `"test authService"` +3. **Use `describe` as a contract** — Each `describe` block should read as a feature/scenario; each `it` should be a verifiable claim +4. **Test isolation is mandatory** — Each test must run independently. No shared mutable state between tests. Reset fixtures before each test. +5. **Investigate before modifying** — When a test fails, determine whether the test is wrong or the code is wrong before changing anything. Failing tests are signals, not obstacles. +6. **Realistic test data** — Use data that represents actual production scenarios. Avoid placeholders (`"x"`, `"test@test.com"`, `1`). + +**For AI agents specifically:** When generating or modifying tests, announce "Test failure detected. Investigating..." and analyze: (1) the assertion, (2) the business requirement, (3) the implementation — before changing either. + +``` +// Business requirement tested directly with concrete values +describe('calculateOrderTotal', () => { + it('should apply 10% discount for orders over $100', () => { + const items = [{ price: 60 }, { price: 50 }]; // $110 total + expect(calculateOrderTotal(items)).toBe(99); // $110 - 10% = $99 + }); + + it('should add 8% sales tax after discounts', () => { + const items = [{ price: 100 }]; + expect(calculateOrderTotal(items)).toBeCloseTo(108); // $100 + 8% tax + }); + + it('should return 0 for empty cart', () => { + expect(calculateOrderTotal([])).toBe(0); + }); +}); + +// Auth: behavioral scenarios, not implementation mirrors +describe('AuthService.login', () => { + beforeEach(() => resetAuthState()); // explicit reset + + it('should issue a JWT token on successful login', async () => { + const result = await auth.login({ email: 'user@example.com', password: 'correct-password' }); + expect(result.token).toMatch(/^eyJ/); // JWT prefix + expect(result.expiresIn).toBe(3600); + }); + + it('should lock the account after 5 consecutive failed attempts', async () => { + for (let i = 0; i < 5; i++) { + await auth.login({ email: 'user@example.com', password: 'wrong' }).catch(() => {}); + } + const result = await auth.login({ email: 'user@example.com', password: 'correct-password' }); + expect(result.error).toBe('account_locked'); + }); +}); +``` +``` +// Self-validating test — mirrors implementation, catches nothing +describe('calculateOrderTotal', () => { + it('should calculate total', () => { + const items = [{ price: 10 }, { price: 20 }]; + const result = calculateOrderTotal(items); + // Tests the function against ITSELF — any bug in the implementation passes + expect(result).toBe(items.reduce((sum, item) => sum + item.price, 0)); + }); +}); + +// Modifying the test to match broken behavior — wrong response to failure +describe('UserService', () => { + it('should return user data', async () => { + const user = await getUser('123'); + // Regression introduced: function now returns null sometimes + // Wrong fix: change assertion to allow null + expect(user?.name || null).toBeDefined(); // ← this hides the bug + }); +}); + +// Shared state between tests — order-dependent failures +let cart: Cart; +describe('Cart', () => { + it('should add items', () => { + cart = new Cart(); // set once + cart.add({ sku: 'A', qty: 1 }); + expect(cart.items).toHaveLength(1); + }); + + it('should calculate total', () => { + // Depends on prior test having run first — brittle + expect(cart.total()).toBe(9.99); + }); +}); +``` + +*See also: `docs/bootstrap-templates/error-handling.md` — pair with testing; errors need test coverage too, Martin Fowler: Tests as executable specifications, Vitest docs: [Coverage configuration](https://vitest.dev/config/#coverage), jsmanifest.com: 5 Test Integrity Rules for AI Agents (Jan 2026)* + +--- + ### Rule: ESM only — no CommonJS `require()` *Signal: `typescript.esm` · Tier: alwaysApply* diff --git a/scripts/audit-report-format.test.ts b/scripts/audit-report-format.test.ts index 89845f8..1c9758f 100644 --- a/scripts/audit-report-format.test.ts +++ b/scripts/audit-report-format.test.ts @@ -1587,6 +1587,87 @@ test("Heuristic empty state does not invent weak lanes when all scores are alrea ); }); +test("PR mining table uses pass/partial/fail colors for rule signal match", () => { + const report = buildReport( + makeResult({ + prMining: { + status: "available", + repo: "lambda-curry/anvil", + reason: null, + analyzedPrs: 4, + reviewedComments: 12, + substantiveComments: 8, + candidateCount: 3, + artifactPath: null, + findings: [ + { + theme: "naming", + label: "Naming", + frequency: 4, + score: 2, + uniquePrs: 2, + severity: "low", + representativeness: "medium", + coverageStatus: "match", + commentAlignmentRate: 1, + commentAlignmentStatus: "strong", + samplePaths: ["src/names.ts"], + }, + { + theme: "error-handling", + label: "Error Handling", + frequency: 3, + score: 2, + uniquePrs: 2, + severity: "medium", + representativeness: "medium", + coverageStatus: "missing", + commentAlignmentRate: 0.2, + commentAlignmentStatus: "weak", + samplePaths: ["src/errors.ts"], + }, + { + theme: "general", + label: "General", + frequency: 2, + score: 1, + uniquePrs: 1, + severity: "low", + representativeness: "low", + coverageStatus: "unknown", + commentAlignmentRate: 0, + commentAlignmentStatus: "unknown", + samplePaths: [], + }, + ], + }, + }), + ); + + assert( + report.includes( + "| Naming | 4 comments | 2 PRs (medium) | low | 🟢 signal match | 100% strong |", + ), + "successful signal matches render as green pass", + ); + assert( + report.includes( + "| Error Handling | 3 comments | 2 PRs (medium) | medium | 🔴 no signal | 20% weak |", + ), + "missing signal matches render as red failure", + ); + assert( + report.includes( + "| General | 2 comments | 1 PRs (low) | low | — unknown | — unknown |", + ), + "unknown coverage stays unlabeled instead of yellow", + ); + assert( + !report.includes("🟡 signal match"), + "does not paint a successful match as degraded yellow", + ); +}); + test("Diagnostic navigation stage-status line points to process stages when blockers exist", () => { const report = buildReport( makeResult({ diff --git a/scripts/audit.ts b/scripts/audit.ts index 9c08cec..c1774b3 100644 --- a/scripts/audit.ts +++ b/scripts/audit.ts @@ -5682,7 +5682,7 @@ export function buildReport( for (const finding of result.prMining.findings.slice(0, 10)) { const coverage = finding.coverageStatus === "match" - ? "🟡 signal match" + ? "🟢 signal match" : finding.coverageStatus === "missing" ? "🔴 no signal" : "— unknown"; diff --git a/scripts/bootstrap-generate.test.ts b/scripts/bootstrap-generate.test.ts index c282029..7452a2f 100644 --- a/scripts/bootstrap-generate.test.ts +++ b/scripts/bootstrap-generate.test.ts @@ -55,7 +55,16 @@ function restoreAll() { } // Import after mocks are set up -import { main } from "./bootstrap-generate.ts"; +import { + DOCUMENTED_TEMPLATE_EXCLUSIONS, + formatTemplateSkip, + loadTemplateCatalog, + main, + matchesSignal, + normalizeLoadingTier, + parseTemplateContent, +} from "./bootstrap-generate.ts"; +import type { StackSignals } from "./bootstrap-detect.ts"; const FIXTURES_DIR = resolve(import.meta.dir, "__fixtures__"); const SAMPLE_CLI = join(FIXTURES_DIR, "sample-cli-repo"); @@ -424,3 +433,122 @@ describe("bootstrap-generate main()", () => { } }); }); + +function emptySignals(): StackSignals { + return { + projectName: "empty", + projectPath: "/tmp/empty", + packageManager: "npm", + runtime: "node", + framework: "none", + frameworkVersion: null, + routerType: "unknown", + ui: [], + styling: [], + orm: null, + validation: [], + testing: null, + typescript: { present: false, strict: false, paths: false, esm: false }, + configFiles: [], + dirPatterns: [], + scripts: {}, + dependencies: [], + devDependencies: [], + }; +} + +describe("bootstrap template inventory and vocabulary", () => { + test("loads all 18 templates or only documented exclusions", () => { + const catalog = loadTemplateCatalog(); + const loadedIds = catalog.templates.map((template) => template.id).sort(); + const skippedFiles = catalog.skipped.map((issue) => issue.file).sort(); + const documented = DOCUMENTED_TEMPLATE_EXCLUSIONS.map( + (entry) => entry.file, + ).sort(); + + expect(loadedIds.length + skippedFiles.length).toBe(18); + expect(skippedFiles).toEqual(documented); + expect(loadedIds).toContain("scope-boundaries"); + expect(loadedIds).toContain("performance-measure-before-optimize"); + }); + + test("onDemand alias is on-demand, never alwaysApply", () => { + expect(normalizeLoadingTier("onDemand")).toBe("on-demand"); + expect(normalizeLoadingTier("on-demand")).toBe("on-demand"); + expect(normalizeLoadingTier("glob-matched")).toBe("glob"); + expect(normalizeLoadingTier("mystery-tier")).toBeNull(); + + const parsed = parseTemplateContent( + "performance-measure-before-optimize.md", + [ + "# Performance work must start with measurement", + "", + "*Signal: general · Tier: onDemand · Glob: —*", + "", + "## Why (Failure Mode)", + "", + "Measure first.", + "", + "## The Rule", + "", + "Do not optimize from vibes.", + ].join("\n"), + ); + expect(parsed.ok).toBe(true); + if (parsed.ok) { + expect(parsed.template.tier).toBe("on-demand"); + expect(parsed.template.signal).toBe("general"); + } + }); + + test("unknown tier and missing signal produce named diagnostics", () => { + const unknownTier = parseTemplateContent( + "mystery.md", + ["# Mystery", "", "*Signal: general · Tier: nightly · Glob: —*"].join( + "\n", + ), + ); + expect(unknownTier.ok).toBe(false); + if (!unknownTier.ok) { + expect(formatTemplateSkip(unknownTier.issue)).toBe( + "mystery.md: unknown Tier: nightly", + ); + } + + const missingSignal = parseTemplateContent( + "scope-boundaries.md", + ["# Scope", "", "*Last validated: 2026-05-27*"].join("\n"), + ); + expect(missingSignal.ok).toBe(false); + if (!missingSignal.ok) { + expect(formatTemplateSkip(missingSignal.issue)).toBe( + "scope-boundaries.md: missing or invalid field: Signal", + ); + } + + const unknownSignal = parseTemplateContent( + "odd.md", + ["# Odd", "", "*Signal: mystery · Tier: alwaysApply · Glob: —*"].join( + "\n", + ), + ); + expect(unknownSignal.ok).toBe(false); + if (!unknownSignal.ok) { + expect(formatTemplateSkip(unknownSignal.issue)).toBe( + "odd.md: unknown Signal: mystery", + ); + } + }); + + test("general is an explicit match-all signal", () => { + const template = { + id: "security-patterns", + title: "Security", + signal: "general", + failureMode: "x", + rule: "y", + tier: "alwaysApply" as const, + }; + expect(matchesSignal(template, emptySignals())).toBe(true); + }); +}); diff --git a/scripts/bootstrap-generate.ts b/scripts/bootstrap-generate.ts index 5e3ed8d..3382c00 100644 --- a/scripts/bootstrap-generate.ts +++ b/scripts/bootstrap-generate.ts @@ -16,9 +16,61 @@ import { existsSync, readdirSync, readFileSync } from "node:fs"; import { basename, join, resolve } from "node:path"; import { detectStack, type StackSignals } from "./bootstrap-detect.ts"; -// ─── Types ─────────────────────────────────────────────────────────────────── +// ─── Types ─────────────────────────────────────────────────────────────────────── -type LoadingTier = "alwaysApply" | "glob" | "on-demand"; +export type LoadingTier = "alwaysApply" | "glob" | "on-demand"; + +export type TemplateParseIssue = { + file: string; + reason: string; +}; + +export type TemplateParseResult = + | { ok: true; template: RuleTemplate } + | { ok: false; issue: TemplateParseIssue }; + +export type TemplateLoadResult = { + templates: RuleTemplate[]; + skipped: TemplateParseIssue[]; +}; + +export const KNOWN_TEMPLATE_SIGNALS = [ + "typescript.strict", + "typescript.esm", + "packageManager:bun", + "packageManager:pnpm", + "framework:nextjs:app", + "ui:react", + "orm:prisma", + "orm:drizzle", + "validation:zod", + "testing:vitest", + "testing:jest", + "styling:tailwind", + "general", + "testing", + "language:typescript", +] as const; + +export type KnownTemplateSignal = (typeof KNOWN_TEMPLATE_SIGNALS)[number]; + +const KNOWN_SIGNAL_SET = new Set(KNOWN_TEMPLATE_SIGNALS); + +const TIER_ALIASES: Record = { + alwaysapply: "alwaysApply", + "always-apply": "alwaysApply", + "always apply": "alwaysApply", + glob: "glob", + "glob-matched": "glob", + "on-demand": "on-demand", + ondemand: "on-demand", + "on demand": "on-demand", +}; + +export function normalizeLoadingTier(raw: string): LoadingTier | null { + const key = raw.trim().toLowerCase(); + return TIER_ALIASES[key] ?? null; +} export type RuleTemplate = { id: string; @@ -282,84 +334,91 @@ const TEMPLATES_DIR = join( "bootstrap-templates", ); -export function parseTemplateFile(filePath: string): RuleTemplate | null { - try { - const content = readFileSync(filePath, "utf8"); - const lines = content.split("\n"); - - // H1 heading → title - const titleLine = lines.find((l) => l.startsWith("# ")); - if (!titleLine) return null; - const title = titleLine.replace(/^# /, "").trim(); - - // id from filename - const id = basename(filePath, ".md"); - - // Signal/Tier/Glob from the italic metadata line - const metaLine = lines.find((l) => l.startsWith("*Signal:")); - if (!metaLine) return null; - const signalMatch = metaLine.match(/Signal:\s*([^·]+)/); - const tierMatch = metaLine.match(/Tier:\s*([^·*]+)/); - const globMatch = metaLine.match(/Glob:\s*([^*]+)/); - if (!signalMatch) return null; - const signal = signalMatch[1].trim(); - const tierRaw = tierMatch ? tierMatch[1].trim() : "alwaysApply"; - const tier: LoadingTier = - tierRaw === "glob" - ? "glob" - : tierRaw === "on-demand" - ? "on-demand" - : "alwaysApply"; - const glob = globMatch - ? globMatch[1].trim().replace(/\s*$/, "") - : undefined; - - // Extract sections by heading - function extractSection(heading: string): string { - const startIdx = lines.findIndex((l) => l.trim() === `## ${heading}`); - if (startIdx === -1) return ""; - const endIdx = lines.findIndex( - (l, i) => i > startIdx && l.startsWith("## "), - ); - const sectionLines = lines.slice( - startIdx + 1, - endIdx === -1 ? undefined : endIdx, - ); - return sectionLines.join("\n").trim(); - } +function failTemplate(file: string, reason: string): TemplateParseResult { + return { ok: false, issue: { file, reason } }; +} - const failureMode = extractSection("Why (Failure Mode)"); - const rule = extractSection("The Rule"); - - // Extract DO/DON'T examples from code blocks in the Examples section - const examplesSection = extractSection("Examples"); - function extractCodeBlock( - text: string, - marker: string, - ): string | undefined { - const markerIdx = text.indexOf(marker); - if (markerIdx === -1) return undefined; - const afterMarker = text.slice(markerIdx + marker.length); - const codeStart = afterMarker.indexOf("```"); - if (codeStart === -1) return undefined; - const codeEnd = afterMarker.indexOf("```", codeStart + 3); - if (codeEnd === -1) return undefined; - return afterMarker - .slice(codeStart + 3, codeEnd) - .replace(/^[a-z]*\n/, "") - .trim(); - } - const doExample = extractCodeBlock(examplesSection, "### ✅ DO"); - const dontExample = extractCodeBlock(examplesSection, "### ❌ DON'T"); +export function parseTemplateContent( + fileName: string, + content: string, +): TemplateParseResult { + const lines = content.split("\n"); - // See Also - const seeAlsoSection = extractSection("See Also"); - const seeAlso = seeAlsoSection - .split("\n") - .map((l) => l.replace(/^- /, "").trim()) - .filter((l) => l.length > 0); + const titleLine = lines.find((l) => l.startsWith("# ")); + if (!titleLine) { + return failTemplate(fileName, "missing or invalid field: Title (H1)"); + } + const title = titleLine.replace(/^# /, "").trim(); + const id = basename(fileName, ".md"); - return { + const metaLine = lines.find((l) => l.startsWith("*Signal:")); + if (!metaLine) { + return failTemplate(fileName, "missing or invalid field: Signal"); + } + const signalMatch = metaLine.match(/Signal:\s*([^·*]+)/); + const tierMatch = metaLine.match(/Tier:\s*([^·*]+)/); + const globMatch = metaLine.match(/Glob:\s*([^*]+)/); + const signal = signalMatch?.[1]?.trim() ?? ""; + if (!signal) { + return failTemplate(fileName, "missing or invalid field: Signal"); + } + if (!KNOWN_SIGNAL_SET.has(signal)) { + return failTemplate(fileName, `unknown Signal: ${signal}`); + } + + const tierRaw = tierMatch?.[1]?.trim() ?? ""; + if (!tierRaw) { + return failTemplate(fileName, "missing or invalid field: Tier"); + } + const tier = normalizeLoadingTier(tierRaw); + if (!tier) { + return failTemplate(fileName, `unknown Tier: ${tierRaw}`); + } + + const glob = globMatch ? globMatch[1].trim().replace(/\s*$/, "") : undefined; + + function extractSection(heading: string): string { + const startIdx = lines.findIndex((l) => l.trim() === `## ${heading}`); + if (startIdx === -1) return ""; + const endIdx = lines.findIndex( + (l, i) => i > startIdx && l.startsWith("## "), + ); + const sectionLines = lines.slice( + startIdx + 1, + endIdx === -1 ? undefined : endIdx, + ); + return sectionLines.join("\n").trim(); + } + + const failureMode = extractSection("Why (Failure Mode)"); + const rule = extractSection("The Rule"); + + const examplesSection = extractSection("Examples"); + function extractCodeBlock(text: string, marker: string): string | undefined { + const markerIdx = text.indexOf(marker); + if (markerIdx === -1) return undefined; + const afterMarker = text.slice(markerIdx + marker.length); + const codeStart = afterMarker.indexOf("```"); + if (codeStart === -1) return undefined; + const codeEnd = afterMarker.indexOf("```", codeStart + 3); + if (codeEnd === -1) return undefined; + return afterMarker + .slice(codeStart + 3, codeEnd) + .replace(/^[a-z]*\n/, "") + .trim(); + } + const doExample = extractCodeBlock(examplesSection, "### ✅ DO"); + const dontExample = extractCodeBlock(examplesSection, "### ❌ DON'T"); + + const seeAlsoSection = extractSection("See Also"); + const seeAlso = seeAlsoSection + .split("\n") + .map((l) => l.replace(/^- /, "").trim()) + .filter((l) => l.length > 0); + + return { + ok: true, + template: { id, title, signal, @@ -370,38 +429,71 @@ export function parseTemplateFile(filePath: string): RuleTemplate | null { tier, glob: glob && glob !== "—" ? glob : undefined, seeAlso: seeAlso.length > 0 ? seeAlso : undefined, - }; - } catch { - return null; + }, + }; +} + +export function parseTemplateFile(filePath: string): TemplateParseResult { + try { + return parseTemplateContent( + basename(filePath), + readFileSync(filePath, "utf8"), + ); + } catch (err) { + return failTemplate( + basename(filePath), + `unreadable template: ${(err as Error).message}`, + ); } } -export function loadTemplatesFromFiles(): RuleTemplate[] { - if (!existsSync(TEMPLATES_DIR)) { - return []; +export const DOCUMENTED_TEMPLATE_EXCLUSIONS: ReadonlyArray<{ + file: string; + reason: string; +}> = []; + +export function loadTemplateCatalog( + templatesDir = TEMPLATES_DIR, +): TemplateLoadResult { + if (!existsSync(templatesDir)) { + return { templates: [], skipped: [] }; } try { - const files = readdirSync(TEMPLATES_DIR) + const files = readdirSync(templatesDir) .filter((f) => f.endsWith(".md")) .sort(); - if (files.length === 0) return []; + if (files.length === 0) return { templates: [], skipped: [] }; const templates: RuleTemplate[] = []; + const skipped: TemplateParseIssue[] = []; + const documented = new Map( + DOCUMENTED_TEMPLATE_EXCLUSIONS.map((entry) => [entry.file, entry.reason]), + ); for (const file of files) { - const parsed = parseTemplateFile(join(TEMPLATES_DIR, file)); - if (parsed) { - templates.push(parsed); + const documentedReason = documented.get(file); + if (documentedReason) { + skipped.push({ file, reason: documentedReason }); + continue; + } + const parsed = parseTemplateFile(join(templatesDir, file)); + if (parsed.ok) { + templates.push(parsed.template); + } else { + skipped.push(parsed.issue); } } - return templates; + return { templates, skipped }; } catch { - return []; + return { templates: [], skipped: [] }; } } -// Load templates: from files if available, fall back to hardcoded -const fileTemplates = loadTemplatesFromFiles(); -const RULE_TEMPLATES: RuleTemplate[] = - fileTemplates.length > 0 ? fileTemplates : RULE_TEMPLATES_HARDCODED; +export function loadTemplatesFromFiles(): RuleTemplate[] { + return loadTemplateCatalog().templates; +} + +export function formatTemplateSkip(issue: TemplateParseIssue): string { + return `${issue.file}: ${issue.reason}`; +} // ─── Signal matching ────────────────────────────────────────────────────────── @@ -425,6 +517,10 @@ export function matchesSignal( if (sig === "testing:vitest") return signals.testing === "vitest"; if (sig === "testing:jest") return signals.testing === "jest"; if (sig === "styling:tailwind") return signals.styling.includes("tailwind"); + if (sig === "testing") return signals.testing !== null; + if (sig === "language:typescript") return signals.typescript.present; + // Match-all: stack-independent hygiene that should be suggested for every project. + if (sig === "general") return true; return false; } @@ -680,8 +776,10 @@ export async function main() { process.exit(1); } - // Match templates to detected signals - const matched = RULE_TEMPLATES.filter((t) => matchesSignal(t, signals)); + const catalog = loadTemplateCatalog(); + const activeTemplates = + catalog.templates.length > 0 ? catalog.templates : RULE_TEMPLATES_HARDCODED; + const matched = activeTemplates.filter((t) => matchesSignal(t, signals)); const draft = buildDraft(signals, matched); @@ -709,7 +807,7 @@ export async function main() { await Bun.write(outPath, draft); const templateSource = - fileTemplates.length > 0 + catalog.templates.length > 0 ? `files (${TEMPLATES_DIR})` : "hardcoded fallback"; console.log(`Bootstrap draft written: ${outPath}`); @@ -717,53 +815,55 @@ export async function main() { `Stack signals detected: ${signals.ui.length > 0 ? signals.ui.join(", ") : "—"} | ${signals.framework} | ${signals.packageManager}`, ); console.log( - `Rules generated: ${matched.length} of ${RULE_TEMPLATES.length} templates matched (source: ${templateSource})`, + `Rules generated: ${matched.length} of ${activeTemplates.length} templates matched (source: ${templateSource})`, ); console.log(`Matched: ${matched.map((r) => r.id).join(", ") || "(none)"}`); + if (catalog.skipped.length > 0) { + console.warn( + `Skipped templates (${catalog.skipped.length}): ${catalog.skipped.map(formatTemplateSkip).join("; ")}`, + ); + } - // Detect stub/placeholder project and advise re-run - if (matched.length < 3) { - // Check for placeholder scripts ("echo 'TODO'") across package.json files (root + workspaces) - const pkgFiles = [join(root, "package.json")]; - // Also check workspace sub-packages up to 2 levels deep - for (const dir of ["apps", "packages", "src"]) { - const subDir = join(root, dir); - if (existsSync(subDir)) { - try { - const { readdirSync } = await import("node:fs"); - for (const entry of readdirSync(subDir)) { - pkgFiles.push(join(subDir, entry, "package.json")); - } - } catch { - /* ignore */ - } - } - } - - let totalStubs = 0; - for (const pkgPath of pkgFiles) { - if (!existsSync(pkgPath)) { - continue; - } + // Detect stub/placeholder project and advise re-run. This is independent of + // how many templates matched — match-all `general` rules would otherwise hide stubs. + const pkgFiles = [join(root, "package.json")]; + for (const dir of ["apps", "packages", "src"]) { + const subDir = join(root, dir); + if (existsSync(subDir)) { try { - const pkg = JSON.parse(await Bun.file(pkgPath).text()); - const scripts: Record = pkg.scripts ?? {}; - totalStubs += Object.values(scripts).filter( - (s) => typeof s === "string" && /echo\s+['"]?TODO/i.test(s), - ).length; + const { readdirSync } = await import("node:fs"); + for (const entry of readdirSync(subDir)) { + pkgFiles.push(join(subDir, entry, "package.json")); + } } catch { /* ignore */ } } + } - if (totalStubs > 0) { - console.log( - `\n⚠️ Stub scripts detected: ${totalStubs} placeholder(s) found (echo 'TODO...') across workspace packages.`, - ); - console.log( - " Re-run bootstrap-generate.ts after wiring the real tech stack for fuller rule coverage.", - ); + let totalStubs = 0; + for (const pkgPath of pkgFiles) { + if (!existsSync(pkgPath)) { + continue; } + try { + const pkg = JSON.parse(await Bun.file(pkgPath).text()); + const scripts: Record = pkg.scripts ?? {}; + totalStubs += Object.values(scripts).filter( + (s) => typeof s === "string" && /echo\s+['"]?TODO/i.test(s), + ).length; + } catch { + /* ignore */ + } + } + + if (totalStubs > 0) { + console.log( + `\n⚠️ Stub scripts detected: ${totalStubs} placeholder(s) found (echo 'TODO...') across workspace packages.`, + ); + console.log( + " Re-run bootstrap-generate.ts after wiring the real tech stack for fuller rule coverage.", + ); } } diff --git a/scripts/verify-self-audit-proof.test.ts b/scripts/verify-self-audit-proof.test.ts index 01f03a4..9debf40 100644 --- a/scripts/verify-self-audit-proof.test.ts +++ b/scripts/verify-self-audit-proof.test.ts @@ -887,13 +887,13 @@ test("a new PR-mined theme row is not a determinism failure", () => { // commit, the rows shifted, and "Documentation" was compared against a newly // inserted "Error Handling". const checkedIn = MINED_TABLE( - `| Naming | 9 comments | 3 PRs (medium) | low | 🟡 signal match | 100% strong | -| Documentation | 5 comments | 2 PRs (medium) | low | 🟡 signal match | 100% strong |`, + `| Naming | 9 comments | 3 PRs (medium) | low | 🟢 signal match | 100% strong | +| Documentation | 5 comments | 2 PRs (medium) | low | 🟢 signal match | 100% strong |`, ); const fresh = MINED_TABLE( - `| Naming | 11 comments | 5 PRs (high) | medium | 🟡 signal match | 100% strong | -| Error Handling | 3 comments | 3 PRs (medium) | medium | 🟡 signal match | 100% strong | -| Documentation | 5 comments | 2 PRs (medium) | low | 🟡 signal match | 100% strong |`, + `| Naming | 11 comments | 5 PRs (high) | medium | 🟢 signal match | 100% strong | +| Error Handling | 3 comments | 3 PRs (medium) | medium | 🟢 signal match | 100% strong | +| Documentation | 5 comments | 2 PRs (medium) | low | 🟢 signal match | 100% strong |`, ); expect(compareSelfAuditReports(checkedIn, fresh).failures).toEqual([]); @@ -903,7 +903,7 @@ test("the mined table disappearing entirely is still a failure", () => { // The line held deliberately: rows changing is churn and normalizes away, but // the table vanishing means mining itself broke, and that should stay loud. const checkedIn = MINED_TABLE( - `| Naming | 9 comments | 3 PRs (medium) | low | 🟡 signal match | 100% strong |`, + `| Naming | 9 comments | 3 PRs (medium) | low | 🟢 signal match | 100% strong |`, ); const fresh = MINED_TABLE(""); @@ -916,7 +916,7 @@ test("a real scoring change is still caught through the mined table", () => { // The guard: normalizing live PR data must not blind the proof to the // deterministic surface it exists to protect. const checkedIn = MINED_TABLE( - `| Naming | 9 comments | 3 PRs (medium) | low | 🟡 signal match | 100% strong |`, + `| Naming | 9 comments | 3 PRs (medium) | low | 🟢 signal match | 100% strong |`, ); const fresh = checkedIn.replace("Issues found | none", "Issues found | 3");