diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 6f53459..fb117c8 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -37,6 +37,9 @@ The URL can be a site root or a specific page. When pointing to a site root, `af | `-v, --verbose` | | Show per-page details for checks with issues | | `--fixes` | | Show per-check fix suggestions for warn/fail checks (only needed for `text` format; the other two formats include fix suggestions automatically) | | `--score` | | Include scoring data in JSON output (only usable with `json` output format) | +| `-q, --quiet` | | Suppress progress output on stderr | + +While a run is in progress, each check writes a numbered progress line to stderr as it starts and completes, e.g. `[7/24] llms-txt-directive-html... done (47 tested, 3 fetch errors, 12s)`. Progress goes to stderr for every format, so piped `json` or `scorecard` output on stdout stays parseable. Use `--quiet` to suppress it (e.g. in CI logs). **Which format to use:** diff --git a/src/cli/commands/check.ts b/src/cli/commands/check.ts index 5bfd494..6cd9f59 100644 --- a/src/cli/commands/check.ts +++ b/src/cli/commands/check.ts @@ -3,7 +3,9 @@ import { normalizeCanonical, normalizeUrl, runChecks } from '../../runner.js'; import { formatText } from '../formatters/text.js'; import { formatJson } from '../formatters/json.js'; import { formatScorecard } from '../formatters/scorecard.js'; +import { formatProgressEvent } from '../formatters/progress.js'; import type { + CheckProgressEvent, PageConfigEntry, RunnerOptions, SamplingStrategy, @@ -45,6 +47,7 @@ export function registerCheckCommand(program: Command): void { .option('--pass-threshold ', 'Pass threshold in characters') .option('--fail-threshold ', 'Fail threshold in characters') .option('-v, --verbose', 'Show per-page details for checks with issues') + .option('-q, --quiet', 'Suppress progress output on stderr') .option('--fixes', 'Show fix suggestions for warn/fail checks') .option('--score', 'Include scoring data in JSON output') .option( @@ -187,7 +190,9 @@ export function registerCheckCommand(program: Command): void { 10, ); - if (format !== 'json') { + const quiet = !!opts.quiet; + + if (format !== 'json' && !quiet) { const parsed = new URL(url); const target = parsed.pathname && parsed.pathname !== '/' @@ -314,6 +319,12 @@ export function registerCheckCommand(program: Command): void { ...(parityPassThreshold != null && { parityPassThreshold }), ...(parityWarnThreshold != null && { parityWarnThreshold }), ...(parityExclusions && { parityExclusions }), + // Progress goes to stderr so it never contaminates parseable stdout + // formats (json, piped scorecard output). + ...(!quiet && { + onProgress: (event: CheckProgressEvent) => + process.stderr.write(formatProgressEvent(event)), + }), }; const validation = validateRunnerOptions(runnerOptions); diff --git a/src/cli/formatters/progress.ts b/src/cli/formatters/progress.ts new file mode 100644 index 0000000..11a0f4c --- /dev/null +++ b/src/cli/formatters/progress.ts @@ -0,0 +1,36 @@ +import type { CheckProgressEvent } from '../../types.js'; + +function formatDuration(ms: number): string { + if (ms < 1000) return `${Math.round(ms)}ms`; + const seconds = Math.round(ms / 1000); + if (seconds < 60) return `${seconds}s`; + return `${Math.floor(seconds / 60)}m ${seconds % 60}s`; +} + +/** + * Render a runner progress event as a stderr line fragment. A check's 'start' + * event opens the line (no newline) so a stalled check leaves its name visible; + * the 'complete' event finishes it. + */ +export function formatProgressEvent(event: CheckProgressEvent): string { + if (event.phase === 'start') { + return `[${event.index}/${event.total}] ${event.checkId}... `; + } + + if (event.result.status === 'skip') { + return 'skipped\n'; + } + + const details = event.result.details ?? {}; + const parts: string[] = []; + const tested = details.testedPages ?? details.tested; + if (typeof tested === 'number') { + parts.push(`${tested} tested`); + } + const fetchErrors = details.fetchErrors; + if (typeof fetchErrors === 'number' && fetchErrors > 0) { + parts.push(`${fetchErrors} fetch error${fetchErrors === 1 ? '' : 's'}`); + } + parts.push(formatDuration(event.durationMs)); + return `done (${parts.join(', ')})\n`; +} diff --git a/src/index.ts b/src/index.ts index 1ca9576..eab0d1c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,6 +6,9 @@ export type { CheckOptions, CheckFunction, CheckDefinition, + CheckProgressEvent, + CheckProgressStartEvent, + CheckProgressCompleteEvent, RunnerOptions, ReportResult, AgentDocsConfig, diff --git a/src/runner.ts b/src/runner.ts index 0089cd7..8137b8e 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -136,63 +136,77 @@ export async function runChecks( } const results: CheckResult[] = []; - - for (const check of allChecks) { - // Filter by requested check IDs if provided - if (checkIds && checkIds.length > 0 && !checkIds.includes(check.id)) { - continue; - } + const onProgress = options?.onProgress; + + // Filter by requested check IDs if provided + const selectedChecks = + checkIds && checkIds.length > 0 + ? allChecks.filter((check) => checkIds.includes(check.id)) + : allChecks; + + let index = 0; + for (const check of selectedChecks) { + index++; + onProgress?.({ phase: 'start', checkId: check.id, index, total: selectedChecks.length }); + const startedAt = Date.now(); + + let result: CheckResult; + // Explicitly excluded skips are intentionally not stored in previousResults so + // dependent checks see "dependency never ran" and can run in standalone mode — + // matching the behaviour of checks filtered out by checkIds. + let storeInPreviousResults = true; // Emit a skip result for explicitly excluded checks without running them. - // Intentionally not stored in previousResults so dependent checks see - // "dependency never ran" and can run in standalone mode — matching the - // behaviour of checks filtered out by checkIds. if (skipCheckIds.includes(check.id)) { - results.push({ + result = { id: check.id, category: check.category, status: 'skip', message: 'Check skipped (excluded via --skip-checks)', - }); - continue; - } - - // Check dependencies — only skip if at least one dependency actually ran and none passed. - // If no dependencies ran at all (e.g. filtered out via --checks), let the check handle - // standalone mode itself. - if (check.dependsOn.length > 0) { - const normalized = normalizeDeps(check.dependsOn); - const anyDepRan = normalized.some((orGroup) => + }; + storeInPreviousResults = false; + } else if ( + // Check dependencies — only skip if at least one dependency actually ran and none + // passed. If no dependencies ran at all (e.g. filtered out via --checks), let the + // check handle standalone mode itself. + check.dependsOn.length > 0 && + normalizeDeps(check.dependsOn).some((orGroup) => orGroup.some((id) => ctx.previousResults.has(id)), - ); - if (anyDepRan && !checkDependenciesMet(check.dependsOn, ctx.previousResults)) { - const result: CheckResult = { + ) && + !checkDependenciesMet(check.dependsOn, ctx.previousResults) + ) { + result = { + id: check.id, + category: check.category, + status: 'skip', + message: 'Skipped: dependency check did not pass', + dependsOn: normalizeDeps(check.dependsOn).flat(), + }; + } else { + try { + result = await check.run(ctx); + } catch (err) { + result = { id: check.id, category: check.category, - status: 'skip', - message: 'Skipped: dependency check did not pass', - dependsOn: normalized.flat(), + status: 'error', + message: `Check error: ${err instanceof Error ? err.message : String(err)}`, }; - results.push(result); - ctx.previousResults.set(check.id, result); - continue; } } - try { - const result = await check.run(ctx); - results.push(result); - ctx.previousResults.set(check.id, result); - } catch (err) { - const result: CheckResult = { - id: check.id, - category: check.category, - status: 'error', - message: `Check error: ${err instanceof Error ? err.message : String(err)}`, - }; - results.push(result); + results.push(result); + if (storeInPreviousResults) { ctx.previousResults.set(check.id, result); } + onProgress?.({ + phase: 'complete', + checkId: check.id, + index, + total: selectedChecks.length, + result, + durationMs: Date.now() - startedAt, + }); } const summary = { diff --git a/src/types.ts b/src/types.ts index 4eded6a..da91b9c 100644 --- a/src/types.ts +++ b/src/types.ts @@ -173,6 +173,29 @@ export interface DiscoveredFile { crossHostRedirect?: boolean; } +export interface CheckProgressStartEvent { + phase: 'start'; + checkId: string; + /** 1-based position among the checks selected for this run. */ + index: number; + /** Number of checks selected for this run. */ + total: number; +} + +export interface CheckProgressCompleteEvent { + phase: 'complete'; + checkId: string; + /** 1-based position among the checks selected for this run. */ + index: number; + /** Number of checks selected for this run. */ + total: number; + result: CheckResult; + /** Wall-clock time the check took, in ms (0 for skipped checks). */ + durationMs: number; +} + +export type CheckProgressEvent = CheckProgressStartEvent | CheckProgressCompleteEvent; + export interface RunnerOptions extends CheckOptions { /** Only run checks matching these IDs. If empty, run all. */ checkIds?: string[]; @@ -180,6 +203,8 @@ export interface RunnerOptions extends CheckOptions { skipCheckIds?: string[]; /** Curated page list from config or --urls. Used when samplingStrategy is 'curated'. */ curatedPages?: PageConfigEntry[]; + /** Called as each selected check starts and completes. The CLI uses this for stderr progress. */ + onProgress?: (event: CheckProgressEvent) => void; } export interface ReportResult { diff --git a/test/unit/cli/check-command.test.ts b/test/unit/cli/check-command.test.ts index 27f5ab9..f40de4c 100644 --- a/test/unit/cli/check-command.test.ts +++ b/test/unit/cli/check-command.test.ts @@ -1089,4 +1089,113 @@ describe('check command config integration', () => { writeSpy.mockRestore(); }); + + it('writes per-check progress lines to stderr', async () => { + server.use( + http.get('http://cmd-progress.local/llms.txt', () => HttpResponse.text(VALID_LLMS_TXT)), + http.get( + 'http://cmd-progress.local/docs/llms.txt', + () => new HttpResponse(null, { status: 404 }), + ), + ); + + const stdoutSpy = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + const stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + + const { run } = await import('../../../src/cli/index.js'); + await run([ + 'node', + 'afdocs', + 'check', + 'http://cmd-progress.local', + '--checks', + 'llms-txt-exists', + '--request-delay', + '0', + ]); + await new Promise((r) => setTimeout(r, 100)); + + const stderr = stderrSpy.mock.calls.map((c) => c[0]).join(''); + expect(stderr).toContain('[1/1] llms-txt-exists... '); + expect(stderr).toContain('done ('); + + stdoutSpy.mockRestore(); + stderrSpy.mockRestore(); + }); + + it('--quiet suppresses the banner and progress lines', async () => { + server.use( + http.get('http://cmd-quiet.local/llms.txt', () => HttpResponse.text(VALID_LLMS_TXT)), + http.get( + 'http://cmd-quiet.local/docs/llms.txt', + () => new HttpResponse(null, { status: 404 }), + ), + ); + + const stdoutSpy = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + const stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + + const { run } = await import('../../../src/cli/index.js'); + await run([ + 'node', + 'afdocs', + 'check', + 'http://cmd-quiet.local', + '--checks', + 'llms-txt-exists', + '--request-delay', + '0', + '--quiet', + ]); + await new Promise((r) => setTimeout(r, 100)); + + const stderr = stderrSpy.mock.calls.map((c) => c[0]).join(''); + expect(stderr).not.toContain('Running checks on'); + expect(stderr).not.toContain('[1/1]'); + + // The report itself still goes to stdout + const stdout = stdoutSpy.mock.calls.map((c) => c[0]).join(''); + expect(stdout).toContain('llms-txt-exists'); + + stdoutSpy.mockRestore(); + stderrSpy.mockRestore(); + }); + + it('keeps stdout parseable in json format while progress goes to stderr', async () => { + server.use( + http.get('http://cmd-json-progress.local/llms.txt', () => HttpResponse.text(VALID_LLMS_TXT)), + http.get( + 'http://cmd-json-progress.local/docs/llms.txt', + () => new HttpResponse(null, { status: 404 }), + ), + ); + + const stdoutSpy = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + const stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + + const { run } = await import('../../../src/cli/index.js'); + await run([ + 'node', + 'afdocs', + 'check', + 'http://cmd-json-progress.local', + '--checks', + 'llms-txt-exists', + '--format', + 'json', + '--request-delay', + '0', + ]); + await new Promise((r) => setTimeout(r, 100)); + + const stdout = stdoutSpy.mock.calls.map((c) => c[0]).join(''); + const parsed = JSON.parse(stdout.trim()); + expect(parsed.results[0].id).toBe('llms-txt-exists'); + + const stderr = stderrSpy.mock.calls.map((c) => c[0]).join(''); + expect(stderr).toContain('[1/1] llms-txt-exists... '); + + stdoutSpy.mockRestore(); + stderrSpy.mockRestore(); + }); }); diff --git a/test/unit/cli/progress-formatter.test.ts b/test/unit/cli/progress-formatter.test.ts new file mode 100644 index 0000000..450d548 --- /dev/null +++ b/test/unit/cli/progress-formatter.test.ts @@ -0,0 +1,97 @@ +import { describe, it, expect } from 'vitest'; +import { formatProgressEvent } from '../../../src/cli/formatters/progress.js'; +import type { CheckResult } from '../../../src/types.js'; + +function result(overrides: Partial = {}): CheckResult { + return { + id: 'llms-txt-directive-html', + category: 'content-discoverability', + status: 'pass', + message: 'ok', + ...overrides, + }; +} + +describe('formatProgressEvent', () => { + it('formats a start event as a numbered partial line', () => { + const line = formatProgressEvent({ + phase: 'start', + checkId: 'llms-txt-directive-html', + index: 7, + total: 24, + }); + expect(line).toBe('[7/24] llms-txt-directive-html... '); + }); + + it('formats a completion with tested pages, fetch errors, and duration', () => { + const line = formatProgressEvent({ + phase: 'complete', + checkId: 'llms-txt-directive-html', + index: 7, + total: 24, + result: result({ details: { testedPages: 47, fetchErrors: 3 } }), + durationMs: 12000, + }); + expect(line).toBe('done (47 tested, 3 fetch errors, 12s)\n'); + }); + + it('omits tested and fetch-error parts when the check reports neither', () => { + const line = formatProgressEvent({ + phase: 'complete', + checkId: 'llms-txt-exists', + index: 1, + total: 24, + result: result(), + durationMs: 200, + }); + expect(line).toBe('done (200ms)\n'); + }); + + it('omits the fetch-error part when the count is zero', () => { + const line = formatProgressEvent({ + phase: 'complete', + checkId: 'http-status-codes', + index: 2, + total: 24, + result: result({ details: { testedPages: 10, fetchErrors: 0 } }), + durationMs: 3000, + }); + expect(line).toBe('done (10 tested, 3s)\n'); + }); + + it('uses the singular form for one fetch error', () => { + const line = formatProgressEvent({ + phase: 'complete', + checkId: 'http-status-codes', + index: 2, + total: 24, + result: result({ details: { fetchErrors: 1 } }), + durationMs: 3000, + }); + expect(line).toBe('done (1 fetch error, 3s)\n'); + }); + + it('formats a skipped check as skipped without timing detail', () => { + const line = formatProgressEvent({ + phase: 'complete', + checkId: 'llms-txt-valid', + index: 3, + total: 24, + result: result({ status: 'skip' }), + durationMs: 0, + }); + expect(line).toBe('skipped\n'); + }); + + it('formats durations of a minute or more as minutes and seconds', () => { + const line = formatProgressEvent({ + phase: 'complete', + checkId: 'markdown-url-support', + index: 9, + total: 24, + result: result(), + durationMs: 90_000, + }); + expect(line).toBe('done (1m 30s)\n'); + }); +}); diff --git a/test/unit/runner.test.ts b/test/unit/runner.test.ts index 906e75d..1d9f37a 100644 --- a/test/unit/runner.test.ts +++ b/test/unit/runner.test.ts @@ -3,6 +3,7 @@ import { http, HttpResponse } from 'msw'; import { setupServer } from 'msw/node'; import { createContext, normalizeCanonical, normalizeUrl, runChecks } from '../../src/runner.js'; import { registerCheck } from '../../src/checks/registry.js'; +import type { CheckProgressEvent } from '../../src/types.js'; import '../../src/checks/index.js'; import { mockSitemapNotFound } from '../helpers/mock-sitemap-not-found.js'; @@ -649,3 +650,125 @@ describe('runner', () => { ).rejects.toThrow('Unknown check ID'); }); }); + +describe('runChecks onProgress', () => { + it('emits start and complete events with position, total, and the check result', async () => { + registerCheck({ + id: '_progress-a', + category: 'content-discoverability', + description: 'Progress test check A', + dependsOn: [], + run: async () => ({ + id: '_progress-a', + category: 'content-discoverability', + status: 'pass' as const, + message: 'ok', + details: { testedPages: 5, fetchErrors: 2 }, + }), + }); + registerCheck({ + id: '_progress-b', + category: 'content-discoverability', + description: 'Progress test check B', + dependsOn: [], + run: async () => ({ + id: '_progress-b', + category: 'content-discoverability', + status: 'pass' as const, + message: 'ok', + }), + }); + + const events: CheckProgressEvent[] = []; + await runChecks('http://progress.local', { + checkIds: ['_progress-a', '_progress-b'], + requestDelay: 0, + onProgress: (e) => events.push(e), + }); + + expect(events).toHaveLength(4); + expect(events[0]).toEqual({ phase: 'start', checkId: '_progress-a', index: 1, total: 2 }); + expect(events[1].phase).toBe('complete'); + expect(events[1].checkId).toBe('_progress-a'); + if (events[1].phase === 'complete') { + expect(events[1].result.status).toBe('pass'); + expect(events[1].result.details).toMatchObject({ testedPages: 5, fetchErrors: 2 }); + expect(events[1].durationMs).toBeGreaterThanOrEqual(0); + } + expect(events[2]).toEqual({ phase: 'start', checkId: '_progress-b', index: 2, total: 2 }); + expect(events[3].phase).toBe('complete'); + expect(events[3].checkId).toBe('_progress-b'); + }); + + it('emits skip completion for checks excluded via skipCheckIds', async () => { + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); + + const events: CheckProgressEvent[] = []; + await runChecks('http://progress-skip.local', { + checkIds: ['_progress-a', '_progress-b'], + skipCheckIds: ['_progress-b'], + requestDelay: 0, + onProgress: (e) => events.push(e), + }); + + expect(events).toHaveLength(4); + const last = events[3]; + expect(last.phase).toBe('complete'); + expect(last.checkId).toBe('_progress-b'); + if (last.phase === 'complete') { + expect(last.result.status).toBe('skip'); + } + warnSpy.mockRestore(); + }); + + it('emits skip completion when a dependency gate fails', async () => { + server.use( + http.get('http://progress-dep.local/llms.txt', () => new HttpResponse(null, { status: 404 })), + http.get( + 'http://progress-dep.local/docs/llms.txt', + () => new HttpResponse(null, { status: 404 }), + ), + ); + + const events: CheckProgressEvent[] = []; + await runChecks('http://progress-dep.local', { + checkIds: ['llms-txt-exists', 'llms-txt-valid'], + requestDelay: 0, + onProgress: (e) => events.push(e), + }); + + expect(events).toHaveLength(4); + const last = events[3]; + expect(last.checkId).toBe('llms-txt-valid'); + if (last.phase === 'complete') { + expect(last.result.status).toBe('skip'); + expect(last.result.message).toContain('dependency'); + } + }); + + it('emits error completion when a check throws', async () => { + registerCheck({ + id: '_progress-throws', + category: 'content-discoverability', + description: 'Progress test check that throws', + dependsOn: [], + run: () => { + throw new Error('progress boom'); + }, + }); + + const events: CheckProgressEvent[] = []; + await runChecks('http://progress-throws.local', { + checkIds: ['_progress-throws'], + requestDelay: 0, + onProgress: (e) => events.push(e), + }); + + expect(events).toHaveLength(2); + const last = events[1]; + if (last.phase === 'complete') { + expect(last.result.status).toBe('error'); + expect(last.result.message).toContain('progress boom'); + } + }); +});