feat: support path-prefix canonical base URLs in --canonical-origin

docs/reference/cli.md

@@ -156,11 +156,11 @@ The same option is available in `agent-docs.config.yml` as `options.urlPathPatte
 
 ### Request behavior
 
-| Flag                       | Default | Description                                 |
-| -------------------------- | ------- | ------------------------------------------- |
-| `--max-concurrency <n>`    | `3`     | Maximum concurrent HTTP requests            |
-| `--request-delay <ms>`     | `200`   | Delay between requests in milliseconds      |
-| `--canonical-origin <url>` |         | The production domain your content links to |
+| Flag                       | Default | Description                                                                          |
+| -------------------------- | ------- | ------------------------------------------------------------------------------------ |
+| `--max-concurrency <n>`    | `3`     | Maximum concurrent HTTP requests                                                     |
+| `--request-delay <ms>`     | `200`   | Delay between requests in milliseconds                                               |
+| `--canonical-origin <url>` |         | The production base URL (origin, or origin plus a path prefix) your content links to |
 
 AFDocs enforces delays between requests and caps concurrent connections to avoid overloading your server. Adjust these if you need gentler or faster runs:
 
@@ -172,11 +172,19 @@ afdocs check https://docs.example.com --request-delay 500 --max-concurrency 1
 afdocs check https://docs.example.com --request-delay 50 --max-concurrency 10
 ```
 
-Use `--canonical-origin` when your site's URLs in `sitemap.xml` and `llms.txt` don't match the domain you're testing, such as preview deployments or localhost.
+Use `--canonical-origin` when your site's URLs in `sitemap.xml` and `llms.txt` don't match the base URL you're testing, such as preview deployments or localhost. It accepts either a bare origin or an origin plus a path prefix:
+
+- **Origin only** — rewrites every URL on that host, regardless of path.
+- **Origin plus a path prefix** — rewrites only URLs under that prefix, remapping them to the base URL you pass to `check` (the prefixes need not match).
+
+Avoid a canonical base that is a path-prefix of the target base itself (for example, previews served under the production docs path): URLs in fetched content that already point at the target would be rewritten again. AFDocs prints a warning when it detects this overlap.
 
 ```bash
-# Test a preview deployment
+# Origin only: rewrite all example.com URLs to the preview host
 afdocs check https://preview-xyz-example.app/docs --canonical-origin https://example.com
+
+# Path prefix: production docs live under /docs, but your preview serves them under /preview
+afdocs check http://localhost:3000/preview --canonical-origin https://example.com/docs
 ```
 
 ### llms.txt selection

docs/reference/config-file.md

@@ -93,7 +93,7 @@ Override default runner options. All fields are optional:
 | `requestTimeout`        | `30000`     | Timeout for individual HTTP requests in milliseconds                                                     |
 | `preferredLocale`       | auto-detect | Preferred locale for URL discovery (e.g. `en`, `fr`, `ja`)                                               |
 | `preferredVersion`      | auto-detect | Preferred version for URL discovery (e.g. `v3`, `2.x`)                                                   |
-| `canonicalOrigin`       |             | The production domain your content links to                                                              |
+| `canonicalOrigin`       |             | The production base URL (origin, or origin plus a path prefix) your content links to                     |
 | `llmsTxtUrl`            |             | Explicit llms.txt URL to use as canonical (overrides the discovery heuristic; see CLI docs)              |
 | `thresholds.pass`       | `50000`     | Page size pass threshold in characters                                                                   |
 | `thresholds.fail`       | `100000`    | Page size fail threshold in characters                                                                   |

docs/run-locally.md

@@ -101,7 +101,7 @@ Some checks may behave differently against a local server:
 
 When you build your site locally, generated files like `llms.txt` and `sitemap.xml` typically contain your production domain. AFDocs sees URLs pointing to `https://docs.example.com` but you're testing `http://localhost:3000`, so origin comparisons fail and checks like `llms-txt-coverage` report 0% coverage.
 
-Use `--canonical-origin` to tell AFDocs which production domain to rewrite:
+Use `--canonical-origin` to tell AFDocs which production base URL (origin, or origin plus a path prefix) to rewrite:
 
 ```bash
 npm run build

src/cli/commands/check.ts

@@ -1,5 +1,5 @@
 import type { Command } from 'commander';
-import { normalizeUrl, runChecks } from '../../runner.js';
+import { normalizeCanonical, normalizeUrl, runChecks } from '../../runner.js';
 import { formatText } from '../formatters/text.js';
 import { formatJson } from '../formatters/json.js';
 import { formatScorecard } from '../formatters/scorecard.js';
@@ -73,7 +73,7 @@ export function registerCheckCommand(program: Command): void {
     )
     .option(
       '--canonical-origin <url>',
-      'The production domain your content links to (for preview/staging testing)',
+      'The production base URL (origin, or origin plus a path prefix) your content links to, rewritten to the target for preview/staging testing',
     )
     .option(
       '--llms-txt-url <url>',
@@ -209,13 +209,31 @@ export function registerCheckCommand(program: Command): void {
       if (rawCanonical) {
         const normalized = normalizeUrl(rawCanonical);
         try {
-          canonicalOrigin = new URL(normalized).origin;
-          const targetOrigin = new URL(url).origin;
-          if (canonicalOrigin === targetOrigin) {
+          const parsedCanonical = new URL(normalized);
+          // Normalize identically to createContext so the warning reflects the value used.
+          canonicalOrigin = normalizeCanonical(normalized);
+          // The flag has no effect when the canonical resolves to what the rewrite would
+          // produce anyway: for a sub-path canonical that's the full target base; for an
+          // origin-only canonical it's just the target origin (the path is untouched).
+          const parsedTarget = new URL(url);
+          const canonicalHasSubPath = parsedCanonical.pathname !== '/';
+          const noEffect = canonicalHasSubPath
+            ? canonicalOrigin === normalizeCanonical(url)
+            : parsedCanonical.origin === parsedTarget.origin;
+          if (noEffect) {
             process.stderr.write(
-              `Warning: --canonical-origin "${canonicalOrigin}" is the same as the target origin. The flag has no effect.\n`,
+              `Warning: --canonical-origin "${canonicalOrigin}" is the same as the target. The flag has no effect.\n`,
             );
             canonicalOrigin = undefined;
+          } else if (
+            canonicalHasSubPath &&
+            normalizeCanonical(url).startsWith(`${canonicalOrigin}/`)
+          ) {
+            // Prefix rewriting can't tell a production URL from one already pointing
+            // at the target, so self-referencing URLs in content get re-prefixed.
+            process.stderr.write(
+              `Warning: --canonical-origin "${canonicalOrigin}" is a path-prefix of the target base. URLs in fetched content that already point at the target will be rewritten again (doubled path segments).\n`,
+            );
           }
         } catch {
           canonicalOrigin = normalized;

src/http.ts

@@ -11,7 +11,9 @@ interface RateLimitedHttpClientOptions {
   requestDelay: number;
   requestTimeout: number;
   maxConcurrency: number;
+  /** Canonical base URL to find in bodies (origin, or origin plus a path prefix). */
   canonicalOrigin?: string;
+  /** Value to replace it with: the target origin, or the full target base for a path-prefix canonical. */
   targetOrigin?: string;
 }
 
@@ -24,9 +26,15 @@ function escapeRegExp(s: string): string {
 export function createHttpClient(options: RateLimitedHttpClientOptions): HttpClient {
   let lastRequestTime = 0;
   let activeRequests = 0;
+  // Match the canonical base only at a URL boundary: end-of-string or one of these
+  // delimiters. `)` and `,` are included so URLs inside markdown links `[x](url)` and
+  // prose `url, next` rewrite; the rare tradeoff is a path segment like `/docs,2024`
+  // being treated as the `/docs` prefix. `<` is included so a URL ending exactly at
+  // the canonical base rewrites when an XML closing tag follows (`<loc>url</loc>` in
+  // sitemaps); a literal `<` can never appear in a valid URL, so it is unambiguous.
   const originPattern =
     options.canonicalOrigin && options.targetOrigin
-      ? new RegExp(escapeRegExp(options.canonicalOrigin) + '(?=[/\\s"\'\\]>]|$)', 'g')
+      ? new RegExp(escapeRegExp(options.canonicalOrigin) + '(?=[/?#\\s"\'\\]),<>]|$)', 'g')
       : null;
 
   async function waitForSlot(): Promise<void> {
@@ -81,7 +89,11 @@ export function createHttpClient(options: RateLimitedHttpClientOptions): HttpCli
             if (/text|xml|json|markdown/.test(ct)) {
               const body = await response.text();
               originPattern.lastIndex = 0;
-              const rewritten = body.replace(originPattern, options.targetOrigin);
+              // Use a function replacer so `$` in the target (e.g. a preview path
+              // containing `$'` or `$&`) is inserted literally, not interpreted as a
+              // String.replace replacement pattern.
+              const target = options.targetOrigin;
+              const rewritten = body.replace(originPattern, () => target);
               return {
                 ok: response.ok,
                 status: response.status,

src/runner.ts

@@ -42,6 +42,18 @@ export function normalizeUrl(url: string): string {
   return url;
 }
 
+/**
+ * Normalize a canonical/base URL for the http.ts rewrite regex, which is literal and
+ * case-sensitive: lowercase the host and drop default ports (via URL.origin) while
+ * preserving any sub-path, then strip the trailing slash so it matches path segments.
+ * Must be applied identically to the canonical value and the target base it is compared
+ * against. Assumes `raw` is already scheme-qualified (see normalizeUrl).
+ */
+export function normalizeCanonical(raw: string): string {
+  const parsed = new URL(raw);
+  return `${parsed.origin}${parsed.pathname}`.replace(/\/+$/, '');
+}
+
 export function createContext(baseUrl: string, options?: Partial<RunnerOptions>): CheckContext {
   if (options) {
     if (options.canonicalOrigin) {
@@ -60,6 +72,18 @@ export function createContext(baseUrl: string, options?: Partial<RunnerOptions>)
   const merged = { ...DEFAULT_OPTIONS, ...options };
   baseUrl = normalizeUrl(baseUrl);
   const url = new URL(baseUrl);
+  const normalizedBaseUrl = baseUrl.replace(/\/$/, '');
+
+  // Normalize the canonical value once, here, so CLI and direct createContext() callers
+  // behave identically. Keep merged.canonicalOrigin in sync with the value wired below.
+  let canonicalOrigin: string | undefined;
+  if (merged.canonicalOrigin) {
+    canonicalOrigin = normalizeCanonical(merged.canonicalOrigin);
+    merged.canonicalOrigin = canonicalOrigin;
+  }
+
+  // A sub-path canonical rewrites to the full preview base; origin-only swaps origins.
+  const canonicalHasSubPath = Boolean(canonicalOrigin && new URL(canonicalOrigin).pathname !== '/');
 
   // Fail fast when the target port is on the WHATWG fetch bad port list:
   // undici would refuse every request, turning one port choice into a wall
@@ -70,15 +94,19 @@ export function createContext(baseUrl: string, options?: Partial<RunnerOptions>)
   }
 
   return {
-    baseUrl: baseUrl.replace(/\/$/, ''),
+    baseUrl: normalizedBaseUrl,
     origin: url.origin,
     previousResults: new Map(),
     http: createHttpClient({
       requestDelay: merged.requestDelay,
       requestTimeout: merged.requestTimeout,
       maxConcurrency: merged.maxConcurrency,
-      canonicalOrigin: merged.canonicalOrigin,
-      targetOrigin: merged.canonicalOrigin ? url.origin : undefined,
+      canonicalOrigin,
+      targetOrigin: canonicalOrigin
+        ? canonicalHasSubPath
+          ? normalizedBaseUrl
+          : url.origin
+        : undefined,
     }),
     options: merged,
     pageCache: new Map(),

src/types.ts

@@ -94,7 +94,12 @@ export interface CheckOptions {
   preferredLocale?: string;
   /** Preferred version for URL discovery (e.g. 'v3', '2.x', 'latest'). Overrides auto-detection from baseUrl. */
   preferredVersion?: string;
-  /** Canonical origin to rewrite in fetched content (for preview/staging testing). */
+  /**
+   * Canonical base URL to rewrite in fetched content (for preview/staging testing).
+   * Accepts an origin (`https://prod.example.com`) or an origin plus a path prefix
+   * (`https://prod.example.com/docs`); when a path prefix is given, matching URLs are
+   * rewritten to the full target base.
+   */
   canonicalOrigin?: string;
   /** Pass threshold for llms-txt-coverage (0–100). Default 95. */
   coveragePassThreshold?: number;

test/unit/cli/check-command.test.ts

@@ -758,7 +758,159 @@ describe('check command config integration', () => {
     await new Promise((r) => setTimeout(r, 100));
 
     const stderr = stderrSpy.mock.calls.map((c) => c[0]).join('');
-    expect(stderr).toContain('same as the target origin');
+    expect(stderr).toContain('same as the target');
+    expect(stderr).toContain('no effect');
+
+    stdoutSpy.mockRestore();
+    stderrSpy.mockRestore();
+  });
+
+  it('warns when origin-only --canonical-origin matches target whose URL has a path', async () => {
+    server.use(
+      http.get('http://cmd-canon-path.local/docs/llms.txt', () =>
+        HttpResponse.text(VALID_LLMS_TXT),
+      ),
+      http.get('http://cmd-canon-path.local/llms.txt', () => HttpResponse.text(VALID_LLMS_TXT)),
+    );
+
+    const stdoutSpy = vi.spyOn(process.stdout, 'write').mockImplementation(() => true);
+    const stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true);
+
+    const { run } = await import('../../../src/cli/index.js');
+    await run([
+      'node',
+      'afdocs',
+      'check',
+      'http://cmd-canon-path.local/docs',
+      '--canonical-origin',
+      'http://cmd-canon-path.local',
+      '--checks',
+      'llms-txt-exists',
+      '--request-delay',
+      '0',
+    ]);
+    await new Promise((r) => setTimeout(r, 100));
+
+    const stderr = stderrSpy.mock.calls.map((c) => c[0]).join('');
+    // Origin-only canonical == target origin → no effect, even though target has a path.
+    expect(stderr).toContain('no effect');
+
+    stdoutSpy.mockRestore();
+    stderrSpy.mockRestore();
+  });
+
+  it('does not suppress --canonical-origin when same origin but different sub-path', async () => {
+    server.use(
+      http.get('http://cmd-canon-subpath.local/preview/llms.txt', () =>
+        HttpResponse.text(VALID_LLMS_TXT),
+      ),
+      http.get(
+        'http://cmd-canon-subpath.local/llms.txt',
+        () => new HttpResponse(null, { status: 404 }),
+      ),
+      http.get(
+        'http://cmd-canon-subpath.local/docs/llms.txt',
+        () => new HttpResponse(null, { status: 404 }),
+      ),
+    );
+
+    const stdoutSpy = vi.spyOn(process.stdout, 'write').mockImplementation(() => true);
+    const stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true);
+
+    const { run } = await import('../../../src/cli/index.js');
+    await run([
+      'node',
+      'afdocs',
+      'check',
+      'http://cmd-canon-subpath.local/preview',
+      '--canonical-origin',
+      'http://cmd-canon-subpath.local/aws/en',
+      '--checks',
+      'llms-txt-exists',
+      '--request-delay',
+      '0',
+    ]);
+    await new Promise((r) => setTimeout(r, 100));
+
+    const stdout = stdoutSpy.mock.calls.map((c) => c[0]).join('');
+    const stderr = stderrSpy.mock.calls.map((c) => c[0]).join('');
+    expect(stderr).not.toContain('no effect');
+    expect(stdout).toContain('llms-txt-exists');
+
+    stdoutSpy.mockRestore();
+    stderrSpy.mockRestore();
+  });
+
+  it('warns without suppressing when --canonical-origin is a path-prefix of the target base', async () => {
+    server.use(
+      http.get('http://cmd-canon-overlap.local/docs/preview/llms.txt', () =>
+        HttpResponse.text(VALID_LLMS_TXT),
+      ),
+      http.get(
+        'http://cmd-canon-overlap.local/llms.txt',
+        () => new HttpResponse(null, { status: 404 }),
+      ),
+      http.get(
+        'http://cmd-canon-overlap.local/docs/llms.txt',
+        () => new HttpResponse(null, { status: 404 }),
+      ),
+    );
+
+    const stdoutSpy = vi.spyOn(process.stdout, 'write').mockImplementation(() => true);
+    const stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true);
+
+    const { run } = await import('../../../src/cli/index.js');
+    await run([
+      'node',
+      'afdocs',
+      'check',
+      'http://cmd-canon-overlap.local/docs/preview',
+      '--canonical-origin',
+      'http://cmd-canon-overlap.local/docs',
+      '--checks',
+      'llms-txt-exists',
+      '--request-delay',
+      '0',
+    ]);
+    await new Promise((r) => setTimeout(r, 100));
+
+    const stdout = stdoutSpy.mock.calls.map((c) => c[0]).join('');
+    const stderr = stderrSpy.mock.calls.map((c) => c[0]).join('');
+    // Self-referencing URLs in content will double-rewrite; warn but stay active.
+    expect(stderr).toContain('path-prefix of the target base');
+    expect(stderr).not.toContain('no effect');
+    expect(stdout).toContain('llms-txt-exists');
+
+    stdoutSpy.mockRestore();
+    stderrSpy.mockRestore();
+  });
+
+  it('warns when a path-prefix --canonical-origin equals the target base', async () => {
+    server.use(
+      http.get('http://cmd-canon-eq.local/docs/llms.txt', () => HttpResponse.text(VALID_LLMS_TXT)),
+      http.get('http://cmd-canon-eq.local/llms.txt', () => HttpResponse.text(VALID_LLMS_TXT)),
+    );
+
+    const stdoutSpy = vi.spyOn(process.stdout, 'write').mockImplementation(() => true);
+    const stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true);
+
+    const { run } = await import('../../../src/cli/index.js');
+    await run([
+      'node',
+      'afdocs',
+      'check',
+      'http://cmd-canon-eq.local/docs',
+      '--canonical-origin',
+      'http://cmd-canon-eq.local/docs',
+      '--checks',
+      'llms-txt-exists',
+      '--request-delay',
+      '0',
+    ]);
+    await new Promise((r) => setTimeout(r, 100));
+
+    const stderr = stderrSpy.mock.calls.map((c) => c[0]).join('');
+    // Path-prefix canonical resolves to the same base as the target → no-op rewrite.
     expect(stderr).toContain('no effect');
 
     stdoutSpy.mockRestore();