Add sitemap.xml and a complete robots.txt for xo.run

docs/implementation.md

@@ -260,6 +260,7 @@ See [`fixtures.md`](fixtures.md) and [`testing.md`](testing.md).
 | Language pages | Leaders, examples, Result/Option, modules |
 | Search index | Content records for new pages |
 | Snippets | Match **current** `syntax.md` |
+| Discovery | `/sitemap.xml` of the public catalog; `/robots.txt` on `https://xo.run` |
 
 ### 2.14 Tooling ecosystem (as needed)
 

www/README.md

@@ -24,8 +24,8 @@ just try     # wasm + npm --prefix www run dev
 
 `npm run dev` starts the local site. `npm run lint`, `npm run format`,
 `npm run test`, and `npm run build` validate the site before publishing
-`www/dist`. The docs-first homepage, primary nav, and Documents catalog live
-in `src/docs/site.ts`.
+`www/dist`. The docs-first homepage, primary nav, Documents catalog, and
+discovery files live in `src/docs/site.ts`.
 
 ## Cloudflare Pages
 
@@ -50,6 +50,10 @@ URLs are real pages. Unknown paths still use the `public/404.html` → `/?/path`
 bounce and `index.html` restore script. Custom domain: `public/CNAME` →
 `xo.run`. Wasm bindings stay in `public/echo-wasm/`.
 
+`/robots.txt` is a static file. `/sitemap.xml` is emitted at build from the
+public catalog (`staticPages` and site chrome). Both use `https://xo.run`.
+Privacy and Terms are listed only when those pages exist.
+
 ## Search
 
 Docs search uses MiniSearch over content in `src/docs/`. Open the palette from

www/SITE.md

@@ -206,6 +206,14 @@ playground run then executes the checked MIR and captures `io.print`.
 Filesystem, net, process, and tasks fail with a playground-host error.
 Compile and native run stay on `xo` (LLVM).
 
+## Discovery
+
+`/sitemap.xml` lists the public catalog on `https://xo.run`: home, Install,
+Try, catalog and footer routes, and every shipped docs, Book, Echo 2026, and
+std page. `/robots.txt` allows crawlers and points at that sitemap. Privacy
+and Terms are listed only when those pages exist. Do not list the GitHub
+Pages host.
+
 ## Out of scope (later)
 
 Richer download tabs, `/e26` URL rename to `/echo-2026`.

www/package.json

@@ -10,11 +10,12 @@
     "format": "oxfmt --check .",
     "lint": "oxlint . --ignore-pattern public/echo-wasm",
     "preview": "vite preview",
-    "test": "npm run test:docs && npm run test:prose && npm run test:std-ref && npm run test:try",
+    "test": "npm run test:docs && npm run test:prose && npm run test:std-ref && npm run test:try && npm run test:sitemap",
     "test:docs": "node scripts/verify-docs-pages.mjs",
     "test:std-ref": "node scripts/verify-std-reference.mjs",
     "test:prose": "node scripts/verify-prose.mjs",
     "test:try": "node scripts/verify-try.mjs",
+    "test:sitemap": "node scripts/verify-sitemap.mjs",
     "sync:tree-sitter": "node scripts/sync-tree-sitter.mjs",
     "postinstall": "npm run sync:tree-sitter"
   },

www/public/_headers

@@ -5,3 +5,9 @@
 /echo-wasm/*.wasm
   Content-Type: application/wasm
   Cache-Control: public, max-age=0, must-revalidate
+
+/sitemap.xml
+  Content-Type: application/xml; charset=utf-8
+
+/robots.txt
+  Content-Type: text/plain; charset=utf-8

www/public/robots.txt

@@ -0,0 +1,4 @@
+User-agent: *
+Allow: /
+
+Sitemap: https://xo.run/sitemap.xml

www/scripts/verify-sitemap.mjs

@@ -0,0 +1,150 @@
+/**
+ * Verifies public discovery files for xo.run:
+ * - sitemap.xml lists the public catalog on https://xo.run
+ * - robots.txt has User-agent rules and a Sitemap: line
+ * - Privacy and Terms are listed only when those pages exist
+ * - github.io is never listed
+ *
+ * Loads src/docs/site.ts, content.ts, and static-html.ts through Vite SSR.
+ */
+import { readFileSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { createServer } from "vite";
+
+const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+
+const server = await createServer({
+  root,
+  logLevel: "error",
+  server: { middlewareMode: true },
+  appType: "custom",
+});
+
+try {
+  const site = await server.ssrLoadModule("/src/docs/site.ts");
+  const content = await server.ssrLoadModule("/src/docs/content.ts");
+  const staticHtml = await server.ssrLoadModule("/src/docs/static-html.ts");
+
+  const {
+    collectPublicCatalogPaths,
+    omittedCatalogPaths,
+    publicCatalogUrl,
+    publicChromePaths,
+    publicSiteOrigin,
+    publicSurfacePaths,
+    renderRobotsTxt,
+    renderSitemapXml,
+  } = site;
+  const { docsPages } = content;
+  const { staticPages } = staticHtml;
+
+  const failures = [];
+
+  function fail(message) {
+    failures.push(message);
+  }
+
+  if (publicSiteOrigin !== "https://xo.run") {
+    fail(`publicSiteOrigin must be https://xo.run, got ${publicSiteOrigin}`);
+  }
+
+  const existingPagePaths = [
+    ...new Set([...staticPages().map((page) => page.path), ...docsPages.map((page) => page.path)]),
+  ];
+  const existing = new Set(existingPagePaths);
+  const catalogPaths = collectPublicCatalogPaths(existingPagePaths);
+  const sitemap = renderSitemapXml(catalogPaths);
+  const robots = renderRobotsTxt();
+  const committedRobots = readFileSync(path.join(root, "public/robots.txt"), "utf8");
+
+  if (!sitemap.startsWith('<?xml version="1.0" encoding="UTF-8"?>')) {
+    fail("sitemap.xml must be a real XML document");
+  }
+  if (!sitemap.includes('<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">')) {
+    fail("sitemap.xml must use the sitemaps.org urlset namespace");
+  }
+
+  const requiredPaths = [
+    ...publicSurfacePaths,
+    ...publicChromePaths(),
+    "/docs",
+    "/docs/std",
+    "/docs/leaders",
+    "/book",
+    "/e26",
+    "/e26/spec",
+    "/try",
+    "/install",
+  ];
+  for (const required of requiredPaths) {
+    if (!catalogPaths.includes(required)) {
+      fail(`public catalog missing ${required}`);
+    }
+    const loc = publicCatalogUrl(required);
+    if (!sitemap.includes(`<loc>${loc}</loc>`)) {
+      fail(`sitemap.xml missing ${loc}`);
+    }
+  }
+
+  for (const page of docsPages) {
+    if (!catalogPaths.includes(page.path)) {
+      fail(`docs page ${page.path} missing from the public catalog`);
+    }
+  }
+
+  for (const candidate of omittedCatalogPaths) {
+    if (existing.has(candidate)) {
+      if (!catalogPaths.includes(candidate)) {
+        fail(`public catalog missing existing page ${candidate}`);
+      }
+      continue;
+    }
+    if (catalogPaths.includes(candidate)) {
+      fail(`public catalog must not invent ${candidate}`);
+    }
+    if (sitemap.toLowerCase().includes(candidate)) {
+      fail(`sitemap.xml must not list ${candidate} until that page exists`);
+    }
+  }
+
+  if (/github\.io/i.test(sitemap) || /github\.io/i.test(robots)) {
+    fail("discovery files must not list the GitHub Pages host");
+  }
+  if (!/https:\/\/xo\.run\//.test(sitemap) || /http:\/\/xo\.run/.test(sitemap)) {
+    fail("sitemap.xml must use https://xo.run as the live host");
+  }
+
+  if (!/^User-agent:\s+\*/m.test(robots)) {
+    fail("robots.txt must include a User-agent rule");
+  }
+  if (!/^Allow:\s+\//m.test(robots)) {
+    fail("robots.txt must include an Allow rule");
+  }
+  if (!/^Sitemap:\s+https:\/\/xo\.run\/sitemap\.xml$/m.test(robots)) {
+    fail("robots.txt must point Sitemap: at https://xo.run/sitemap.xml");
+  }
+  if (committedRobots !== robots) {
+    fail("public/robots.txt must match renderRobotsTxt()");
+  }
+
+  if (failures.length) {
+    console.error(JSON.stringify({ ok: false, failures }, null, 2));
+    process.exitCode = 1;
+  } else {
+    console.log(
+      JSON.stringify(
+        {
+          ok: true,
+          origin: publicSiteOrigin,
+          urls: catalogPaths.length,
+          sitemapBytes: sitemap.length,
+        },
+        null,
+        2,
+      ),
+    );
+  }
+} finally {
+  await server.close();
+}

www/src/docs/site.ts

@@ -1,8 +1,21 @@
 /**
- * Public site chrome: homepage, primary nav, and the Documents hub catalog.
- * Pages and tests load this module; do not duplicate the lists in UI or fixtures.
+ * Public site chrome: homepage, primary nav, Documents hub catalog, and the
+ * discovery files (`/sitemap.xml`, `/robots.txt`). Pages and tests load this
+ * module; do not duplicate the lists in UI or fixtures.
  */
 
+/** Live public host. Do not emit github.io URLs in discovery files. */
+export const publicSiteOrigin = "https://xo.run";
+
+/** Top-level routes that are not docs pages. */
+export const publicSurfacePaths = ["/", "/install", "/try"] as const;
+
+/**
+ * Legal routes stay out of the sitemap until the pages exist.
+ * Do not add these paths here when inventing placeholder URLs.
+ */
+export const omittedCatalogPaths = ["/privacy", "/terms"] as const;
+
 export type SiteNavItem = {
   label: string;
   to: string;
@@ -279,6 +292,58 @@ export const docsHubCatalog: DocsCatalogGroup[] = [
   },
 ];
 
+/**
+ * Public catalog paths for `/sitemap.xml`.
+ * `existingPagePaths` is the shipped HTML catalog (`staticPages` / `docsPages`).
+ * Privacy and Terms are included only when those pages already exist.
+ */
+export function collectPublicCatalogPaths(existingPagePaths: readonly string[]): string[] {
+  const existing = new Set(existingPagePaths);
+  const paths = new Set<string>([
+    ...publicSurfacePaths,
+    ...publicChromePaths(),
+    ...existingPagePaths,
+  ]);
+
+  for (const omitted of omittedCatalogPaths) {
+    if (!existing.has(omitted)) {
+      paths.delete(omitted);
+    }
+  }
+
+  return [...paths].sort((left, right) => left.localeCompare(right));
+}
+
+export function publicCatalogUrl(path: string): string {
+  if (!path.startsWith("/")) {
+    throw new Error(`catalog path must be absolute, got ${path}`);
+  }
+  if (path === "/") {
+    return `${publicSiteOrigin}/`;
+  }
+  return `${publicSiteOrigin}${path}`;
+}
+
+export function renderSitemapXml(paths: readonly string[]): string {
+  const urls = paths
+    .map((path) => `  <url>\n    <loc>${escapeHtml(publicCatalogUrl(path))}</loc>\n  </url>`)
+    .join("\n");
+
+  return [
+    `<?xml version="1.0" encoding="UTF-8"?>`,
+    `<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">`,
+    urls,
+    `</urlset>`,
+    ``,
+  ].join("\n");
+}
+
+export function renderRobotsTxt(): string {
+  return ["User-agent: *", "Allow: /", "", `Sitemap: ${publicSiteOrigin}/sitemap.xml`, ""].join(
+    "\n",
+  );
+}
+
 function escapeHtml(text: string): string {
   return text
     .replaceAll("&", "&amp;")

www/vite.config.ts

@@ -10,7 +10,12 @@ import {
   type DocsSearchAsset,
   type DocsSemanticAsset,
 } from "./src/docs/search";
-import { renderStaticHomeAndHub } from "./src/docs/site";
+import {
+  collectPublicCatalogPaths,
+  renderRobotsTxt,
+  renderSitemapXml,
+  renderStaticHomeAndHub,
+} from "./src/docs/site";
 import { distFileForPath, escapeHtml, staticPages, type StaticPage } from "./src/docs/static-html";
 
 const docsSearchIndexDevFileName = "indices/search.json";
@@ -232,6 +237,47 @@ function applyStaticPage(html: string, page: StaticPage): string {
   });
 }
 
+function publicCatalogPaths() {
+  return collectPublicCatalogPaths(staticPages().map((page) => page.path));
+}
+
+function siteDiscoveryPlugin(): Plugin {
+  return {
+    name: "site-discovery",
+    configureServer(server) {
+      server.middlewares.use((request, response, next) => {
+        const requestPath = request.url?.split("?", 1)[0] ?? "";
+
+        if (requestPath === "/sitemap.xml") {
+          response.setHeader("Content-Type", "application/xml; charset=utf-8");
+          response.end(renderSitemapXml(publicCatalogPaths()));
+          return;
+        }
+
+        if (requestPath === "/robots.txt") {
+          response.setHeader("Content-Type", "text/plain; charset=utf-8");
+          response.end(renderRobotsTxt());
+          return;
+        }
+
+        next();
+      });
+    },
+    generateBundle() {
+      this.emitFile({
+        type: "asset",
+        fileName: "sitemap.xml",
+        source: renderSitemapXml(publicCatalogPaths()),
+      });
+      this.emitFile({
+        type: "asset",
+        fileName: "robots.txt",
+        source: renderRobotsTxt(),
+      });
+    },
+  };
+}
+
 function docsFirstStaticPlugin(): Plugin {
   const fallbackMarker = '<noscript id="docs-first-fallback"></noscript>';
 
@@ -310,5 +356,11 @@ export default defineConfig({
       },
     },
   },
-  plugins: [docsSearchIndexPlugin(), docsFirstStaticPlugin(), react(), tailwindcss()],
+  plugins: [
+    siteDiscoveryPlugin(),
+    docsSearchIndexPlugin(),
+    docsFirstStaticPlugin(),
+    react(),
+    tailwindcss(),
+  ],
 });