diff --git a/docs/implementation.md b/docs/implementation.md index 66b3c8ab..a5719537 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -260,6 +260,7 @@ See [`fixtures.md`](fixtures.md) and [`testing.md`](testing.md). | Language pages | Leaders, examples, Result/Option, modules | | Search index | Content records for new pages | | Snippets | Match **current** `syntax.md` | +| Discovery | `/sitemap.xml` of the public catalog; `/robots.txt` on `https://xo.run` | ### 2.14 Tooling ecosystem (as needed) diff --git a/www/README.md b/www/README.md index 51903dff..f9be3cdb 100644 --- a/www/README.md +++ b/www/README.md @@ -24,8 +24,8 @@ just try # wasm + npm --prefix www run dev `npm run dev` starts the local site. `npm run lint`, `npm run format`, `npm run test`, and `npm run build` validate the site before publishing -`www/dist`. The docs-first homepage, primary nav, and Documents catalog live -in `src/docs/site.ts`. +`www/dist`. The docs-first homepage, primary nav, Documents catalog, and +discovery files live in `src/docs/site.ts`. ## Cloudflare Pages @@ -50,6 +50,10 @@ URLs are real pages. Unknown paths still use the `public/404.html` → `/?/path` bounce and `index.html` restore script. Custom domain: `public/CNAME` → `xo.run`. Wasm bindings stay in `public/echo-wasm/`. +`/robots.txt` is a static file. `/sitemap.xml` is emitted at build from the +public catalog (`staticPages` and site chrome). Both use `https://xo.run`. +Privacy and Terms are listed only when those pages exist. + ## Search Docs search uses MiniSearch over content in `src/docs/`. Open the palette from diff --git a/www/SITE.md b/www/SITE.md index 6342d1c5..6960bc8e 100644 --- a/www/SITE.md +++ b/www/SITE.md @@ -206,6 +206,14 @@ playground run then executes the checked MIR and captures `io.print`. Filesystem, net, process, and tasks fail with a playground-host error. Compile and native run stay on `xo` (LLVM). +## Discovery + +`/sitemap.xml` lists the public catalog on `https://xo.run`: home, Install, +Try, catalog and footer routes, and every shipped docs, Book, Echo 2026, and +std page. `/robots.txt` allows crawlers and points at that sitemap. Privacy +and Terms are listed only when those pages exist. Do not list the GitHub +Pages host. + ## Out of scope (later) Richer download tabs, `/e26` URL rename to `/echo-2026`. diff --git a/www/package.json b/www/package.json index 1bde3d8f..543b0296 100644 --- a/www/package.json +++ b/www/package.json @@ -10,11 +10,12 @@ "format": "oxfmt --check .", "lint": "oxlint . --ignore-pattern public/echo-wasm", "preview": "vite preview", - "test": "npm run test:docs && npm run test:prose && npm run test:std-ref && npm run test:try", + "test": "npm run test:docs && npm run test:prose && npm run test:std-ref && npm run test:try && npm run test:sitemap", "test:docs": "node scripts/verify-docs-pages.mjs", "test:std-ref": "node scripts/verify-std-reference.mjs", "test:prose": "node scripts/verify-prose.mjs", "test:try": "node scripts/verify-try.mjs", + "test:sitemap": "node scripts/verify-sitemap.mjs", "sync:tree-sitter": "node scripts/sync-tree-sitter.mjs", "postinstall": "npm run sync:tree-sitter" }, diff --git a/www/public/_headers b/www/public/_headers index f2bb5c8f..21d9a9ed 100644 --- a/www/public/_headers +++ b/www/public/_headers @@ -5,3 +5,9 @@ /echo-wasm/*.wasm Content-Type: application/wasm Cache-Control: public, max-age=0, must-revalidate + +/sitemap.xml + Content-Type: application/xml; charset=utf-8 + +/robots.txt + Content-Type: text/plain; charset=utf-8 diff --git a/www/public/robots.txt b/www/public/robots.txt new file mode 100644 index 00000000..96b5b483 --- /dev/null +++ b/www/public/robots.txt @@ -0,0 +1,4 @@ +User-agent: * +Allow: / + +Sitemap: https://xo.run/sitemap.xml diff --git a/www/scripts/verify-sitemap.mjs b/www/scripts/verify-sitemap.mjs new file mode 100644 index 00000000..87cd106a --- /dev/null +++ b/www/scripts/verify-sitemap.mjs @@ -0,0 +1,150 @@ +/** + * Verifies public discovery files for xo.run: + * - sitemap.xml lists the public catalog on https://xo.run + * - robots.txt has User-agent rules and a Sitemap: line + * - Privacy and Terms are listed only when those pages exist + * - github.io is never listed + * + * Loads src/docs/site.ts, content.ts, and static-html.ts through Vite SSR. + */ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { createServer } from "vite"; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + +const server = await createServer({ + root, + logLevel: "error", + server: { middlewareMode: true }, + appType: "custom", +}); + +try { + const site = await server.ssrLoadModule("/src/docs/site.ts"); + const content = await server.ssrLoadModule("/src/docs/content.ts"); + const staticHtml = await server.ssrLoadModule("/src/docs/static-html.ts"); + + const { + collectPublicCatalogPaths, + omittedCatalogPaths, + publicCatalogUrl, + publicChromePaths, + publicSiteOrigin, + publicSurfacePaths, + renderRobotsTxt, + renderSitemapXml, + } = site; + const { docsPages } = content; + const { staticPages } = staticHtml; + + const failures = []; + + function fail(message) { + failures.push(message); + } + + if (publicSiteOrigin !== "https://xo.run") { + fail(`publicSiteOrigin must be https://xo.run, got ${publicSiteOrigin}`); + } + + const existingPagePaths = [ + ...new Set([...staticPages().map((page) => page.path), ...docsPages.map((page) => page.path)]), + ]; + const existing = new Set(existingPagePaths); + const catalogPaths = collectPublicCatalogPaths(existingPagePaths); + const sitemap = renderSitemapXml(catalogPaths); + const robots = renderRobotsTxt(); + const committedRobots = readFileSync(path.join(root, "public/robots.txt"), "utf8"); + + if (!sitemap.startsWith('')) { + fail("sitemap.xml must be a real XML document"); + } + if (!sitemap.includes('')) { + fail("sitemap.xml must use the sitemaps.org urlset namespace"); + } + + const requiredPaths = [ + ...publicSurfacePaths, + ...publicChromePaths(), + "/docs", + "/docs/std", + "/docs/leaders", + "/book", + "/e26", + "/e26/spec", + "/try", + "/install", + ]; + for (const required of requiredPaths) { + if (!catalogPaths.includes(required)) { + fail(`public catalog missing ${required}`); + } + const loc = publicCatalogUrl(required); + if (!sitemap.includes(`${loc}`)) { + fail(`sitemap.xml missing ${loc}`); + } + } + + for (const page of docsPages) { + if (!catalogPaths.includes(page.path)) { + fail(`docs page ${page.path} missing from the public catalog`); + } + } + + for (const candidate of omittedCatalogPaths) { + if (existing.has(candidate)) { + if (!catalogPaths.includes(candidate)) { + fail(`public catalog missing existing page ${candidate}`); + } + continue; + } + if (catalogPaths.includes(candidate)) { + fail(`public catalog must not invent ${candidate}`); + } + if (sitemap.toLowerCase().includes(candidate)) { + fail(`sitemap.xml must not list ${candidate} until that page exists`); + } + } + + if (/github\.io/i.test(sitemap) || /github\.io/i.test(robots)) { + fail("discovery files must not list the GitHub Pages host"); + } + if (!/https:\/\/xo\.run\//.test(sitemap) || /http:\/\/xo\.run/.test(sitemap)) { + fail("sitemap.xml must use https://xo.run as the live host"); + } + + if (!/^User-agent:\s+\*/m.test(robots)) { + fail("robots.txt must include a User-agent rule"); + } + if (!/^Allow:\s+\//m.test(robots)) { + fail("robots.txt must include an Allow rule"); + } + if (!/^Sitemap:\s+https:\/\/xo\.run\/sitemap\.xml$/m.test(robots)) { + fail("robots.txt must point Sitemap: at https://xo.run/sitemap.xml"); + } + if (committedRobots !== robots) { + fail("public/robots.txt must match renderRobotsTxt()"); + } + + if (failures.length) { + console.error(JSON.stringify({ ok: false, failures }, null, 2)); + process.exitCode = 1; + } else { + console.log( + JSON.stringify( + { + ok: true, + origin: publicSiteOrigin, + urls: catalogPaths.length, + sitemapBytes: sitemap.length, + }, + null, + 2, + ), + ); + } +} finally { + await server.close(); +} diff --git a/www/src/docs/site.ts b/www/src/docs/site.ts index 9858ecf1..2dc0340c 100644 --- a/www/src/docs/site.ts +++ b/www/src/docs/site.ts @@ -1,8 +1,21 @@ /** - * Public site chrome: homepage, primary nav, and the Documents hub catalog. - * Pages and tests load this module; do not duplicate the lists in UI or fixtures. + * Public site chrome: homepage, primary nav, Documents hub catalog, and the + * discovery files (`/sitemap.xml`, `/robots.txt`). Pages and tests load this + * module; do not duplicate the lists in UI or fixtures. */ +/** Live public host. Do not emit github.io URLs in discovery files. */ +export const publicSiteOrigin = "https://xo.run"; + +/** Top-level routes that are not docs pages. */ +export const publicSurfacePaths = ["/", "/install", "/try"] as const; + +/** + * Legal routes stay out of the sitemap until the pages exist. + * Do not add these paths here when inventing placeholder URLs. + */ +export const omittedCatalogPaths = ["/privacy", "/terms"] as const; + export type SiteNavItem = { label: string; to: string; @@ -279,6 +292,58 @@ export const docsHubCatalog: DocsCatalogGroup[] = [ }, ]; +/** + * Public catalog paths for `/sitemap.xml`. + * `existingPagePaths` is the shipped HTML catalog (`staticPages` / `docsPages`). + * Privacy and Terms are included only when those pages already exist. + */ +export function collectPublicCatalogPaths(existingPagePaths: readonly string[]): string[] { + const existing = new Set(existingPagePaths); + const paths = new Set([ + ...publicSurfacePaths, + ...publicChromePaths(), + ...existingPagePaths, + ]); + + for (const omitted of omittedCatalogPaths) { + if (!existing.has(omitted)) { + paths.delete(omitted); + } + } + + return [...paths].sort((left, right) => left.localeCompare(right)); +} + +export function publicCatalogUrl(path: string): string { + if (!path.startsWith("/")) { + throw new Error(`catalog path must be absolute, got ${path}`); + } + if (path === "/") { + return `${publicSiteOrigin}/`; + } + return `${publicSiteOrigin}${path}`; +} + +export function renderSitemapXml(paths: readonly string[]): string { + const urls = paths + .map((path) => ` \n ${escapeHtml(publicCatalogUrl(path))}\n `) + .join("\n"); + + return [ + ``, + ``, + urls, + ``, + ``, + ].join("\n"); +} + +export function renderRobotsTxt(): string { + return ["User-agent: *", "Allow: /", "", `Sitemap: ${publicSiteOrigin}/sitemap.xml`, ""].join( + "\n", + ); +} + function escapeHtml(text: string): string { return text .replaceAll("&", "&") diff --git a/www/vite.config.ts b/www/vite.config.ts index 0806831a..2002fa82 100644 --- a/www/vite.config.ts +++ b/www/vite.config.ts @@ -10,7 +10,12 @@ import { type DocsSearchAsset, type DocsSemanticAsset, } from "./src/docs/search"; -import { renderStaticHomeAndHub } from "./src/docs/site"; +import { + collectPublicCatalogPaths, + renderRobotsTxt, + renderSitemapXml, + renderStaticHomeAndHub, +} from "./src/docs/site"; import { distFileForPath, escapeHtml, staticPages, type StaticPage } from "./src/docs/static-html"; const docsSearchIndexDevFileName = "indices/search.json"; @@ -232,6 +237,47 @@ function applyStaticPage(html: string, page: StaticPage): string { }); } +function publicCatalogPaths() { + return collectPublicCatalogPaths(staticPages().map((page) => page.path)); +} + +function siteDiscoveryPlugin(): Plugin { + return { + name: "site-discovery", + configureServer(server) { + server.middlewares.use((request, response, next) => { + const requestPath = request.url?.split("?", 1)[0] ?? ""; + + if (requestPath === "/sitemap.xml") { + response.setHeader("Content-Type", "application/xml; charset=utf-8"); + response.end(renderSitemapXml(publicCatalogPaths())); + return; + } + + if (requestPath === "/robots.txt") { + response.setHeader("Content-Type", "text/plain; charset=utf-8"); + response.end(renderRobotsTxt()); + return; + } + + next(); + }); + }, + generateBundle() { + this.emitFile({ + type: "asset", + fileName: "sitemap.xml", + source: renderSitemapXml(publicCatalogPaths()), + }); + this.emitFile({ + type: "asset", + fileName: "robots.txt", + source: renderRobotsTxt(), + }); + }, + }; +} + function docsFirstStaticPlugin(): Plugin { const fallbackMarker = ''; @@ -310,5 +356,11 @@ export default defineConfig({ }, }, }, - plugins: [docsSearchIndexPlugin(), docsFirstStaticPlugin(), react(), tailwindcss()], + plugins: [ + siteDiscoveryPlugin(), + docsSearchIndexPlugin(), + docsFirstStaticPlugin(), + react(), + tailwindcss(), + ], });