Restore the web fetch public contract

docs/MCP.md

@@ -56,11 +56,12 @@ explicitly re-enable the preset or override it with a custom transport-bearing
 
 The preset overlaps with the native `web_search` and `web_fetch` tools. Native
 `web_search` remains lazy and unchanged. When the built-in Exa preset is active,
-the canonical `web_fetch` tool calls Exa MCP's `web_fetch_exa`; the raw
-`mcp__exa__web_fetch_exa` name is hidden to avoid duplicate fetch tools. If the
-built-in Exa connection fails or does not advertise `web_fetch_exa`, `web_fetch`
-returns an explicit Exa MCP error rather than falling back to direct fetch.
-Native direct `web_fetch` is available only when the built-in Exa preset is
+the canonical `web_fetch` tool uses Exa MCP's markdown-returning `web_fetch_exa`
+for markdown requests and the in-process fetcher for text or HTML requests. The
+raw `mcp__exa__web_fetch_exa` name is hidden to avoid duplicate fetch tools. If
+the built-in Exa connection fails or does not advertise `web_fetch_exa`, a
+markdown request returns an explicit Exa MCP error rather than falling back to
+direct fetch. Native direct `web_fetch` is available only when the built-in Exa preset is
 disabled or overridden by a custom `exa` MCP server. Other Exa MCP tools, such as
 `mcp__exa__web_search_exa`, remain exposed through MCP namespacing.
 

src/agent/exa-web-fetch-alias.test.ts

@@ -134,15 +134,15 @@ describe("built-in Exa web_fetch alias", () => {
     }
   });
 
-  test("canonical web_fetch maps native args to Exa MCP fetch shape", async () => {
+  test("canonical web_fetch preserves markdown while mapping to Exa MCP fetch shape", async () => {
     const toolset = await makeToolset();
     try {
       const controller = new AbortController();
       const connecting = connect(toolset);
       const result = await runTool(
         toolset,
         "web_fetch",
-        { url: "https://example.com", format: "html", timeout: 12 },
+        { url: "https://example.com", format: "markdown", timeout: 12 },
         controller.signal,
       );
       await connecting;
@@ -162,6 +162,20 @@ describe("built-in Exa web_fetch alias", () => {
     }
   });
 
+  test("canonical web_fetch rejects non-http URLs before invoking Exa MCP", async () => {
+    const toolset = await makeToolset();
+    try {
+      await connect(toolset);
+      const result = await runTool(toolset, "web_fetch", { url: "ftp://example.com/file" });
+
+      expect(result.isError).toBe(true);
+      expect(result.content).toContain("http or https");
+      expect(calls).toHaveLength(0);
+    } finally {
+      await toolset.dispose();
+    }
+  });
+
   test("canonical web_fetch returns explicit Exa MCP errors without native fallback", async () => {
     connectMode = "missing-fetch";
     const toolset = await makeToolset();

src/tools/web-fetch.test.ts

@@ -1,6 +1,7 @@
 import { afterEach, beforeEach, describe, expect, test } from "bun:test";
 import { createServer, type Server } from "node:http";
-import { runWebFetch, MAX_FETCH_BYTES } from "./web-fetch.js";
+import type { MCPClient } from "../mcp/client.js";
+import { createExaMCPWebFetchTool, runWebFetch, MAX_FETCH_BYTES } from "./web-fetch.js";
 
 let server: Server;
 let baseUrl: string;
@@ -117,3 +118,74 @@ describe("runWebFetch", () => {
     expect(outcome.ok).toBe(false);
   });
 });
+
+describe("createExaMCPWebFetchTool", () => {
+  function createTool(call: MCPClient["call"]) {
+    return createExaMCPWebFetchTool({
+      connect: async () => ({
+        ok: true,
+        client: {
+          serverName: "exa",
+          tools: [{ name: "web_fetch_exa", description: "Fetch", inputSchema: {} }],
+          call,
+          close: async () => undefined,
+        },
+      }),
+    });
+  }
+
+  test("honors the per-call timeout with the native timeout error contract", async () => {
+    const tool = createTool(
+      async (_name, _args, signal) =>
+        new Promise<string>((_resolve, reject) => {
+          signal.addEventListener("abort", () => reject(signal.reason), { once: true });
+        }),
+    );
+    const result = await tool.handler(
+      {
+        id: "timeout-call",
+        name: "web_fetch",
+        arguments: { url: "https://example.com", timeout: 1 },
+      },
+      new AbortController().signal,
+    );
+
+    expect(result).toEqual({
+      callId: "timeout-call",
+      content:
+        "Error: Request to https://example.com timed out after 1s. Retry with a larger timeout parameter (up to 120s) if the site is slow.",
+      isError: true,
+    });
+  });
+
+  test("returns distinct markdown, text, and html representations", async () => {
+    handler = (_req, res) => {
+      res.writeHead(200, { "content-type": "text/html" });
+      res.end("<html><body><h1>Heading</h1><p>Body</p></body></html>");
+    };
+    const tool = createTool(async () => "# Heading\n\nBody");
+    const invoke = async (format: "markdown" | "text" | "html") => {
+      const result = await tool.handler(
+        {
+          id: `format-${format}`,
+          name: "web_fetch",
+          arguments: { url: `${baseUrl}/`, format },
+        },
+        new AbortController().signal,
+      );
+      if (typeof result === "string") throw new Error("expected a full tool result");
+      return result;
+    };
+
+    const [markdown, text, html] = await Promise.all([
+      invoke("markdown"),
+      invoke("text"),
+      invoke("html"),
+    ]);
+    expect(markdown.content).toBe("# Heading\n\nBody");
+    expect(text.content).toContain("Heading");
+    expect(text.content).not.toContain("# Heading");
+    expect(text.content).not.toContain("<h1>");
+    expect(html.content).toContain("<h1>Heading</h1>");
+  });
+});

src/tools/web-fetch.ts

@@ -27,6 +27,8 @@ const WebFetchArgs = type({
   "timeout?": "number",
 });
 
+type WebFetchFormat = "text" | "markdown" | "html";
+
 export const webFetchDefinition: ToolDefinition = {
   name: "web_fetch",
   description:
@@ -49,7 +51,7 @@ export const webFetchDefinition: ToolDefinition = {
   },
 };
 
-function acceptHeaderFor(format: "text" | "markdown" | "html"): string {
+function acceptHeaderFor(format: WebFetchFormat): string {
   if (format === "html") return "text/html,application/xhtml+xml";
   return "text/html,application/xhtml+xml;q=0.9,text/plain;q=0.8,*/*;q=0.5";
 }
@@ -102,7 +104,7 @@ async function readCapped(
 async function fetchOnce(
   url: string,
   userAgent: string,
-  format: "text" | "markdown" | "html",
+  format: WebFetchFormat,
   timeoutMs: number,
 ): Promise<Response> {
   const controller = new AbortController();
@@ -126,7 +128,7 @@ export type WebFetchOutcome =
 
 export async function runWebFetch(
   rawUrl: string,
-  format: "text" | "markdown" | "html",
+  format: WebFetchFormat,
   timeoutSeconds: number,
 ): Promise<WebFetchOutcome> {
   const timeoutMs = Math.min(Math.max(timeoutSeconds, 1), MAX_TIMEOUT_S) * 1000;
@@ -235,35 +237,92 @@ export function createExaMCPWebFetchTool(args: {
           isError: true,
         };
       }
-      const connection = await args.connect(signal);
-      if (!connection.ok) {
+
+      let url: URL;
+      try {
+        url = new URL(parsed.url);
+      } catch {
         return {
           callId: call.id,
-          content: `Error: Exa MCP web_fetch unavailable: ${connection.error}`,
+          content: "Error: web_fetch URL must use http or https.",
           isError: true,
         };
       }
-      if (!connection.client.tools.some((tool) => tool.name === "web_fetch_exa")) {
+      if (url.protocol !== "http:" && url.protocol !== "https:") {
         return {
           callId: call.id,
-          content:
-            "Error: Exa MCP web_fetch unavailable: connected Exa server did not advertise web_fetch_exa.",
+          content: "Error: web_fetch URL must use http or https.",
           isError: true,
         };
       }
+
+      const format = parsed.format ?? "markdown";
+      const timeout = parsed.timeout ?? DEFAULT_TIMEOUT_S;
+      if (format !== "markdown") {
+        const outcome = await runWebFetch(parsed.url, format, timeout);
+        if (!outcome.ok) {
+          return { callId: call.id, content: `Error: ${outcome.error}`, isError: true };
+        }
+        const suffix = outcome.truncated
+          ? `\n\n[content truncated at ${MAX_FETCH_BYTES} bytes]`
+          : "";
+        return { callId: call.id, content: outcome.content + suffix };
+      }
+
+      const timeoutSeconds = Math.min(Math.max(timeout, 1), MAX_TIMEOUT_S);
+      const timeoutController = new AbortController();
+      const timeoutError = new Error("web_fetch timed out");
+      let timer: ReturnType<typeof setTimeout> | undefined;
+      const timeoutPromise = new Promise<never>((_resolve, reject) => {
+        timer = setTimeout(() => {
+          timeoutController.abort();
+          reject(timeoutError);
+        }, timeoutSeconds * 1000);
+      });
+      const callSignal = AbortSignal.any([signal, timeoutController.signal]);
       try {
-        const content = await connection.client.call(
-          "web_fetch_exa",
-          { urls: [parsed.url] },
-          signal,
-        );
-        return { callId: call.id, content };
+        return await Promise.race([
+          (async (): Promise<ToolResult> => {
+            const connection = await args.connect(callSignal);
+            if (!connection.ok) {
+              return {
+                callId: call.id,
+                content: `Error: Exa MCP web_fetch unavailable: ${connection.error}`,
+                isError: true,
+              };
+            }
+            if (!connection.client.tools.some((tool) => tool.name === "web_fetch_exa")) {
+              return {
+                callId: call.id,
+                content:
+                  "Error: Exa MCP web_fetch unavailable: connected Exa server did not advertise web_fetch_exa.",
+                isError: true,
+              };
+            }
+            const content = await connection.client.call(
+              "web_fetch_exa",
+              { urls: [parsed.url] },
+              callSignal,
+            );
+            return { callId: call.id, content };
+          })(),
+          timeoutPromise,
+        ]);
       } catch (err) {
+        if (err === timeoutError) {
+          return {
+            callId: call.id,
+            content: `Error: Request to ${parsed.url} timed out after ${timeoutSeconds}s. Retry with a larger timeout parameter (up to 120s) if the site is slow.`,
+            isError: true,
+          };
+        }
         return {
           callId: call.id,
           content: `Error: Exa MCP web_fetch failed: ${err instanceof Error ? err.message : String(err)}`,
           isError: true,
         };
+      } finally {
+        if (timer !== undefined) clearTimeout(timer);
       }
     },
   };