diff --git a/docs/MCP.md b/docs/MCP.md index ac2f4bb63..83a770803 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -56,11 +56,12 @@ explicitly re-enable the preset or override it with a custom transport-bearing The preset overlaps with the native `web_search` and `web_fetch` tools. Native `web_search` remains lazy and unchanged. When the built-in Exa preset is active, -the canonical `web_fetch` tool calls Exa MCP's `web_fetch_exa`; the raw -`mcp__exa__web_fetch_exa` name is hidden to avoid duplicate fetch tools. If the -built-in Exa connection fails or does not advertise `web_fetch_exa`, `web_fetch` -returns an explicit Exa MCP error rather than falling back to direct fetch. -Native direct `web_fetch` is available only when the built-in Exa preset is +the canonical `web_fetch` tool uses Exa MCP's markdown-returning `web_fetch_exa` +for markdown requests and the in-process fetcher for text or HTML requests. The +raw `mcp__exa__web_fetch_exa` name is hidden to avoid duplicate fetch tools. If +the built-in Exa connection fails or does not advertise `web_fetch_exa`, a +markdown request returns an explicit Exa MCP error rather than falling back to +direct fetch. Native direct `web_fetch` is available only when the built-in Exa preset is disabled or overridden by a custom `exa` MCP server. Other Exa MCP tools, such as `mcp__exa__web_search_exa`, remain exposed through MCP namespacing. diff --git a/src/agent/exa-web-fetch-alias.test.ts b/src/agent/exa-web-fetch-alias.test.ts index 3ce24e45f..914777756 100644 --- a/src/agent/exa-web-fetch-alias.test.ts +++ b/src/agent/exa-web-fetch-alias.test.ts @@ -134,7 +134,7 @@ describe("built-in Exa web_fetch alias", () => { } }); - test("canonical web_fetch maps native args to Exa MCP fetch shape", async () => { + test("canonical web_fetch preserves markdown while mapping to Exa MCP fetch shape", async () => { const toolset = await makeToolset(); try { const controller = new AbortController(); @@ -142,7 +142,7 @@ describe("built-in Exa web_fetch alias", () => { const result = await runTool( toolset, "web_fetch", - { url: "https://example.com", format: "html", timeout: 12 }, + { url: "https://example.com", format: "markdown", timeout: 12 }, controller.signal, ); await connecting; @@ -162,6 +162,20 @@ describe("built-in Exa web_fetch alias", () => { } }); + test("canonical web_fetch rejects non-http URLs before invoking Exa MCP", async () => { + const toolset = await makeToolset(); + try { + await connect(toolset); + const result = await runTool(toolset, "web_fetch", { url: "ftp://example.com/file" }); + + expect(result.isError).toBe(true); + expect(result.content).toContain("http or https"); + expect(calls).toHaveLength(0); + } finally { + await toolset.dispose(); + } + }); + test("canonical web_fetch returns explicit Exa MCP errors without native fallback", async () => { connectMode = "missing-fetch"; const toolset = await makeToolset(); diff --git a/src/tools/web-fetch.test.ts b/src/tools/web-fetch.test.ts index 93f6076e4..0339424e5 100644 --- a/src/tools/web-fetch.test.ts +++ b/src/tools/web-fetch.test.ts @@ -1,6 +1,7 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; import { createServer, type Server } from "node:http"; -import { runWebFetch, MAX_FETCH_BYTES } from "./web-fetch.js"; +import type { MCPClient } from "../mcp/client.js"; +import { createExaMCPWebFetchTool, runWebFetch, MAX_FETCH_BYTES } from "./web-fetch.js"; let server: Server; let baseUrl: string; @@ -117,3 +118,74 @@ describe("runWebFetch", () => { expect(outcome.ok).toBe(false); }); }); + +describe("createExaMCPWebFetchTool", () => { + function createTool(call: MCPClient["call"]) { + return createExaMCPWebFetchTool({ + connect: async () => ({ + ok: true, + client: { + serverName: "exa", + tools: [{ name: "web_fetch_exa", description: "Fetch", inputSchema: {} }], + call, + close: async () => undefined, + }, + }), + }); + } + + test("honors the per-call timeout with the native timeout error contract", async () => { + const tool = createTool( + async (_name, _args, signal) => + new Promise((_resolve, reject) => { + signal.addEventListener("abort", () => reject(signal.reason), { once: true }); + }), + ); + const result = await tool.handler( + { + id: "timeout-call", + name: "web_fetch", + arguments: { url: "https://example.com", timeout: 1 }, + }, + new AbortController().signal, + ); + + expect(result).toEqual({ + callId: "timeout-call", + content: + "Error: Request to https://example.com timed out after 1s. Retry with a larger timeout parameter (up to 120s) if the site is slow.", + isError: true, + }); + }); + + test("returns distinct markdown, text, and html representations", async () => { + handler = (_req, res) => { + res.writeHead(200, { "content-type": "text/html" }); + res.end("

Heading

Body

"); + }; + const tool = createTool(async () => "# Heading\n\nBody"); + const invoke = async (format: "markdown" | "text" | "html") => { + const result = await tool.handler( + { + id: `format-${format}`, + name: "web_fetch", + arguments: { url: `${baseUrl}/`, format }, + }, + new AbortController().signal, + ); + if (typeof result === "string") throw new Error("expected a full tool result"); + return result; + }; + + const [markdown, text, html] = await Promise.all([ + invoke("markdown"), + invoke("text"), + invoke("html"), + ]); + expect(markdown.content).toBe("# Heading\n\nBody"); + expect(text.content).toContain("Heading"); + expect(text.content).not.toContain("# Heading"); + expect(text.content).not.toContain("

"); + expect(html.content).toContain("

Heading

"); + }); +}); diff --git a/src/tools/web-fetch.ts b/src/tools/web-fetch.ts index 5a68e2937..23a66fb21 100644 --- a/src/tools/web-fetch.ts +++ b/src/tools/web-fetch.ts @@ -27,6 +27,8 @@ const WebFetchArgs = type({ "timeout?": "number", }); +type WebFetchFormat = "text" | "markdown" | "html"; + export const webFetchDefinition: ToolDefinition = { name: "web_fetch", description: @@ -49,7 +51,7 @@ export const webFetchDefinition: ToolDefinition = { }, }; -function acceptHeaderFor(format: "text" | "markdown" | "html"): string { +function acceptHeaderFor(format: WebFetchFormat): string { if (format === "html") return "text/html,application/xhtml+xml"; return "text/html,application/xhtml+xml;q=0.9,text/plain;q=0.8,*/*;q=0.5"; } @@ -102,7 +104,7 @@ async function readCapped( async function fetchOnce( url: string, userAgent: string, - format: "text" | "markdown" | "html", + format: WebFetchFormat, timeoutMs: number, ): Promise { const controller = new AbortController(); @@ -126,7 +128,7 @@ export type WebFetchOutcome = export async function runWebFetch( rawUrl: string, - format: "text" | "markdown" | "html", + format: WebFetchFormat, timeoutSeconds: number, ): Promise { const timeoutMs = Math.min(Math.max(timeoutSeconds, 1), MAX_TIMEOUT_S) * 1000; @@ -235,35 +237,92 @@ export function createExaMCPWebFetchTool(args: { isError: true, }; } - const connection = await args.connect(signal); - if (!connection.ok) { + + let url: URL; + try { + url = new URL(parsed.url); + } catch { return { callId: call.id, - content: `Error: Exa MCP web_fetch unavailable: ${connection.error}`, + content: "Error: web_fetch URL must use http or https.", isError: true, }; } - if (!connection.client.tools.some((tool) => tool.name === "web_fetch_exa")) { + if (url.protocol !== "http:" && url.protocol !== "https:") { return { callId: call.id, - content: - "Error: Exa MCP web_fetch unavailable: connected Exa server did not advertise web_fetch_exa.", + content: "Error: web_fetch URL must use http or https.", isError: true, }; } + + const format = parsed.format ?? "markdown"; + const timeout = parsed.timeout ?? DEFAULT_TIMEOUT_S; + if (format !== "markdown") { + const outcome = await runWebFetch(parsed.url, format, timeout); + if (!outcome.ok) { + return { callId: call.id, content: `Error: ${outcome.error}`, isError: true }; + } + const suffix = outcome.truncated + ? `\n\n[content truncated at ${MAX_FETCH_BYTES} bytes]` + : ""; + return { callId: call.id, content: outcome.content + suffix }; + } + + const timeoutSeconds = Math.min(Math.max(timeout, 1), MAX_TIMEOUT_S); + const timeoutController = new AbortController(); + const timeoutError = new Error("web_fetch timed out"); + let timer: ReturnType | undefined; + const timeoutPromise = new Promise((_resolve, reject) => { + timer = setTimeout(() => { + timeoutController.abort(); + reject(timeoutError); + }, timeoutSeconds * 1000); + }); + const callSignal = AbortSignal.any([signal, timeoutController.signal]); try { - const content = await connection.client.call( - "web_fetch_exa", - { urls: [parsed.url] }, - signal, - ); - return { callId: call.id, content }; + return await Promise.race([ + (async (): Promise => { + const connection = await args.connect(callSignal); + if (!connection.ok) { + return { + callId: call.id, + content: `Error: Exa MCP web_fetch unavailable: ${connection.error}`, + isError: true, + }; + } + if (!connection.client.tools.some((tool) => tool.name === "web_fetch_exa")) { + return { + callId: call.id, + content: + "Error: Exa MCP web_fetch unavailable: connected Exa server did not advertise web_fetch_exa.", + isError: true, + }; + } + const content = await connection.client.call( + "web_fetch_exa", + { urls: [parsed.url] }, + callSignal, + ); + return { callId: call.id, content }; + })(), + timeoutPromise, + ]); } catch (err) { + if (err === timeoutError) { + return { + callId: call.id, + content: `Error: Request to ${parsed.url} timed out after ${timeoutSeconds}s. Retry with a larger timeout parameter (up to 120s) if the site is slow.`, + isError: true, + }; + } return { callId: call.id, content: `Error: Exa MCP web_fetch failed: ${err instanceof Error ? err.message : String(err)}`, isError: true, }; + } finally { + if (timer !== undefined) clearTimeout(timer); } }, };