From 7e6c396413b7850baaeb1eda670c9d7eb9078a37 Mon Sep 17 00:00:00 2001 From: Jaideep Date: Sun, 23 Nov 2025 20:21:07 -0800 Subject: [PATCH 1/7] refactor: replace GenericOpenAIChatAgent with OpenAIChatAgent across the codebase --- examples/openai_compatible_agent.py | 4 ++-- hud/agents/__init__.py | 5 ++--- hud/agents/grounded_openai.py | 4 ++-- hud/agents/lite_llm.py | 6 +++--- hud/agents/{openai_chat_generic.py => openai_chat.py} | 6 +++--- hud/rl/actor.py | 6 +++--- hud/utils/agent_factories.py | 6 +++--- hud/utils/tests/test_agent_factories.py | 4 ++-- 8 files changed, 20 insertions(+), 21 deletions(-) rename hud/agents/{openai_chat_generic.py => openai_chat.py} (99%) diff --git a/examples/openai_compatible_agent.py b/examples/openai_compatible_agent.py index 659f8fbc7..420f78d13 100644 --- a/examples/openai_compatible_agent.py +++ b/examples/openai_compatible_agent.py @@ -23,7 +23,7 @@ from openai import AsyncOpenAI import hud -from hud.agents.openai_chat_generic import GenericOpenAIChatAgent +from hud.agents.openai_chat import OpenAIChatAgent from hud.datasets import Task @@ -127,7 +127,7 @@ async def run_example(mode: Literal["text", "browser"], target: int) -> None: allowed_tools = ["computer"] if mode == "browser" else ["move"] # Create OpenAI-compatible agent - agent = GenericOpenAIChatAgent( + agent = OpenAIChatAgent( openai_client=openai_client, model_name=model_name, allowed_tools=allowed_tools, diff --git a/hud/agents/__init__.py b/hud/agents/__init__.py index f69d4ed53..399ad16d8 100644 --- a/hud/agents/__init__.py +++ b/hud/agents/__init__.py @@ -3,13 +3,12 @@ from .base import MCPAgent from .claude import ClaudeAgent from .gemini import GeminiAgent -from .openai import OperatorAgent -from .openai_chat_generic import GenericOpenAIChatAgent +from .openai_chat import OpenAIChatAgent __all__ = [ "ClaudeAgent", "GeminiAgent", - "GenericOpenAIChatAgent", + "OpenAIChatAgent", "MCPAgent", "OperatorAgent", ] diff --git a/hud/agents/grounded_openai.py b/hud/agents/grounded_openai.py index 4ad2a0afe..63ad94e8a 100644 --- a/hud/agents/grounded_openai.py +++ b/hud/agents/grounded_openai.py @@ -9,10 +9,10 @@ from hud.tools.grounding import GroundedComputerTool, Grounder, GrounderConfig from hud.types import AgentResponse, MCPToolCall, MCPToolResult -from .openai_chat_generic import GenericOpenAIChatAgent +from .openai_chat import OpenAIChatAgent -class GroundedOpenAIChatAgent(GenericOpenAIChatAgent): +class GroundedOpenAIChatAgent(OpenAIChatAgent): """OpenAI agent that uses a separate grounding model for element detection. This agent: diff --git a/hud/agents/lite_llm.py b/hud/agents/lite_llm.py index 4445a6346..c16081737 100644 --- a/hud/agents/lite_llm.py +++ b/hud/agents/lite_llm.py @@ -11,7 +11,7 @@ import litellm -from .openai_chat_generic import GenericOpenAIChatAgent +from .openai_chat import OpenAIChatAgent logger = logging.getLogger(__name__) @@ -24,7 +24,7 @@ transform_mcp_tool_to_openai_tool = None # type: ignore -class LiteAgent(GenericOpenAIChatAgent): +class LiteAgent(OpenAIChatAgent): """ Same OpenAI chat-completions shape + MCP tool plumbing, but transport is LiteLLM and (optionally) tools are shaped by LiteLLM's MCP transformer. @@ -55,7 +55,7 @@ def get_tool_schemas(self) -> list[Any]: for t in self.get_available_tools() ] # Fallback to the generic OpenAI sanitizer - return GenericOpenAIChatAgent.get_tool_schemas(self) + return OpenAIChatAgent.get_tool_schemas(self) async def _invoke_chat_completion( self, diff --git a/hud/agents/openai_chat_generic.py b/hud/agents/openai_chat.py similarity index 99% rename from hud/agents/openai_chat_generic.py rename to hud/agents/openai_chat.py index 934886a53..2d0137386 100644 --- a/hud/agents/openai_chat_generic.py +++ b/hud/agents/openai_chat.py @@ -1,4 +1,4 @@ -"""Generic OpenAI chat-completions agent. +"""OpenAI Chat Completions Agent. This class provides the minimal glue required to connect any endpoint that implements the OpenAI compatible *chat.completions* API with MCP tool calling @@ -34,7 +34,7 @@ logger = logging.getLogger(__name__) -class GenericOpenAIChatAgent(MCPAgent): +class OpenAIChatAgent(MCPAgent): """MCP-enabled agent that speaks the OpenAI *chat.completions* protocol.""" metadata: ClassVar[dict[str, Any]] = {} @@ -62,7 +62,7 @@ def __init__( else: raise ValueError("Either openai_client or (api_key and base_url) must be provided") - self.model_name = "GenericOpenAI" + self.model_name = "OpenAI" self.checkpoint_name = model_name self.completion_kwargs: dict[str, Any] = completion_kwargs or {} self.mcp_schemas = [] diff --git a/hud/rl/actor.py b/hud/rl/actor.py index 9c073b5d0..4c9a3390c 100644 --- a/hud/rl/actor.py +++ b/hud/rl/actor.py @@ -9,7 +9,7 @@ from openai import AsyncOpenAI import hud -from hud.agents.openai_chat_generic import GenericOpenAIChatAgent +from hud.agents.openai_chat import OpenAIChatAgent from hud.clients.utils.retry_transport import create_retry_httpx_client from hud.types import Task, Trace from hud.utils.hud_console import HUDConsole @@ -46,9 +46,9 @@ def _create_openai_client(self, base_url: str) -> AsyncOpenAI: max_retries=2, ) - def create_agent(self) -> GenericOpenAIChatAgent: + def create_agent(self) -> OpenAIChatAgent: """Create an agent with the current adapter.""" - return GenericOpenAIChatAgent( + return OpenAIChatAgent( openai_client=self.openai_client, model_name=self.current_adapter, allowed_tools=self.actor_config.allowed_tools, diff --git a/hud/utils/agent_factories.py b/hud/utils/agent_factories.py index e15cb2404..a12a8a45b 100644 --- a/hud/utils/agent_factories.py +++ b/hud/utils/agent_factories.py @@ -7,11 +7,11 @@ from openai import AsyncOpenAI from hud.agents.grounded_openai import GroundedOpenAIChatAgent -from hud.agents.openai_chat_generic import GenericOpenAIChatAgent +from hud.agents.openai_chat import OpenAIChatAgent from hud.tools.grounding import GrounderConfig -def create_openai_agent(**kwargs: Any) -> GenericOpenAIChatAgent: +def create_openai_agent(**kwargs: Any) -> OpenAIChatAgent: """Factory for GenericOpenAIChatAgent with run_dataset compatibility. Args: @@ -36,7 +36,7 @@ def create_openai_agent(**kwargs: Any) -> GenericOpenAIChatAgent: api_key = kwargs.pop("api_key", None) base_url = kwargs.pop("base_url", None) - return GenericOpenAIChatAgent(api_key=api_key, base_url=base_url, **kwargs) + return OpenAIChatAgent(api_key=api_key, base_url=base_url, **kwargs) def create_grounded_agent(**kwargs: Any) -> GroundedOpenAIChatAgent: diff --git a/hud/utils/tests/test_agent_factories.py b/hud/utils/tests/test_agent_factories.py index 2064c4172..f9b10d61c 100644 --- a/hud/utils/tests/test_agent_factories.py +++ b/hud/utils/tests/test_agent_factories.py @@ -4,13 +4,13 @@ def test_create_openai_agent(): - from hud.agents.openai_chat_generic import GenericOpenAIChatAgent + from hud.agents.openai_chat import OpenAIChatAgent from hud.utils.agent_factories import create_openai_agent agent = create_openai_agent( api_key="test_key", model_name="test_model", completion_kwargs={"temperature": 0.5} ) - assert isinstance(agent, GenericOpenAIChatAgent) + assert isinstance(agent, OpenAIChatAgent) assert agent.model_name == "GenericOpenAI" assert agent.checkpoint_name == "test_model" assert agent.completion_kwargs["temperature"] == 0.5 From 512d3abc5c101faca6856011d0ad24f0dd31fcca Mon Sep 17 00:00:00 2001 From: Jaideep Date: Sun, 23 Nov 2025 20:41:15 -0800 Subject: [PATCH 2/7] docs for openai chat agent refactor --- docs/reference/agents.mdx | 6 +++--- hud/agents/grounded_openai.py | 2 +- hud/agents/openai_chat.py | 2 +- hud/utils/agent_factories.py | 6 +++--- hud/utils/tests/test_agent_factories.py | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/reference/agents.mdx b/docs/reference/agents.mdx index 58a97d835..f6d8f4f24 100644 --- a/docs/reference/agents.mdx +++ b/docs/reference/agents.mdx @@ -147,10 +147,10 @@ OpenAI Operator-style agent built on the Responses API with computer-use. - Operator-style system prompt guidance - Display metadata injection -### GenericOpenAIChatAgent +### OpenAIChatAgent ```python -from hud.agents import GenericOpenAIChatAgent +from hud.agents import OpenAIChatAgent ``` OpenAI-compatible chat.completions agent that works with any endpoint implementing the OpenAI schema (OpenAI, vLLM, Ollama, Together, custom, etc.). @@ -171,7 +171,7 @@ openai_client = AsyncOpenAI( api_key="not-needed", ) -agent = GenericOpenAIChatAgent( +agent = OpenAIChatAgent( openai_client=openai_client, model_name="llama3.1", completion_kwargs={"temperature": 0.2}, diff --git a/hud/agents/grounded_openai.py b/hud/agents/grounded_openai.py index 63ad94e8a..d7c91ee38 100644 --- a/hud/agents/grounded_openai.py +++ b/hud/agents/grounded_openai.py @@ -45,7 +45,7 @@ def __init__( openai_client: OpenAI client for the planning model model: Name of the OpenAI model to use for planning (e.g., "gpt-4o", "gpt-4o-mini") real_computer_tool_name: Name of the actual computer tool to execute - **kwargs: Additional arguments passed to GenericOpenAIChatAgent + **kwargs: Additional arguments passed to OpenAIChatAgent """ # Set defaults for grounded agent if allowed_tools is None: diff --git a/hud/agents/openai_chat.py b/hud/agents/openai_chat.py index 2d0137386..a3493a1d1 100644 --- a/hud/agents/openai_chat.py +++ b/hud/agents/openai_chat.py @@ -195,7 +195,7 @@ async def _invoke_chat_completion( extra: dict[str, Any], ) -> Any: if self.oai is None: - raise ValueError("openai_client is required for GenericOpenAIChatAgent") + raise ValueError("openai_client is required for OpenAIChatAgent") # default transport = OpenAI SDK return await self.oai.chat.completions.create( model=self.checkpoint_name, diff --git a/hud/utils/agent_factories.py b/hud/utils/agent_factories.py index a12a8a45b..ed636007b 100644 --- a/hud/utils/agent_factories.py +++ b/hud/utils/agent_factories.py @@ -12,16 +12,16 @@ def create_openai_agent(**kwargs: Any) -> OpenAIChatAgent: - """Factory for GenericOpenAIChatAgent with run_dataset compatibility. + """Factory for OpenAIChatAgent with run_dataset compatibility. Args: api_key: OpenAI API key base_url: Optional custom API endpoint model_name: Model to use (e.g., "gpt-4o-mini") - **kwargs: Additional arguments passed to GenericOpenAIChatAgent + **kwargs: Additional arguments passed to OpenAIChatAgent Returns: - Configured GenericOpenAIChatAgent instance + Configured OpenAIChatAgent instance Example: >>> from hud.datasets import run_dataset diff --git a/hud/utils/tests/test_agent_factories.py b/hud/utils/tests/test_agent_factories.py index f9b10d61c..111388fef 100644 --- a/hud/utils/tests/test_agent_factories.py +++ b/hud/utils/tests/test_agent_factories.py @@ -11,7 +11,7 @@ def test_create_openai_agent(): api_key="test_key", model_name="test_model", completion_kwargs={"temperature": 0.5} ) assert isinstance(agent, OpenAIChatAgent) - assert agent.model_name == "GenericOpenAI" + assert agent.model_name == "OpenAI" assert agent.checkpoint_name == "test_model" assert agent.completion_kwargs["temperature"] == 0.5 From 8b722cb42511c7bd496a2901cd61ef010aa72475 Mon Sep 17 00:00:00 2001 From: Jaideep Date: Sun, 23 Nov 2025 21:02:30 -0800 Subject: [PATCH 3/7] introduce OpenAIAgent based on responses api --- examples/run_evaluation.py | 16 +- hud/agents/__init__.py | 10 +- hud/agents/openai.py | 517 +++++++++--------- hud/agents/operator.py | 250 +++++++++ .../{test_openai.py => test_operator.py} | 2 +- hud/cli/__init__.py | 3 +- hud/cli/eval.py | 80 ++- hud/types.py | 1 + hud/utils/strict_schema.py | 168 ++++++ 9 files changed, 743 insertions(+), 304 deletions(-) create mode 100644 hud/agents/operator.py rename hud/agents/tests/{test_openai.py => test_operator.py} (99%) create mode 100644 hud/utils/strict_schema.py diff --git a/examples/run_evaluation.py b/examples/run_evaluation.py index fd58801da..eb5de4c60 100644 --- a/examples/run_evaluation.py +++ b/examples/run_evaluation.py @@ -59,13 +59,13 @@ def _build_agent( - agent_type: Literal[AgentType.CLAUDE, AgentType.OPENAI], + agent_type: Literal[AgentType.CLAUDE, AgentType.OPERATOR], *, model: str | None = None, allowed_tools: list[str] | None = None, ) -> ClaudeAgent | OperatorAgent: """Create and return the requested agent type.""" - if agent_type == AgentType.OPENAI: + if agent_type == AgentType.OPERATOR: return OperatorAgent(allowed_tools=allowed_tools, validate_api_key=False) model = model or "claude-sonnet-4-5-20250929" @@ -80,7 +80,7 @@ def _build_agent( async def run_single_task( dataset_name: str, *, - agent_type: Literal[AgentType.CLAUDE, AgentType.OPENAI] = AgentType.CLAUDE, + agent_type: Literal[AgentType.CLAUDE, AgentType.OPERATOR] = AgentType.CLAUDE, model: str | None = None, allowed_tools: list[str] | None = None, max_steps: int = 10, @@ -111,16 +111,18 @@ async def run_single_task( async def run_full_dataset( dataset_name: str, *, - agent_type: Literal[AgentType.CLAUDE, AgentType.OPENAI] = AgentType.CLAUDE, + agent_type: Literal[AgentType.CLAUDE, AgentType.OPERATOR] = AgentType.CLAUDE, model: str | None = None, allowed_tools: list[str] | None = None, max_concurrent: int = 50, max_steps: int = 10, ) -> list[Any]: """Run evaluation across entire dataset with asyncio concurrency.""" - if agent_type == AgentType.OPENAI: + if agent_type == AgentType.OPERATOR: agent_class = OperatorAgent - agent_config: dict[str, Any] = {"validate_api_key": False} + agent_config = {"validate_api_key": False} + if model: + agent_config["model"] = model if allowed_tools: # Only pass allowed tools if they are provided, otherwise all tools are enabled agent_config["allowed_tools"] = allowed_tools @@ -171,7 +173,7 @@ def parse_args() -> argparse.Namespace: # type: ignore[valid-type] parser.add_argument("--full", action="store_true", help="Run entire dataset") # Agent - parser.add_argument("--agent", choices=["claude", "openai"], default="claude") + parser.add_argument("--agent", choices=["claude", "operator"], default="claude") parser.add_argument("--model", default=None, help="Model override") parser.add_argument( "--allowed-tools", dest="allowed_tools", help="Tool allowlist (comma-separated)" diff --git a/hud/agents/__init__.py b/hud/agents/__init__.py index 399ad16d8..95ab9f343 100644 --- a/hud/agents/__init__.py +++ b/hud/agents/__init__.py @@ -3,12 +3,8 @@ from .base import MCPAgent from .claude import ClaudeAgent from .gemini import GeminiAgent +from .openai import OpenAIAgent +from .operator import OperatorAgent from .openai_chat import OpenAIChatAgent -__all__ = [ - "ClaudeAgent", - "GeminiAgent", - "OpenAIChatAgent", - "MCPAgent", - "OperatorAgent", -] +__all__ = ["ClaudeAgent", "GeminiAgent", "OpenAIAgent", "OpenAIChatAgent", "MCPAgent", "OperatorAgent"] \ No newline at end of file diff --git a/hud/agents/openai.py b/hud/agents/openai.py index 7d09c753e..ae7f81ff4 100644 --- a/hud/agents/openai.py +++ b/hud/agents/openai.py @@ -2,355 +2,328 @@ from __future__ import annotations +import copy +import json import logging -from typing import Any, ClassVar, Literal +from typing import Any, Literal, cast import mcp.types as types from openai import AsyncOpenAI, OpenAI from openai.types.responses import ( - ResponseComputerToolCall, + ResponseFunctionToolCall, + ResponseInputImageParam, ResponseInputMessageContentListParam, ResponseInputParam, + ResponseInputTextParam, ResponseOutputMessage, ResponseOutputText, + ResponseReasoningItem, ToolParam, ) +from openai.types.responses.response_input_param import ( + FunctionCallOutput, + Message, +) import hud from hud.settings import settings -from hud.tools.computer.settings import computer_settings from hud.types import AgentResponse, MCPToolCall, MCPToolResult, Trace +from hud.utils.strict_schema import ensure_strict_json_schema from .base import MCPAgent logger = logging.getLogger(__name__) -class OperatorAgent(MCPAgent): - """ - Operator agent that uses MCP servers for tool execution. - - This agent uses OpenAI's Computer Use API format but executes - tools through MCP servers instead of direct implementation. - """ +class OpenAIAgent(MCPAgent): + """Generic OpenAI agent that can execute MCP tools through the Responses API.""" - metadata: ClassVar[dict[str, Any]] = { - "display_width": computer_settings.OPENAI_COMPUTER_WIDTH, - "display_height": computer_settings.OPENAI_COMPUTER_HEIGHT, - } - required_tools: ClassVar[list[str]] = ["openai_computer"] + metadata: dict[str, Any] | None = None def __init__( self, model_client: AsyncOpenAI | None = None, - model: str = "computer-use-preview", - environment: Literal["windows", "mac", "linux", "browser"] = "linux", + model: str = "gpt-5.1", + max_output_tokens: int | None = None, + temperature: float | None = None, + reasoning: dict[str, Any] | Literal["auto"] | None = None, + tool_choice: dict[str, Any] | Literal["auto"] | None = None, + parallel_tool_calls: bool | None = None, validate_api_key: bool = True, **kwargs: Any, ) -> None: - """ - Initialize Operator MCP agent. - - Args: - client: AsyncOpenAI client (created if not provided) - model: OpenAI model to use - environment: Environment type for computer use - display_width: Display width for computer use - display_height: Display height for computer use - **kwargs: Additional arguments passed to MCPAgent - """ super().__init__(**kwargs) - # Initialize client if not provided if model_client is None: api_key = settings.openai_api_key if not api_key: raise ValueError("OpenAI API key not found. Set OPENAI_API_KEY.") model_client = AsyncOpenAI(api_key=api_key) + if validate_api_key: + try: + OpenAI(api_key=model_client.api_key).models.list() + except Exception as exc: # pragma: no cover - network validation + raise ValueError(f"OpenAI API key is invalid: {exc}") from exc + self.openai_client = model_client self.model = model - self.checkpoint_name = self.model - self.environment = environment + self.max_output_tokens = max_output_tokens + self.temperature = temperature + self.reasoning = reasoning + self.tool_choice = tool_choice + self.parallel_tool_calls = parallel_tool_calls + + self._openai_tools: list[ToolParam] = [] + self._tool_name_map: dict[str, str] = {} - # State tracking for OpenAI's stateful API self.last_response_id: str | None = None self.pending_call_id: str | None = None self.pending_safety_checks: list[Any] = [] + self._message_cursor = 0 - # validate api key if requested - if validate_api_key: + self.model_name = "OpenAI" + self.checkpoint_name = self.model + + async def initialize(self, task: Any | None = None) -> None: + """Initialize agent and build tool metadata.""" + await super().initialize(task) + self._build_openai_tools() + + def _build_openai_tools(self) -> None: + """Convert MCP tools into OpenAI Responses tool definitions.""" + self._openai_tools = [] + self._tool_name_map = {} + + for tool in self.get_available_tools(): + if tool.description is None or tool.inputSchema is None: + self.console.warning_log( + f"Skipping tool '{tool.name}' – description and input schema are required for OpenAI tools." + ) + continue + self._tool_name_map[tool.name] = tool.name + schema_copy = copy.deepcopy(tool.inputSchema) + strict_schema = schema_copy + strict_enforced = True try: - OpenAI(api_key=self.openai_client.api_key).models.list() - except Exception as e: - raise ValueError(f"OpenAI API key is invalid: {e}") from e - - self.model_name = "Operator" - - # Append OpenAI-specific instructions to the base system prompt - openai_instructions = """ - You are an autonomous computer-using agent. Follow these guidelines: - - 1. NEVER ask for confirmation. Complete all tasks autonomously. - 2. Do NOT send messages like "I need to confirm before..." or "Do you want me to continue?" - just proceed. - 3. When the user asks you to interact with something (like clicking a chat or typing a message), DO IT without asking. - 4. Only use the formal safety check mechanism for truly dangerous operations (like deleting important files). - 5. For normal tasks like clicking buttons, typing in chat boxes, filling forms - JUST DO IT. - 6. The user has already given you permission by running this agent. No further confirmation is needed. - 7. Be decisive and action-oriented. Complete the requested task fully. - - Remember: You are expected to complete tasks autonomously. The user trusts you to do what they asked. - """.strip() # noqa: E501 - - # Append OpenAI instructions to any base system prompt - if self.system_prompt: - self.system_prompt = f"{self.system_prompt}\n\n{openai_instructions}" - else: - self.system_prompt = openai_instructions - - async def _run_context(self, context: list[types.ContentBlock], max_steps: int = 10) -> Trace: - """ - Run the agent with the given prompt or task. - - Override to reset OpenAI-specific state. - """ - # Reset state for new run - self.last_response_id = None - self.pending_call_id = None - self.pending_safety_checks = [] + strict_schema = ensure_strict_json_schema(schema_copy) + except Exception as exc: # pragma: no cover - defensive + strict_enforced = False + self.console.warning_log( + f"Failed to convert schema for tool '{tool.name}' to strict mode: {exc}" + ) + + function_tool = cast( + ToolParam, + { + "type": "function", + "name": tool.name, + "description": tool.description, + "parameters": strict_schema, + "strict": strict_enforced, + }, + ) + self._openai_tools.append(function_tool) - # Use base implementation + async def _run_context( + self, context: list[types.ContentBlock], *, max_steps: int = 10 + ) -> Trace: + """Reset internal state before delegating to the base loop.""" + self._reset_response_state() return await super()._run_context(context, max_steps=max_steps) - async def get_system_messages(self) -> list[Any]: - """ - Create initial messages for OpenAI. + def _reset_response_state(self) -> None: + self.last_response_id = None + self.pending_call_id = None + self.pending_safety_checks = [] + self._message_cursor = 0 - OpenAI uses a different message format - we'll store the prompt - and screenshot for use in get_model_response. - """ + async def get_system_messages(self) -> list[types.ContentBlock]: + """System messages are provided via the `instructions` field.""" return [] async def format_blocks( self, blocks: list[types.ContentBlock] - ) -> ResponseInputMessageContentListParam: - """ - Format blocks for OpenAI input format. - - Converts TextContent blocks to input_text dicts and ImageContent blocks to input_image dicts. - """ # noqa: E501 - formatted = [] + ) -> ResponseInputParam: + """Convert MCP content blocks into OpenAI user messages.""" + content: ResponseInputMessageContentListParam = [] for block in blocks: if isinstance(block, types.TextContent): - formatted.append({"type": "input_text", "text": block.text}) + content.append( + cast( + ResponseInputTextParam, + {"type": "input_text", "text": block.text}, + ) + ) elif isinstance(block, types.ImageContent): mime_type = getattr(block, "mimeType", "image/png") - formatted.append( - {"type": "input_image", "image_url": f"data:{mime_type};base64,{block.data}"} + content.append( + cast( + ResponseInputImageParam, + { + "type": "input_image", + "image_url": f"data:{mime_type};base64,{block.data}", + }, + ) ) - return formatted + if not content: + content.append( + cast(ResponseInputTextParam, {"type": "input_text", "text": ""}) + ) + return [cast(Message, {"role": "user", "content": content})] @hud.instrument( span_type="agent", - record_args=False, # Messages can be large + record_args=False, record_result=True, ) - async def get_response(self, messages: ResponseInputMessageContentListParam) -> AgentResponse: - """Get response from OpenAI including any tool calls.""" - # OpenAI's API is stateful, so we handle messages differently - - # Get the computer tool (guaranteed to exist due to required_tools) - computer_tool_name = "openai_computer" - - # Define the computer use tool - computer_tool: ToolParam = { # type: ignore[reportAssignmentType] - "type": "computer_use_preview", - "display_width": self.metadata["display_width"], - "display_height": self.metadata["display_height"], - "environment": self.environment, - } - - # Build the request based on whether this is first step or follow-up - if self.pending_call_id is None and self.last_response_id is None: - # First step - messages are already formatted dicts from format_blocks - # format_blocks returns type ResponseInputMessageContentListParam, which is a list of dicts # noqa: E501 - input_content: ResponseInputMessageContentListParam = [] - - input_content.extend(messages) - - # If no content was added, add empty text to avoid empty request - if not input_content: - input_content.append({"type": "input_text", "text": ""}) - - input_param: ResponseInputParam = [{"role": "user", "content": input_content}] # type: ignore[reportUnknownMemberType] - - response = await self.openai_client.responses.create( - model=self.model, - tools=[computer_tool], - input=input_param, - instructions=self.system_prompt, - truncation="auto", - reasoning={"summary": "auto"}, # type: ignore[arg-type] - ) - else: - # Follow-up step - check if this is user input or tool result - latest_message = messages[-1] if messages else {} - - if latest_message.get("type") == "input_text": - # User provided input in conversation mode - user_text = latest_message.get("text", "") - input_param_followup: ResponseInputParam = [ # type: ignore[reportAssignmentType] - {"role": "user", "content": [{"type": "input_text", "text": user_text}]} - ] - # Reset pending_call_id since this is user input, not a tool response - self.pending_call_id = None + async def get_response(self, messages: ResponseInputParam) -> AgentResponse: + """Send the latest input items to OpenAI's Responses API.""" + new_items = cast(ResponseInputParam, messages[self._message_cursor :]) + if not new_items: + if self.last_response_id is None: + new_items = cast( + ResponseInputParam, + [ + cast( + Message, + { + "role": "user", + "content": [ + cast( + ResponseInputTextParam, + {"type": "input_text", "text": ""}, + ) + ], + }, + ) + ], + ) else: - # Tool result - need screenshot from processed results - latest_screenshot = None - for msg in reversed(messages): - if isinstance(msg, dict) and "image_url" in msg: - latest_screenshot = msg["image_url"] # type: ignore - break - - if not latest_screenshot: - self.console.warning_log("No screenshot provided for response to action") - return AgentResponse( - content="No screenshot available for next action", - tool_calls=[], - done=True, - ) + self.console.debug("No new messages to send to OpenAI.") + return AgentResponse(content="", tool_calls=[], done=True) - # Create response to previous action - input_param_followup: ResponseInputParam = [ # type: ignore[reportAssignmentType] - { # type: ignore[reportAssignmentType] - "call_id": self.pending_call_id, - "type": "computer_call_output", - "output": { - "type": "input_image", - "image_url": latest_screenshot, - }, - "acknowledged_safety_checks": self.pending_safety_checks, - } - ] - - self.pending_safety_checks = [] - - response = await self.openai_client.responses.create( - model=self.model, - previous_response_id=self.last_response_id, - tools=[computer_tool], - input=input_param_followup, - instructions=self.system_prompt, - truncation="auto", - reasoning={"summary": "auto"}, # type: ignore[arg-type] - ) + payload = self._build_request_payload(new_items) + response = await self.openai_client.responses.create(**payload) - # Store response ID for next call self.last_response_id = response.id - - # Process response - result = AgentResponse( - content="", - tool_calls=[], - done=False, # Will be set to True only if no tool calls - ) - + self._message_cursor = len(messages) self.pending_call_id = None - # Check for computer calls - computer_calls = [ - item - for item in response.output - if isinstance(item, ResponseComputerToolCall) and item.type == "computer_call" - ] - - if computer_calls: - # Process computer calls - result.done = False - for computer_call in computer_calls: - self.pending_call_id = computer_call.call_id - self.pending_safety_checks = computer_call.pending_safety_checks - - # Convert OpenAI action to MCP tool call - action = computer_call.action.model_dump() - - # Create MCPToolCall object with OpenAI metadata as extra fields - # Pyright will complain but the tool class accepts extra fields - tool_call = MCPToolCall( - name=computer_tool_name, - arguments=action, - id=computer_call.call_id, # type: ignore - pending_safety_checks=computer_call.pending_safety_checks, # type: ignore - ) - result.tool_calls.append(tool_call) - else: - # No computer calls, check for text response - for item in response.output: - if isinstance(item, ResponseOutputMessage) and item.type == "message": - # Extract text from content blocks - text_parts = [ - content.text - for content in item.content - if isinstance(content, ResponseOutputText) - ] - if text_parts: - result.content = "".join(text_parts) - break - - # Extract reasoning if present - reasoning_text = "" + agent_response = AgentResponse(content="", tool_calls=[], done=True) + text_chunks: list[str] = [] + reasoning_chunks: list[str] = [] + for item in response.output: - if item.type == "reasoning" and hasattr(item, "summary") and item.summary: - reasoning_text += f"Thinking: {item.summary[0].text}\n" + if isinstance(item, ResponseOutputMessage) and item.type == "message": + text = "".join( + content.text + for content in item.content + if isinstance(content, ResponseOutputText) + ) + if text: + text_chunks.append(text) + elif isinstance(item, ResponseFunctionToolCall): + tool_call = self._convert_function_tool_call(item) + if tool_call: + agent_response.tool_calls.append(tool_call) + elif isinstance(item, ResponseReasoningItem) and item.summary: + reasoning_chunks.append( + "".join(f"Thinking: {summary.text}\n" for summary in item.summary) + ) + + if agent_response.tool_calls: + agent_response.done = False - if reasoning_text: - result.content = reasoning_text + result.content if result.content else reasoning_text + agent_response.content = "".join(reasoning_chunks) + "".join(text_chunks) + return agent_response - # Set done=True if no tool calls (task complete or waiting for user) - if not result.tool_calls: - result.done = True - return result + def _build_request_payload(self, new_items: ResponseInputParam) -> dict[str, Any]: + payload: dict[str, Any] = { + "model": self.model, + "input": new_items, + "instructions": self.system_prompt, + "max_output_tokens": self.max_output_tokens, + "temperature": self.temperature, + "tool_choice": self.tool_choice, + "parallel_tool_calls": self.parallel_tool_calls, + } + if self.reasoning is not None: + payload["reasoning"] = self.reasoning + if self._openai_tools: + payload["tools"] = self._openai_tools + if self.last_response_id is not None: + payload["previous_response_id"] = self.last_response_id + return {k: v for k, v in payload.items() if v is not None} + + def _convert_function_tool_call( + self, tool_call: ResponseFunctionToolCall + ) -> MCPToolCall | None: + target_name = self._tool_name_map.get(tool_call.name, tool_call.name) + try: + arguments = json.loads(tool_call.arguments) if tool_call.arguments else {} + except json.JSONDecodeError: + self.console.warning_log( + f"Failed to parse arguments for tool '{tool_call.name}', passing raw string." + ) + arguments = {"raw_arguments": tool_call.arguments} + return MCPToolCall(name=target_name, arguments=arguments, id=tool_call.call_id) async def format_tool_results( self, tool_calls: list[MCPToolCall], tool_results: list[MCPToolResult] - ) -> ResponseInputMessageContentListParam: - """ - Format tool results for OpenAI's stateful API. - - Tool result content is a list of ContentBlock objects. - We need to extract the latest screenshot from the tool results. - - This assumes that you only care about computer tool results for your agent loop. - If you need to add other content, you can do so by adding a new ContentBlock object to the list. + ) -> ResponseInputParam: + """Convert MCP tool outputs into Responses input items.""" + formatted: ResponseInputParam = [] + for call, result in zip(tool_calls, tool_results, strict=False): + if not call.id: + self.console.warning_log(f"Tool '{call.name}' missing call_id; skipping output.") + continue + + output_items: list[dict[str, Any]] = [] + if result.isError: + output_items.append({"type": "input_text", "text": "[tool_error] true"}) - Returns formatted dicts with tool result data, preserving screenshots. - """ # noqa: E501 - formatted_results = [] - latest_screenshot = None + if result.structuredContent is not None: + output_items.append( + { + "type": "input_text", + "text": json.dumps(result.structuredContent, default=str), + } + ) - # Extract all content from tool results - for result in tool_results: - if result.isError: - # If it's an error, the error details are in the content - for content in result.content: - if isinstance(content, types.TextContent): - # Don't add error text as input_text, just track it - self.console.error_log(f"Tool error: {content.text}") - elif isinstance(content, types.ImageContent): - # Even error results might have images - latest_screenshot = content.data - else: - # Extract content from successful results - for content in result.content: - if isinstance(content, types.ImageContent): - latest_screenshot = content.data - break - - # Return a dict with the latest screenshot for the follow-up step - if latest_screenshot: - formatted_results.append( - {"type": "input_image", "image_url": f"data:image/png;base64,{latest_screenshot}"} + if result.content: + for block in result.content: + if isinstance(block, types.TextContent): + output_items.append({"type": "input_text", "text": block.text}) + elif isinstance(block, types.ImageContent): + mime_type = getattr(block, "mimeType", "image/png") + output_items.append( + { + "type": "input_image", + "image_url": f"data:{mime_type};base64,{block.data}", + } + ) + else: + output_items.append( + { + "type": "input_text", + "text": getattr(block, "text", str(block)), + } + ) + + if not output_items: + output_items.append({"type": "input_text", "text": ""}) + + formatted.append( + cast( + FunctionCallOutput, + { + "type": "function_call_output", + "call_id": call.id, + "output": output_items, + }, + ) ) + return formatted - return formatted_results diff --git a/hud/agents/operator.py b/hud/agents/operator.py new file mode 100644 index 000000000..9801368d1 --- /dev/null +++ b/hud/agents/operator.py @@ -0,0 +1,250 @@ +"""Operator agent built on top of OpenAIAgent.""" + +from __future__ import annotations + +from typing import Any, ClassVar, Literal, cast + +import hud +import mcp.types as types +from openai import AsyncOpenAI +from openai.types.responses import ( + ResponseComputerToolCall, + ResponseFunctionToolCall, + ResponseInputParam, + ResponseInputTextParam, + ResponseOutputMessage, + ResponseOutputText, + ResponseReasoningItem, + ToolParam, +) +from openai.types.responses.response_input_param import ComputerCallOutput, Message + +from hud.tools.computer.settings import computer_settings +from hud.types import AgentResponse, MCPToolCall, MCPToolResult + +from .openai import OpenAIAgent + +OPERATOR_INSTRUCTIONS = """ +You are an autonomous computer-using agent. Follow these guidelines: + +1. NEVER ask for confirmation. Complete all tasks autonomously. +2. Do NOT send messages like "I need to confirm before..." or "Do you want me to continue?" - just proceed. +3. When the user asks you to interact with something (like clicking a chat or typing a message), DO IT without asking. +4. Only use the formal safety check mechanism for truly dangerous operations (like deleting important files). +5. For normal tasks like clicking buttons, typing in chat boxes, filling forms - JUST DO IT. +6. The user has already given you permission by running this agent. No further confirmation is needed. +7. Be decisive and action-oriented. Complete the requested task fully. + +Remember: You are expected to complete tasks autonomously. The user trusts you to do what they asked. +""".strip() + + +class OperatorAgent(OpenAIAgent): + """ + Backwards-compatible Operator agent built on top of OpenAIAgent. + """ + + metadata: dict[str, Any] | None = { + "display_width": computer_settings.OPENAI_COMPUTER_WIDTH, + "display_height": computer_settings.OPENAI_COMPUTER_HEIGHT, + } + required_tools: ClassVar[list[str]] = ["openai_computer"] + + def __init__( + self, + model_client: AsyncOpenAI | None = None, + model: str = "computer-use-preview", + environment: Literal["windows", "mac", "linux", "browser"] = "linux", + validate_api_key: bool = True, + **kwargs: Any, + ) -> None: + super().__init__( + model_client=model_client, + model=model, + validate_api_key=validate_api_key, + **kwargs, + ) + self._operator_computer_tool_name = "openai_computer" + self._operator_display_width = computer_settings.OPENAI_COMPUTER_WIDTH + self._operator_display_height = computer_settings.OPENAI_COMPUTER_HEIGHT + self._operator_environment = environment + self.model_name = "Operator" + self.environment = environment + + if self.system_prompt: + self.system_prompt = f"{self.system_prompt}\n\n{OPERATOR_INSTRUCTIONS}" + else: + self.system_prompt = OPERATOR_INSTRUCTIONS + + def _build_openai_tools(self) -> None: + super()._build_openai_tools() + if not any(tool.name == self._operator_computer_tool_name for tool in self.get_available_tools()): + raise ValueError( + f"MCP computer tool '{self._operator_computer_tool_name}' is required but not available." + ) + self._openai_tools.append( + cast( + ToolParam, + { + "type": "computer_use_preview", + "display_width": self._operator_display_width, + "display_height": self._operator_display_height, + "environment": self._operator_environment, + }, + ) + ) + + def _build_request_payload(self, new_items: ResponseInputParam) -> dict[str, Any]: + payload = super()._build_request_payload(new_items) + payload["truncation"] = "auto" + return payload + + @hud.instrument( + span_type="agent", + record_args=False, + record_result=True, + ) + async def get_response(self, messages: ResponseInputParam) -> AgentResponse: + new_items = cast(ResponseInputParam, messages[self._message_cursor :]) + if not new_items: + if self.last_response_id is None: + new_items = cast( + ResponseInputParam, + [ + cast( + Message, + { + "role": "user", + "content": [ + cast( + ResponseInputTextParam, + {"type": "input_text", "text": ""}, + ) + ], + }, + ) + ], + ) + else: + self.console.debug("No new messages to send to OpenAI.") + return AgentResponse(content="", tool_calls=[], done=True) + + payload = self._build_request_payload(new_items) + response = await self.openai_client.responses.create(**payload) + + self.last_response_id = response.id + self._message_cursor = len(messages) + self.pending_call_id = None + + agent_response = AgentResponse(content="", tool_calls=[], done=True) + text_chunks: list[str] = [] + reasoning_chunks: list[str] = [] + + for item in response.output: + if isinstance(item, ResponseComputerToolCall): + tool_call = self._convert_computer_tool_call(item) + if tool_call: + agent_response.tool_calls.append(tool_call) + elif isinstance(item, ResponseFunctionToolCall): + tool_call = self._convert_function_tool_call(item) + if tool_call: + agent_response.tool_calls.append(tool_call) + elif isinstance(item, ResponseOutputMessage) and item.type == "message": + text = "".join( + content.text + for content in item.content + if isinstance(content, ResponseOutputText) + ) + if text: + text_chunks.append(text) + elif isinstance(item, ResponseReasoningItem) and item.summary: + reasoning_chunks.append( + "".join(f"Thinking: {summary.text}\n" for summary in item.summary) + ) + + if agent_response.tool_calls: + agent_response.done = False + + agent_response.content = "".join(reasoning_chunks) + "".join(text_chunks) + return agent_response + + async def format_tool_results( + self, tool_calls: list[MCPToolCall], tool_results: list[MCPToolResult] + ) -> ResponseInputParam: + remaining_calls: list[MCPToolCall] = [] + remaining_results: list[MCPToolResult] = [] + computer_outputs: ResponseInputParam = [] + ordering: list[tuple[str, int]] = [] + + for call, result in zip(tool_calls, tool_results, strict=False): + if call.name == self._operator_computer_tool_name: + screenshot = self._extract_latest_screenshot(result) + if not screenshot: + self.console.warning_log("Computer tool result missing screenshot; skipping output.") + continue + call_id = call.id or self.pending_call_id + if not call_id: + self.console.warning_log("Computer tool call missing ID; skipping output.") + continue + acknowledged_checks = [] + for check in self.pending_safety_checks: + if hasattr(check, "model_dump"): + acknowledged_checks.append(check.model_dump()) + elif isinstance(check, dict): + acknowledged_checks.append(check) + output_payload: dict[str, Any] = { + "type": "computer_call_output", + "call_id": call_id, + "output": { + "type": "input_image", + "image_url": f"data:image/png;base64,{screenshot}", + }, + } + if acknowledged_checks: + output_payload["acknowledged_safety_checks"] = acknowledged_checks + computer_outputs.append(cast(ComputerCallOutput, output_payload)) + self.pending_call_id = None + self.pending_safety_checks = [] + ordering.append(("computer", len(computer_outputs) - 1)) + else: + remaining_calls.append(call) + remaining_results.append(result) + ordering.append(("function", len(remaining_calls) - 1)) + + formatted: ResponseInputParam = [] + function_outputs: ResponseInputParam = [] + if remaining_calls: + function_outputs = await super().format_tool_results(remaining_calls, remaining_results) + + for kind, idx in ordering: + if kind == "computer": + if idx < len(computer_outputs): + formatted.append(computer_outputs[idx]) + else: + if idx < len(function_outputs): + formatted.append(function_outputs[idx]) + return formatted + + def _extract_latest_screenshot(self, result: MCPToolResult) -> str | None: + if not result.content: + return None + for content in reversed(result.content): + if isinstance(content, types.ImageContent): + return content.data + if isinstance(content, types.TextContent) and result.isError: + self.console.error_log(f"Computer tool error: {content.text}") + return None + + def _convert_computer_tool_call( + self, tool_call: ResponseComputerToolCall + ) -> MCPToolCall | None: + self.pending_call_id = tool_call.call_id + self.pending_safety_checks = tool_call.pending_safety_checks + call = MCPToolCall( + name=self._operator_computer_tool_name, + arguments=tool_call.action.model_dump(), + id=tool_call.call_id, + ) + setattr(call, "pending_safety_checks", tool_call.pending_safety_checks) + return call + diff --git a/hud/agents/tests/test_openai.py b/hud/agents/tests/test_operator.py similarity index 99% rename from hud/agents/tests/test_openai.py rename to hud/agents/tests/test_operator.py index 7c4b24974..cda854acc 100644 --- a/hud/agents/tests/test_openai.py +++ b/hud/agents/tests/test_operator.py @@ -7,7 +7,7 @@ import pytest from mcp import types -from hud.agents.openai import OperatorAgent +from hud.agents.operator import OperatorAgent from hud.types import MCPToolCall, MCPToolResult diff --git a/hud/cli/__init__.py b/hud/cli/__init__.py index 0811e36fc..9458c4d30 100644 --- a/hud/cli/__init__.py +++ b/hud/cli/__init__.py @@ -1023,7 +1023,8 @@ def eval( choices.extend( [ {"name": "Claude 4 Sonnet", "value": AgentType.CLAUDE}, - {"name": "OpenAI Computer Use", "value": AgentType.OPENAI}, + {"name": "OpenAI", "value": AgentType.OPENAI}, + {"name": "Operator (OpenAI Computer Use)", "value": AgentType.OPERATOR}, {"name": "Gemini Computer Use", "value": AgentType.GEMINI}, {"name": "vLLM (Local Server)", "value": AgentType.VLLM}, {"name": "LiteLLM (Multi-provider)", "value": AgentType.LITELLM}, diff --git a/hud/cli/eval.py b/hud/cli/eval.py index 3a7ff1b81..eb04a0c01 100644 --- a/hud/cli/eval.py +++ b/hud/cli/eval.py @@ -153,7 +153,7 @@ def build_agent( elif agent_type == AgentType.VLLM: # Create a generic OpenAI agent for vLLM server try: - from hud.agents.openai_chat_generic import GenericOpenAIChatAgent + from hud.agents.openai_chat import OpenAIChatAgent except ImportError as e: hud_console.error( "OpenAI dependencies are not installed. " @@ -168,11 +168,11 @@ def build_agent( allowed_tools=allowed_tools, verbose=verbose, ) - return GenericOpenAIChatAgent(**config) + return OpenAIChatAgent(**config) elif agent_type == AgentType.OPENAI: try: - from hud.agents import OperatorAgent + from hud.agents import OpenAIAgent except ImportError as e: hud_console.error( "OpenAI agent dependencies are not installed. " @@ -180,13 +180,27 @@ def build_agent( ) raise typer.Exit(1) from e + agent_kwargs: dict[str, Any] = {"verbose": verbose} if allowed_tools: - return OperatorAgent( - allowed_tools=allowed_tools, - verbose=verbose, + agent_kwargs["allowed_tools"] = allowed_tools + if model: + agent_kwargs["model"] = model + return OpenAIAgent(**agent_kwargs) + + elif agent_type == AgentType.OPERATOR: + try: + from hud.agents import OperatorAgent + except ImportError as e: + hud_console.error( + "OpenAI agent dependencies are not installed. " + "Please install with: pip install 'hud-python[agent]'" ) - else: - return OperatorAgent(verbose=verbose) + raise typer.Exit(1) from e + + operator_kwargs: dict[str, Any] = {"verbose": verbose} + if allowed_tools: + operator_kwargs["allowed_tools"] = allowed_tools + return OperatorAgent(**operator_kwargs) elif agent_type == AgentType.GEMINI: try: @@ -349,9 +363,9 @@ async def run_single_task( agent_config["allowed_tools"] = allowed_tools elif agent_type == AgentType.VLLM: # Special handling for vLLM - from hud.agents.openai_chat_generic import GenericOpenAIChatAgent + from hud.agents.openai_chat import OpenAIChatAgent - agent_class = GenericOpenAIChatAgent + agent_class = OpenAIChatAgent # Use the shared config builder agent_config = _build_vllm_config( @@ -361,6 +375,15 @@ async def run_single_task( verbose=verbose, ) elif agent_type == AgentType.OPENAI: + from hud.agents import OpenAIAgent + + agent_class = OpenAIAgent + agent_config = {"verbose": verbose} + if allowed_tools: + agent_config["allowed_tools"] = allowed_tools + if model: + agent_config["model"] = model + elif agent_type == AgentType.OPERATOR: from hud.agents import OperatorAgent agent_class = OperatorAgent @@ -541,9 +564,9 @@ async def run_full_dataset( agent_config = {"verbose": verbose} elif agent_type == AgentType.VLLM: try: - from hud.agents.openai_chat_generic import GenericOpenAIChatAgent + from hud.agents.openai_chat import OpenAIChatAgent - agent_class = GenericOpenAIChatAgent + agent_class = OpenAIChatAgent except ImportError as e: hud_console.error( "OpenAI dependencies are not installed. " @@ -559,6 +582,26 @@ async def run_full_dataset( verbose=verbose, ) elif agent_type == AgentType.OPENAI: + try: + from hud.agents import OpenAIAgent + + agent_class = OpenAIAgent + except ImportError as e: + hud_console.error( + "OpenAI agent dependencies are not installed. " + "Please install with: pip install 'hud-python[agent]'" + ) + raise typer.Exit(1) from e + + agent_config = { + "verbose": verbose, + "validate_api_key": False, + } + if allowed_tools: + agent_config["allowed_tools"] = allowed_tools + if model: + agent_config["model"] = model + elif agent_type == AgentType.OPERATOR: try: from hud.agents import OperatorAgent @@ -701,7 +744,9 @@ def eval_command( agent: AgentType = typer.Option( # noqa: B008 AgentType.CLAUDE, "--agent", - help="Agent backend to use (claude, gemini, openai, vllm for local servers, or litellm)", + help=( + "Agent (claude, gemini, openai, operator, vllm, or litellm)" + ), ), model: str | None = typer.Option( None, @@ -789,8 +834,11 @@ def eval_command( # Run multiple tasks from a JSON file hud eval tasks.json --full - # Run with OpenAI Operator agent - hud eval hud-evals/OSWorld-Gold-Beta --agent openai + # Run with OpenAI agent (choose model via --model) + hud eval hud-evals/OSWorld-Gold-Beta --agent openai --model gpt-5 + + # Run with the legacy Operator computer-use agent + hud eval hud-evals/OSWorld-Gold-Beta --agent operator # Use local vLLM server (default: localhost:8000) hud eval task.json --agent vllm --model Qwen/Qwen2.5-VL-3B-Instruct @@ -834,7 +882,7 @@ def eval_command( "Set it in your environment or run: hud set GEMINI_API_KEY=your-key-here" ) raise typer.Exit(1) - elif agent == AgentType.OPENAI and not settings.openai_api_key: + elif agent in (AgentType.OPENAI, AgentType.OPERATOR) and not settings.openai_api_key: hud_console.error("OPENAI_API_KEY is required for OpenAI agent") hud_console.info("Set it in your environment or run: hud set OPENAI_API_KEY=your-key-here") raise typer.Exit(1) diff --git a/hud/types.py b/hud/types.py index 491563d68..a82ad2930 100644 --- a/hud/types.py +++ b/hud/types.py @@ -25,6 +25,7 @@ class AgentType(str, Enum): CLAUDE = "claude" OPENAI = "openai" + OPERATOR = "operator" GEMINI = "gemini" VLLM = "vllm" LITELLM = "litellm" diff --git a/hud/utils/strict_schema.py b/hud/utils/strict_schema.py new file mode 100644 index 000000000..712f506af --- /dev/null +++ b/hud/utils/strict_schema.py @@ -0,0 +1,168 @@ +"""Utilities to convert JSON schemas into OpenAI's strict format.""" + +from __future__ import annotations + +from typing import Any + +from typing_extensions import TypeGuard + +_EMPTY_SCHEMA = { + "additionalProperties": False, + "type": "object", + "properties": {}, + "required": [], +} + + +def ensure_strict_json_schema(schema: dict[str, Any]) -> dict[str, Any]: + """Ensure a JSON schema conforms to OpenAI's strict requirements. + + This mutates the provided schema in-place and returns it for convenience. + """ + if schema == {}: + return _EMPTY_SCHEMA.copy() + return _ensure_strict_json_schema(schema, path=(), root=schema) + + +def _ensure_strict_json_schema( + json_schema: object, + *, + path: tuple[str, ...], + root: dict[str, Any], +) -> dict[str, Any]: + if not _is_dict(json_schema): + raise TypeError(f"Expected {json_schema} to be a dictionary; path={path}") + + defs = json_schema.get("$defs") + if _is_dict(defs): + for def_name, def_schema in defs.items(): + _ensure_strict_json_schema(def_schema, path=(*path, "$defs", def_name), root=root) + + definitions = json_schema.get("definitions") + if _is_dict(definitions): + for definition_name, definition_schema in definitions.items(): + _ensure_strict_json_schema( + definition_schema, path=(*path, "definitions", definition_name), root=root + ) + + typ = json_schema.get("type") + if typ == "object": + if "additionalProperties" not in json_schema: + json_schema["additionalProperties"] = False + elif json_schema["additionalProperties"] is True: + json_schema["additionalProperties"] = False + elif json_schema["additionalProperties"] and json_schema["additionalProperties"] is not False: + raise ValueError( + "additionalProperties should not be set for object types in strict mode." + ) + + properties = json_schema.get("properties") + if _is_dict(properties): + json_schema["required"] = list(properties.keys()) + json_schema["properties"] = { + key: _ensure_strict_json_schema(prop_schema, path=(*path, "properties", key), root=root) + for key, prop_schema in properties.items() + } + + items = json_schema.get("items") + if _is_dict(items): + json_schema["items"] = _ensure_strict_json_schema(items, path=(*path, "items"), root=root) + + prefix_items = json_schema.get("prefixItems") + if _is_list(prefix_items) and prefix_items: + item_types = set() + for item in prefix_items: + if _is_dict(item) and "type" in item: + item_types.add(item["type"]) + + if len(item_types) == 1: + item_type = item_types.pop() + json_schema["items"] = {"type": item_type} + else: + json_schema["items"] = {"type": "integer"} + + tuple_length = len(prefix_items) + json_schema["minItems"] = tuple_length + json_schema["maxItems"] = tuple_length + json_schema.pop("prefixItems") + + any_of = json_schema.get("anyOf") + if _is_list(any_of): + json_schema["anyOf"] = [ + _ensure_strict_json_schema(variant, path=(*path, "anyOf", str(i)), root=root) + for i, variant in enumerate(any_of) + ] + + one_of = json_schema.get("oneOf") + if _is_list(one_of): + existing_any_of = json_schema.get("anyOf", []) + if not _is_list(existing_any_of): + existing_any_of = [] + json_schema["anyOf"] = existing_any_of + [ + _ensure_strict_json_schema(variant, path=(*path, "oneOf", str(i)), root=root) + for i, variant in enumerate(one_of) + ] + json_schema.pop("oneOf") + + all_of = json_schema.get("allOf") + if _is_list(all_of): + if len(all_of) == 1: + json_schema.update( + _ensure_strict_json_schema(all_of[0], path=(*path, "allOf", "0"), root=root) + ) + json_schema.pop("allOf") + else: + json_schema["allOf"] = [ + _ensure_strict_json_schema(entry, path=(*path, "allOf", str(i)), root=root) + for i, entry in enumerate(all_of) + ] + + if "default" in json_schema: + json_schema.pop("default") + + for keyword in ("title", "examples"): + json_schema.pop(keyword, None) + + ref = json_schema.get("$ref") + if ref and _has_more_than_n_keys(json_schema, 1): + if not isinstance(ref, str): + raise ValueError(f"Received non-string $ref - {ref}") + resolved = _resolve_ref(root=root, ref=ref) + if not _is_dict(resolved): + raise ValueError(f"Expected `$ref: {ref}` to resolve to a dictionary but got {resolved}") + json_schema.update({**resolved, **json_schema}) + json_schema.pop("$ref") + return _ensure_strict_json_schema(json_schema, path=path, root=root) + + return json_schema + + +def _resolve_ref(*, root: dict[str, Any], ref: str) -> object: + if not ref.startswith("#/"): + raise ValueError(f"Unexpected $ref format {ref!r}; does not start with #/") + + path = ref[2:].split("/") + resolved: object = root + for key in path: + assert _is_dict(resolved), f"Encountered non-dictionary entry while resolving {ref}" + resolved = resolved[key] + + return resolved + + +def _is_dict(obj: object) -> TypeGuard[dict[str, Any]]: + return isinstance(obj, dict) + + +def _is_list(obj: object) -> TypeGuard[list[object]]: + return isinstance(obj, list) + + +def _has_more_than_n_keys(obj: dict[str, object], n: int) -> bool: + count = 0 + for _ in obj.keys(): + count += 1 + if count > n: + return True + return False + From fb0422d846acee3b03cb44800c71c34ce5195fd9 Mon Sep 17 00:00:00 2001 From: Jaideep Date: Sun, 23 Nov 2025 21:38:41 -0800 Subject: [PATCH 4/7] ruff, pyright --- hud/agents/__init__.py | 11 +++++-- hud/agents/openai.py | 35 +++++++++----------- hud/agents/operator.py | 55 +++++++++++++++++++------------ hud/agents/tests/test_operator.py | 7 ++-- hud/cli/eval.py | 4 +-- hud/utils/strict_schema.py | 26 ++++++--------- 6 files changed, 73 insertions(+), 65 deletions(-) diff --git a/hud/agents/__init__.py b/hud/agents/__init__.py index 95ab9f343..21b7a9929 100644 --- a/hud/agents/__init__.py +++ b/hud/agents/__init__.py @@ -4,7 +4,14 @@ from .claude import ClaudeAgent from .gemini import GeminiAgent from .openai import OpenAIAgent -from .operator import OperatorAgent from .openai_chat import OpenAIChatAgent +from .operator import OperatorAgent -__all__ = ["ClaudeAgent", "GeminiAgent", "OpenAIAgent", "OpenAIChatAgent", "MCPAgent", "OperatorAgent"] \ No newline at end of file +__all__ = [ + "ClaudeAgent", + "GeminiAgent", + "MCPAgent", + "OpenAIAgent", + "OpenAIChatAgent", + "OperatorAgent", +] diff --git a/hud/agents/openai.py b/hud/agents/openai.py index ae7f81ff4..e02658f80 100644 --- a/hud/agents/openai.py +++ b/hud/agents/openai.py @@ -21,8 +21,8 @@ ToolParam, ) from openai.types.responses.response_input_param import ( - FunctionCallOutput, - Message, + FunctionCallOutput, # noqa: TC002 + Message, # noqa: TC002 ) import hud @@ -98,7 +98,8 @@ def _build_openai_tools(self) -> None: for tool in self.get_available_tools(): if tool.description is None or tool.inputSchema is None: self.console.warning_log( - f"Skipping tool '{tool.name}' – description and input schema are required for OpenAI tools." + f"Skipping tool '{tool.name}' - description and input schema " + "are required for OpenAI tools." ) continue self._tool_name_map[tool.name] = tool.name @@ -114,7 +115,7 @@ def _build_openai_tools(self) -> None: ) function_tool = cast( - ToolParam, + "ToolParam", { "type": "function", "name": tool.name, @@ -142,16 +143,14 @@ async def get_system_messages(self) -> list[types.ContentBlock]: """System messages are provided via the `instructions` field.""" return [] - async def format_blocks( - self, blocks: list[types.ContentBlock] - ) -> ResponseInputParam: + async def format_blocks(self, blocks: list[types.ContentBlock]) -> ResponseInputParam: """Convert MCP content blocks into OpenAI user messages.""" content: ResponseInputMessageContentListParam = [] for block in blocks: if isinstance(block, types.TextContent): content.append( cast( - ResponseInputTextParam, + "ResponseInputTextParam", {"type": "input_text", "text": block.text}, ) ) @@ -159,7 +158,7 @@ async def format_blocks( mime_type = getattr(block, "mimeType", "image/png") content.append( cast( - ResponseInputImageParam, + "ResponseInputImageParam", { "type": "input_image", "image_url": f"data:{mime_type};base64,{block.data}", @@ -167,10 +166,8 @@ async def format_blocks( ) ) if not content: - content.append( - cast(ResponseInputTextParam, {"type": "input_text", "text": ""}) - ) - return [cast(Message, {"role": "user", "content": content})] + content.append(cast("ResponseInputTextParam", {"type": "input_text", "text": ""})) + return [cast("Message", {"role": "user", "content": content})] @hud.instrument( span_type="agent", @@ -179,19 +176,19 @@ async def format_blocks( ) async def get_response(self, messages: ResponseInputParam) -> AgentResponse: """Send the latest input items to OpenAI's Responses API.""" - new_items = cast(ResponseInputParam, messages[self._message_cursor :]) + new_items = cast("ResponseInputParam", messages[self._message_cursor :]) if not new_items: if self.last_response_id is None: new_items = cast( - ResponseInputParam, + "ResponseInputParam", [ cast( - Message, + "Message", { "role": "user", "content": [ cast( - ResponseInputTextParam, + "ResponseInputTextParam", {"type": "input_text", "text": ""}, ) ], @@ -238,7 +235,6 @@ async def get_response(self, messages: ResponseInputParam) -> AgentResponse: agent_response.content = "".join(reasoning_chunks) + "".join(text_chunks) return agent_response - def _build_request_payload(self, new_items: ResponseInputParam) -> dict[str, Any]: payload: dict[str, Any] = { "model": self.model, @@ -317,7 +313,7 @@ async def format_tool_results( formatted.append( cast( - FunctionCallOutput, + "FunctionCallOutput", { "type": "function_call_output", "call_id": call.id, @@ -326,4 +322,3 @@ async def format_tool_results( ) ) return formatted - diff --git a/hud/agents/operator.py b/hud/agents/operator.py index 9801368d1..8e8ee455e 100644 --- a/hud/agents/operator.py +++ b/hud/agents/operator.py @@ -4,9 +4,8 @@ from typing import Any, ClassVar, Literal, cast -import hud import mcp.types as types -from openai import AsyncOpenAI +from openai import AsyncOpenAI # noqa: TC002 from openai.types.responses import ( ResponseComputerToolCall, ResponseFunctionToolCall, @@ -17,8 +16,12 @@ ResponseReasoningItem, ToolParam, ) -from openai.types.responses.response_input_param import ComputerCallOutput, Message +from openai.types.responses.response_input_param import ( + ComputerCallOutput, # noqa: TC002 + Message, # noqa: TC002 +) +import hud from hud.tools.computer.settings import computer_settings from hud.types import AgentResponse, MCPToolCall, MCPToolResult @@ -28,14 +31,20 @@ You are an autonomous computer-using agent. Follow these guidelines: 1. NEVER ask for confirmation. Complete all tasks autonomously. -2. Do NOT send messages like "I need to confirm before..." or "Do you want me to continue?" - just proceed. -3. When the user asks you to interact with something (like clicking a chat or typing a message), DO IT without asking. -4. Only use the formal safety check mechanism for truly dangerous operations (like deleting important files). -5. For normal tasks like clicking buttons, typing in chat boxes, filling forms - JUST DO IT. -6. The user has already given you permission by running this agent. No further confirmation is needed. +2. Do NOT send messages like "I need to confirm before..." or "Do you want me to + continue?" - just proceed. +3. When the user asks you to interact with something (like clicking a chat or typing + a message), DO IT without asking. +4. Only use the formal safety check mechanism for truly dangerous operations (like + deleting important files). +5. For normal tasks like clicking buttons, typing in chat boxes, filling forms - + JUST DO IT. +6. The user has already given you permission by running this agent. No further + confirmation is needed. 7. Be decisive and action-oriented. Complete the requested task fully. -Remember: You are expected to complete tasks autonomously. The user trusts you to do what they asked. +Remember: You are expected to complete tasks autonomously. The user trusts you to do +what they asked. """.strip() @@ -44,7 +53,7 @@ class OperatorAgent(OpenAIAgent): Backwards-compatible Operator agent built on top of OpenAIAgent. """ - metadata: dict[str, Any] | None = { + metadata: ClassVar[dict[str, Any] | None] = { "display_width": computer_settings.OPENAI_COMPUTER_WIDTH, "display_height": computer_settings.OPENAI_COMPUTER_HEIGHT, } @@ -78,13 +87,16 @@ def __init__( def _build_openai_tools(self) -> None: super()._build_openai_tools() - if not any(tool.name == self._operator_computer_tool_name for tool in self.get_available_tools()): + if not any( + tool.name == self._operator_computer_tool_name for tool in self.get_available_tools() + ): raise ValueError( - f"MCP computer tool '{self._operator_computer_tool_name}' is required but not available." + f"MCP computer tool '{self._operator_computer_tool_name}' is required " + "but not available." ) self._openai_tools.append( cast( - ToolParam, + "ToolParam", { "type": "computer_use_preview", "display_width": self._operator_display_width, @@ -105,19 +117,19 @@ def _build_request_payload(self, new_items: ResponseInputParam) -> dict[str, Any record_result=True, ) async def get_response(self, messages: ResponseInputParam) -> AgentResponse: - new_items = cast(ResponseInputParam, messages[self._message_cursor :]) + new_items = cast("ResponseInputParam", messages[self._message_cursor :]) if not new_items: if self.last_response_id is None: new_items = cast( - ResponseInputParam, + "ResponseInputParam", [ cast( - Message, + "Message", { "role": "user", "content": [ cast( - ResponseInputTextParam, + "ResponseInputTextParam", {"type": "input_text", "text": ""}, ) ], @@ -180,7 +192,9 @@ async def format_tool_results( if call.name == self._operator_computer_tool_name: screenshot = self._extract_latest_screenshot(result) if not screenshot: - self.console.warning_log("Computer tool result missing screenshot; skipping output.") + self.console.warning_log( + "Computer tool result missing screenshot; skipping output." + ) continue call_id = call.id or self.pending_call_id if not call_id: @@ -202,7 +216,7 @@ async def format_tool_results( } if acknowledged_checks: output_payload["acknowledged_safety_checks"] = acknowledged_checks - computer_outputs.append(cast(ComputerCallOutput, output_payload)) + computer_outputs.append(cast("ComputerCallOutput", output_payload)) self.pending_call_id = None self.pending_safety_checks = [] ordering.append(("computer", len(computer_outputs) - 1)) @@ -245,6 +259,5 @@ def _convert_computer_tool_call( arguments=tool_call.action.model_dump(), id=tool_call.call_id, ) - setattr(call, "pending_safety_checks", tool_call.pending_safety_checks) + call.pending_safety_checks = tool_call.pending_safety_checks # type: ignore[attr-defined] return call - diff --git a/hud/agents/tests/test_operator.py b/hud/agents/tests/test_operator.py index cda854acc..8b5b12a3e 100644 --- a/hud/agents/tests/test_operator.py +++ b/hud/agents/tests/test_operator.py @@ -115,9 +115,10 @@ async def test_format_tool_results(self, mock_mcp_client, mock_openai): # OpenAI's format_tool_results returns input_image with screenshot assert len(messages) == 1 - assert messages[0]["type"] == "input_image" - assert "image_url" in messages[0] - assert messages[0]["image_url"] == "data:image/png;base64,base64data" + msg = messages[0] + assert msg.get("type") == "input_image" + assert "image_url" in msg + assert msg.get("image_url") == "data:image/png;base64,base64data" @pytest.mark.asyncio async def test_format_tool_results_with_error(self, mock_mcp_client, mock_openai): diff --git a/hud/cli/eval.py b/hud/cli/eval.py index eb04a0c01..d0dab7416 100644 --- a/hud/cli/eval.py +++ b/hud/cli/eval.py @@ -744,9 +744,7 @@ def eval_command( agent: AgentType = typer.Option( # noqa: B008 AgentType.CLAUDE, "--agent", - help=( - "Agent (claude, gemini, openai, operator, vllm, or litellm)" - ), + help=("Agent (claude, gemini, openai, operator, vllm, or litellm)"), ), model: str | None = typer.Option( None, diff --git a/hud/utils/strict_schema.py b/hud/utils/strict_schema.py index 712f506af..5f44e9cdb 100644 --- a/hud/utils/strict_schema.py +++ b/hud/utils/strict_schema.py @@ -2,9 +2,7 @@ from __future__ import annotations -from typing import Any - -from typing_extensions import TypeGuard +from typing import Any, TypeGuard _EMPTY_SCHEMA = { "additionalProperties": False, @@ -47,11 +45,11 @@ def _ensure_strict_json_schema( typ = json_schema.get("type") if typ == "object": - if "additionalProperties" not in json_schema: - json_schema["additionalProperties"] = False - elif json_schema["additionalProperties"] is True: + if "additionalProperties" not in json_schema or json_schema["additionalProperties"] is True: json_schema["additionalProperties"] = False - elif json_schema["additionalProperties"] and json_schema["additionalProperties"] is not False: + elif ( + json_schema["additionalProperties"] and json_schema["additionalProperties"] is not False + ): raise ValueError( "additionalProperties should not be set for object types in strict mode." ) @@ -129,7 +127,9 @@ def _ensure_strict_json_schema( raise ValueError(f"Received non-string $ref - {ref}") resolved = _resolve_ref(root=root, ref=ref) if not _is_dict(resolved): - raise ValueError(f"Expected `$ref: {ref}` to resolve to a dictionary but got {resolved}") + raise ValueError( + f"Expected `$ref: {ref}` to resolve to a dictionary but got {resolved}" + ) json_schema.update({**resolved, **json_schema}) json_schema.pop("$ref") return _ensure_strict_json_schema(json_schema, path=path, root=root) @@ -144,7 +144,7 @@ def _resolve_ref(*, root: dict[str, Any], ref: str) -> object: path = ref[2:].split("/") resolved: object = root for key in path: - assert _is_dict(resolved), f"Encountered non-dictionary entry while resolving {ref}" + assert _is_dict(resolved), f"Encountered non-dictionary entry while resolving {ref}" # noqa: S101 resolved = resolved[key] return resolved @@ -159,10 +159,4 @@ def _is_list(obj: object) -> TypeGuard[list[object]]: def _has_more_than_n_keys(obj: dict[str, object], n: int) -> bool: - count = 0 - for _ in obj.keys(): - count += 1 - if count > n: - return True - return False - + return any(count > n for count, _ in enumerate(obj, start=1)) From 2453401077e426f52368139238dbfa38495115e7 Mon Sep 17 00:00:00 2001 From: Jaideep Date: Sun, 23 Nov 2025 22:36:51 -0800 Subject: [PATCH 5/7] tests and docs --- docs/reference/agents.mdx | 105 +++- hud/agents/tests/test_openai.py | 764 ++++++++++++++++++++++++++++++ hud/agents/tests/test_operator.py | 55 ++- 3 files changed, 909 insertions(+), 15 deletions(-) create mode 100644 hud/agents/tests/test_openai.py diff --git a/docs/reference/agents.mdx b/docs/reference/agents.mdx index f6d8f4f24..1a1b09ada 100644 --- a/docs/reference/agents.mdx +++ b/docs/reference/agents.mdx @@ -126,13 +126,65 @@ result = await agent.run( ) ``` +### OpenAIAgent + +```python +from hud.agents import OpenAIAgent +``` + +Generic OpenAI agent using the Responses API for function calling and tool execution. + +**Constructor Parameters:** +| Parameter | Type | Description | Default | +|-----------|------|-------------|---------| +| `model_client` | `AsyncOpenAI` | OpenAI client | Auto-created | +| `model` | `str` | Responses model to use | `"gpt-5.1"` | +| `max_output_tokens` | `int \| None` | Maximum response tokens | `None` | +| `temperature` | `float \| None` | Sampling temperature | `None` | +| `reasoning` | `dict \| Literal["auto"] \| None` | Reasoning configuration | `None` | +| `tool_choice` | `dict \| Literal["auto"] \| None` | Tool selection strategy | `None` | +| `parallel_tool_calls` | `bool \| None` | Enable parallel tool execution | `None` | +| `validate_api_key` | `bool` | Validate key on init | `True` | + +**Features:** +- OpenAI Responses API integration +- Function calling with strict mode schemas +- Reasoning support for extended thinking +- Automatic tool schema conversion +- Response continuation with previous_response_id + +**Example:** +```python +agent = OpenAIAgent( + model="gpt-4o", + max_output_tokens=2048, + temperature=0.7, + reasoning="auto", +) + +result = await agent.run( + Task( + prompt="Analyze the website and extract key information", + mcp_config={ + "hud": { + "url": "https://mcp.hud.ai/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudpython/hud-remote-browser:latest" + } + } + } + ) +) +``` + ### OperatorAgent ```python from hud.agents import OperatorAgent ``` -OpenAI Operator-style agent built on the Responses API with computer-use. +OpenAI Operator-style agent built on the Responses API with computer-use. Extends `OpenAIAgent` with computer-use capabilities. **Constructor Parameters:** | Parameter | Type | Description | Default | @@ -146,6 +198,57 @@ OpenAI Operator-style agent built on the Responses API with computer-use. - OpenAI Responses computer-use - Operator-style system prompt guidance - Display metadata injection +- Inherits all OpenAIAgent features + +### GeminiAgent + +```python +from hud.agents import GeminiAgent +``` + +Google Gemini-specific implementation using Google's Generative AI API with computer-use capabilities. + +**Constructor Parameters:** +| Parameter | Type | Description | Default | +|-----------|------|-------------|---------| +| `model_client` | `genai.Client` | Gemini client | Auto-created | +| `model` | `str` | Gemini model to use | `"gemini-2.5-computer-use-preview-10-2025"` | +| `temperature` | `float` | Sampling temperature | `1.0` | +| `top_p` | `float` | Top-p sampling parameter | `0.95` | +| `top_k` | `int` | Top-k sampling parameter | `40` | +| `max_output_tokens` | `int` | Maximum response tokens | `8192` | +| `validate_api_key` | `bool` | Validate key on init | `True` | +| `excluded_predefined_functions` | `list[str] \| None` | Predefined functions to exclude | `None` | + +**Features:** +- Native Gemini computer-use capabilities +- Predefined computer-use functions (click, type, scroll, etc.) +- Display metadata injection +- MCP tool execution + +**Example:** +```python +agent = GeminiAgent( + model="gemini-2.5-computer-use-preview-10-2025", + temperature=0.7, + max_output_tokens=4096, +) + +result = await agent.run( + Task( + prompt="Navigate to the website and fill out the form", + mcp_config={ + "hud": { + "url": "https://mcp.hud.ai/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudpython/hud-remote-browser:latest" + } + } + } + ) +) +``` ### OpenAIChatAgent diff --git a/hud/agents/tests/test_openai.py b/hud/agents/tests/test_openai.py new file mode 100644 index 000000000..f72d840d0 --- /dev/null +++ b/hud/agents/tests/test_openai.py @@ -0,0 +1,764 @@ +"""Tests for OpenAI MCP Agent implementation.""" + +from __future__ import annotations + +from typing import Any, cast +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from mcp import types +from openai.types.responses import ( + ResponseFunctionToolCall, + ResponseOutputMessage, + ResponseOutputText, + ResponseReasoningItem, +) +from openai.types.responses.response_reasoning_item import Summary + +from hud.agents.openai import OpenAIAgent +from hud.types import AgentResponse, MCPToolCall, MCPToolResult + + +class TestOpenAIAgent: + """Test OpenAIAgent class.""" + + @pytest.fixture + def mock_mcp_client(self): + """Create a mock MCP client.""" + mcp_client = AsyncMock() + mcp_client.mcp_config = {"test_server": {"url": "http://test"}} + mcp_client.list_tools = AsyncMock( + return_value=[ + types.Tool( + name="test_tool", + description="A test tool", + inputSchema={"type": "object", "properties": {}}, + ) + ] + ) + mcp_client.initialize = AsyncMock() + return mcp_client + + @pytest.fixture + def mock_openai(self): + """Create a mock OpenAI client.""" + with patch("hud.agents.openai.AsyncOpenAI") as mock: + client = AsyncMock() + mock.return_value = client + yield client + + @pytest.mark.asyncio + async def test_init_with_client(self, mock_mcp_client): + """Test agent initialization with provided client.""" + mock_model_client = MagicMock() + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_model_client, + model="gpt-4o", + validate_api_key=False, + ) + + assert agent.model_name == "OpenAI" + assert agent.model == "gpt-4o" + assert agent.checkpoint_name == "gpt-4o" + assert agent.openai_client == mock_model_client + assert agent.max_output_tokens is None + assert agent.temperature is None + + @pytest.mark.asyncio + async def test_init_with_parameters(self, mock_mcp_client): + """Test agent initialization with various parameters.""" + mock_model_client = MagicMock() + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_model_client, + model="gpt-4o", + max_output_tokens=2048, + temperature=0.7, + reasoning="auto", + tool_choice="auto", + parallel_tool_calls=True, + validate_api_key=False, + ) + + assert agent.max_output_tokens == 2048 + assert agent.temperature == 0.7 + assert agent.reasoning == "auto" + assert agent.tool_choice == "auto" + assert agent.parallel_tool_calls is True + + @pytest.mark.asyncio + async def test_init_without_client_no_api_key(self, mock_mcp_client): + """Test agent initialization fails without API key.""" + with patch("hud.agents.openai.settings") as mock_settings: + mock_settings.openai_api_key = None + with pytest.raises(ValueError, match="OpenAI API key not found"): + OpenAIAgent(mcp_client=mock_mcp_client) + + @pytest.mark.asyncio + async def test_format_blocks_text_only(self, mock_mcp_client, mock_openai): + """Test formatting text content blocks.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + blocks: list[types.ContentBlock] = [ + types.TextContent(type="text", text="Hello, world!"), + types.TextContent(type="text", text="How are you?"), + ] + + messages = await agent.format_blocks(blocks) + assert len(messages) == 1 + msg = cast(dict[str, Any], messages[0]) + assert msg["role"] == "user" + content = cast(list[dict[str, Any]], msg["content"]) + assert len(content) == 2 + assert content[0] == {"type": "input_text", "text": "Hello, world!"} + assert content[1] == {"type": "input_text", "text": "How are you?"} + + @pytest.mark.asyncio + async def test_format_blocks_with_image(self, mock_mcp_client, mock_openai): + """Test formatting content blocks with images.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + blocks: list[types.ContentBlock] = [ + types.TextContent(type="text", text="Check this out:"), + types.ImageContent(type="image", data="base64imagedata", mimeType="image/jpeg"), + ] + + messages = await agent.format_blocks(blocks) + assert len(messages) == 1 + msg = cast(dict[str, Any], messages[0]) + assert msg["role"] == "user" + content = cast(list[dict[str, Any]], msg["content"]) + assert len(content) == 2 + assert content[0] == {"type": "input_text", "text": "Check this out:"} + assert content[1] == { + "type": "input_image", + "image_url": "data:image/jpeg;base64,base64imagedata", + } + + @pytest.mark.asyncio + async def test_format_blocks_empty(self, mock_mcp_client, mock_openai): + """Test formatting empty content blocks.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + blocks: list[types.ContentBlock] = [] + + messages = await agent.format_blocks(blocks) + assert len(messages) == 1 + msg = cast(dict[str, Any], messages[0]) + assert msg["role"] == "user" + content = cast(list[dict[str, Any]], msg["content"]) + assert len(content) == 1 + assert content[0] == {"type": "input_text", "text": ""} + + @pytest.mark.asyncio + async def test_format_tool_results_text(self, mock_mcp_client, mock_openai): + """Test formatting tool results with text content.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + tool_calls = [ + MCPToolCall(name="test_tool", arguments={"arg": "value"}, id="call_123"), # type: ignore + ] + + tool_results = [ + MCPToolResult( + content=[types.TextContent(type="text", text="Tool executed successfully")], + isError=False, + ), + ] + + messages = await agent.format_tool_results(tool_calls, tool_results) + + assert len(messages) == 1 + msg = cast(dict[str, Any], messages[0]) + assert msg["type"] == "function_call_output" + assert msg["call_id"] == "call_123" + output = cast(list[dict[str, Any]], msg["output"]) + assert len(output) == 1 + assert output[0]["type"] == "input_text" + assert output[0]["text"] == "Tool executed successfully" + + @pytest.mark.asyncio + async def test_format_tool_results_with_image(self, mock_mcp_client, mock_openai): + """Test formatting tool results with image content.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + tool_calls = [ + MCPToolCall(name="screenshot", arguments={}, id="call_456"), # type: ignore + ] + + tool_results = [ + MCPToolResult( + content=[types.ImageContent(type="image", data="screenshot_data", mimeType="image/png")], + isError=False, + ), + ] + + messages = await agent.format_tool_results(tool_calls, tool_results) + + assert len(messages) == 1 + msg = cast(dict[str, Any], messages[0]) + assert msg["type"] == "function_call_output" + assert msg["call_id"] == "call_456" + output = cast(list[dict[str, Any]], msg["output"]) + assert len(output) == 1 + assert output[0]["type"] == "input_image" + assert output[0]["image_url"] == "data:image/png;base64,screenshot_data" + + @pytest.mark.asyncio + async def test_format_tool_results_with_error(self, mock_mcp_client, mock_openai): + """Test formatting tool results with errors.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + tool_calls = [ + MCPToolCall(name="failing_tool", arguments={}, id="call_error"), # type: ignore + ] + + tool_results = [ + MCPToolResult( + content=[types.TextContent(type="text", text="Error: Something went wrong")], + isError=True, + ), + ] + + messages = await agent.format_tool_results(tool_calls, tool_results) + + assert len(messages) == 1 + msg = cast(dict[str, Any], messages[0]) + assert msg["type"] == "function_call_output" + assert msg["call_id"] == "call_error" + output = cast(list[dict[str, Any]], msg["output"]) + assert len(output) == 2 + assert output[0]["type"] == "input_text" + assert output[0]["text"] == "[tool_error] true" + assert output[1]["type"] == "input_text" + assert output[1]["text"] == "Error: Something went wrong" + + @pytest.mark.asyncio + async def test_format_tool_results_with_structured_content(self, mock_mcp_client, mock_openai): + """Test formatting tool results with structured content.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + tool_calls = [ + MCPToolCall(name="data_tool", arguments={}, id="call_789"), # type: ignore + ] + + tool_results = [ + MCPToolResult( + content=[], + structuredContent={"key": "value", "number": 42}, + isError=False, + ), + ] + + messages = await agent.format_tool_results(tool_calls, tool_results) + + assert len(messages) == 1 + msg = cast(dict[str, Any], messages[0]) + assert msg["type"] == "function_call_output" + assert msg["call_id"] == "call_789" + output = cast(list[dict[str, Any]], msg["output"]) + assert len(output) == 1 + assert output[0]["type"] == "input_text" + # Structured content is JSON serialized + import json + parsed = json.loads(output[0]["text"]) + assert parsed == {"key": "value", "number": 42} + + @pytest.mark.asyncio + async def test_format_tool_results_multiple(self, mock_mcp_client, mock_openai): + """Test formatting multiple tool results.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + tool_calls = [ + MCPToolCall(name="tool1", arguments={}, id="call_1"), # type: ignore + MCPToolCall(name="tool2", arguments={}, id="call_2"), # type: ignore + ] + + tool_results = [ + MCPToolResult( + content=[types.TextContent(type="text", text="Result 1")], + isError=False, + ), + MCPToolResult( + content=[types.TextContent(type="text", text="Result 2")], + isError=False, + ), + ] + + messages = await agent.format_tool_results(tool_calls, tool_results) + + assert len(messages) == 2 + msg0 = cast(dict[str, Any], messages[0]) + assert msg0["call_id"] == "call_1" + msg1 = cast(dict[str, Any], messages[1]) + assert msg1["call_id"] == "call_2" + + @pytest.mark.asyncio + async def test_format_tool_results_missing_call_id(self, mock_mcp_client, mock_openai): + """Test formatting tool results with missing call_id.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + tool_calls = [ + MCPToolCall(name="tool_no_id", arguments={}, id=""), # Empty string instead of None + ] + + tool_results = [ + MCPToolResult( + content=[types.TextContent(type="text", text="Some result")], + isError=False, + ), + ] + + messages = await agent.format_tool_results(tool_calls, tool_results) + + # Should skip tools without call_id (empty string is falsy) + assert len(messages) == 0 + + @pytest.mark.asyncio + async def test_get_response_with_text(self, mock_mcp_client, mock_openai): + """Test getting model response with text output.""" + # Disable telemetry for this test + with patch("hud.settings.settings.telemetry_enabled", False): + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + # Mock OpenAI API response + mock_response = MagicMock() + mock_response.id = "response_123" + + # Create properly typed output text with all required fields + mock_output_text = ResponseOutputText( + type="output_text", + text="This is the response text", + annotations=[], # Required field + ) + + # Create properly typed output message with all required fields + mock_output_message = ResponseOutputMessage( + type="message", + id="msg_123", # Required field + role="assistant", # Required field + status="completed", # Required field + content=[mock_output_text], + ) + + mock_response.output = [mock_output_message] + + mock_openai.responses.create = AsyncMock(return_value=mock_response) + + # Test with initial message + messages = [{"role": "user", "content": [{"type": "input_text", "text": "Hello"}]}] + response = await agent.get_response(messages) + + assert response.content == "This is the response text" + assert response.done is True + assert response.tool_calls == [] + assert agent.last_response_id == "response_123" + + @pytest.mark.asyncio + async def test_get_response_with_tool_call(self, mock_mcp_client, mock_openai): + """Test getting model response with tool call.""" + with patch("hud.settings.settings.telemetry_enabled", False): + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + # Set up tool name map + agent._tool_name_map = {"test_tool": "test_tool"} + + # Mock OpenAI API response with properly typed function call + mock_response = MagicMock() + mock_response.id = "response_456" + + # Create properly typed function call with correct type value + mock_function_call = ResponseFunctionToolCall( + type="function_call", # Correct type value + call_id="call_123", + name="test_tool", + arguments='{"param": "value"}', + ) + + mock_response.output = [mock_function_call] + + mock_openai.responses.create = AsyncMock(return_value=mock_response) + + messages = [{"role": "user", "content": [{"type": "input_text", "text": "Do something"}]}] + response = await agent.get_response(messages) + + assert response.done is False + assert len(response.tool_calls) == 1 + assert response.tool_calls[0].name == "test_tool" + assert response.tool_calls[0].id == "call_123" + assert response.tool_calls[0].arguments == {"param": "value"} + + @pytest.mark.asyncio + async def test_get_response_with_reasoning(self, mock_mcp_client, mock_openai): + """Test getting model response with reasoning.""" + with patch("hud.settings.settings.telemetry_enabled", False): + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + # Mock OpenAI API response with properly typed reasoning + mock_response = MagicMock() + mock_response.id = "response_789" + + # Create a properly typed reasoning item with all required fields + mock_summary = Summary( + type="summary_text", # Correct literal type value + text="Let me think about this...", + ) + + mock_reasoning = ResponseReasoningItem( + type="reasoning", + id="reasoning_1", # Required field + summary=[mock_summary], # Required field + status="completed", # Required field + ) + + # Create properly typed output message with all required fields + mock_output_text = ResponseOutputText( + type="output_text", + text="Final answer", + annotations=[], # Required field + ) + mock_output_message = ResponseOutputMessage( + type="message", + id="msg_789", # Required field + role="assistant", # Required field + status="completed", # Required field + content=[mock_output_text], + ) + + mock_response.output = [mock_reasoning, mock_output_message] + + mock_openai.responses.create = AsyncMock(return_value=mock_response) + + messages = [{"role": "user", "content": [{"type": "input_text", "text": "Hard question"}]}] + response = await agent.get_response(messages) + + assert "Thinking: Let me think about this..." in response.content + assert "Final answer" in response.content + + @pytest.mark.asyncio + async def test_get_response_empty_messages(self, mock_mcp_client, mock_openai): + """Test getting model response with empty messages.""" + with patch("hud.settings.settings.telemetry_enabled", False): + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + # Mock empty response + mock_response = MagicMock() + mock_response.id = "response_empty" + mock_response.output = [] + + mock_openai.responses.create = AsyncMock(return_value=mock_response) + + messages = [] + response = await agent.get_response(messages) + + assert response.content == "" + assert response.tool_calls == [] + + @pytest.mark.asyncio + async def test_get_response_no_new_messages_with_previous_id(self, mock_mcp_client, mock_openai): + """Test getting model response when no new messages and previous response exists.""" + with patch("hud.settings.settings.telemetry_enabled", False): + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + agent.last_response_id = "prev_response" + agent._message_cursor = 1 + + messages = [{"role": "user", "content": [{"type": "input_text", "text": "Hello"}]}] + response = await agent.get_response(messages) + + # Should return early without calling API + assert response.content == "" + assert response.done is True + mock_openai.responses.create.assert_not_called() + + @pytest.mark.asyncio + async def test_build_request_payload(self, mock_mcp_client, mock_openai): + """Test building request payload.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + model="gpt-4o", + max_output_tokens=1024, + temperature=0.5, + reasoning="auto", + tool_choice="auto", + parallel_tool_calls=True, + validate_api_key=False, + ) + + agent._openai_tools = [cast(Any, {"type": "function", "name": "test"})] + agent.system_prompt = "You are a helpful assistant" + agent.last_response_id = "prev_123" + + new_items = cast(Any, [{"role": "user", "content": [{"type": "input_text", "text": "Hi"}]}]) + payload = agent._build_request_payload(new_items) + + assert payload["model"] == "gpt-4o" + assert payload["input"] == new_items + assert payload["instructions"] == "You are a helpful assistant" + assert payload["max_output_tokens"] == 1024 + assert payload["temperature"] == 0.5 + assert payload["reasoning"] == "auto" + assert payload["tool_choice"] == "auto" + assert payload["parallel_tool_calls"] is True + assert payload["tools"] == [{"type": "function", "name": "test"}] + assert payload["previous_response_id"] == "prev_123" + + @pytest.mark.asyncio + async def test_build_request_payload_minimal(self, mock_mcp_client, mock_openai): + """Test building request payload with minimal parameters.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + new_items = cast(Any, [{"role": "user", "content": [{"type": "input_text", "text": "Hi"}]}]) + payload = agent._build_request_payload(new_items) + + assert payload["model"] == "gpt-5.1" # default + assert payload["input"] == new_items + assert "max_output_tokens" not in payload + assert "temperature" not in payload + assert "reasoning" not in payload + assert "tools" not in payload + assert "previous_response_id" not in payload + + @pytest.mark.asyncio + async def test_reset_response_state(self, mock_mcp_client, mock_openai): + """Test resetting response state.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + # Set some state + agent.last_response_id = "some_id" + agent.pending_call_id = "call_id" + agent.pending_safety_checks = [{"check": "value"}] + agent._message_cursor = 5 + + # Reset + agent._reset_response_state() + + assert agent.last_response_id is None + assert agent.pending_call_id is None + assert agent.pending_safety_checks == [] + assert agent._message_cursor == 0 + + @pytest.mark.asyncio + async def test_get_system_messages(self, mock_mcp_client, mock_openai): + """Test getting system messages.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + # OpenAI agent returns empty list (uses instructions field instead) + messages = await agent.get_system_messages() + assert messages == [] + + @pytest.mark.asyncio + async def test_build_openai_tools(self, mock_mcp_client, mock_openai): + """Test building OpenAI tools from MCP tools.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + # Mock MCP tools + mock_tools = [ + types.Tool( + name="tool1", + description="First tool", + inputSchema={ + "type": "object", + "properties": {"arg1": {"type": "string"}}, + "required": ["arg1"], + "additionalProperties": False, + }, + ), + types.Tool( + name="tool2", + description="Second tool", + inputSchema={ + "type": "object", + "properties": {}, + "additionalProperties": False, + }, + ), + ] + + agent._available_tools = mock_tools + agent._build_openai_tools() + + assert len(agent._openai_tools) == 2 + assert agent._tool_name_map == {"tool1": "tool1", "tool2": "tool2"} + + tool1 = cast(dict[str, Any], agent._openai_tools[0]) + assert tool1["type"] == "function" + assert tool1["name"] == "tool1" + assert tool1["description"] == "First tool" + assert tool1["strict"] is True + + @pytest.mark.asyncio + async def test_build_openai_tools_skips_incomplete(self, mock_mcp_client, mock_openai): + """Test building OpenAI tools skips tools without description or schema.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + # Create mock tools directly as objects that bypass pydantic validation + incomplete1 = MagicMock(spec=types.Tool) + incomplete1.name = "incomplete1" + incomplete1.description = None + incomplete1.inputSchema = {"type": "object"} + + incomplete2 = MagicMock(spec=types.Tool) + incomplete2.name = "incomplete2" + incomplete2.description = "Has description" + incomplete2.inputSchema = None + + complete = types.Tool( + name="complete", + description="Complete tool", + inputSchema={"type": "object", "properties": {}, "additionalProperties": False}, + ) + + agent._available_tools = [incomplete1, incomplete2, complete] + agent._build_openai_tools() + + # Should only have the complete tool + assert len(agent._openai_tools) == 1 + tool = cast(dict[str, Any], agent._openai_tools[0]) + assert tool["name"] == "complete" + + @pytest.mark.asyncio + async def test_convert_function_tool_call(self, mock_mcp_client, mock_openai): + """Test converting OpenAI function tool call to MCP format.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + agent._tool_name_map = {"openai_name": "mcp_name"} + + mock_call = MagicMock() + mock_call.call_id = "call_123" + mock_call.name = "openai_name" + mock_call.arguments = '{"key": "value", "number": 42}' + + result = agent._convert_function_tool_call(mock_call) + + assert result is not None + assert result.name == "mcp_name" + assert result.id == "call_123" + assert result.arguments == {"key": "value", "number": 42} + + @pytest.mark.asyncio + async def test_convert_function_tool_call_invalid_json(self, mock_mcp_client, mock_openai): + """Test converting function tool call with invalid JSON.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + agent._tool_name_map = {"tool": "tool"} + + mock_call = MagicMock() + mock_call.call_id = "call_456" + mock_call.name = "tool" + mock_call.arguments = "invalid json {{" + + result = agent._convert_function_tool_call(mock_call) + + assert result is not None + assert result.name == "tool" + assert result.id == "call_456" + # Should wrap invalid JSON in raw_arguments + assert result.arguments == {"raw_arguments": "invalid json {{"} + + @pytest.mark.asyncio + async def test_convert_function_tool_call_empty_args(self, mock_mcp_client, mock_openai): + """Test converting function tool call with empty arguments.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + agent._tool_name_map = {"tool": "tool"} + + mock_call = MagicMock() + mock_call.call_id = "call_789" + mock_call.name = "tool" + mock_call.arguments = None + + result = agent._convert_function_tool_call(mock_call) + + assert result is not None + assert result.arguments == {} diff --git a/hud/agents/tests/test_operator.py b/hud/agents/tests/test_operator.py index 8b5b12a3e..6fddfceee 100644 --- a/hud/agents/tests/test_operator.py +++ b/hud/agents/tests/test_operator.py @@ -2,6 +2,7 @@ from __future__ import annotations +from typing import Any, cast from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -71,9 +72,13 @@ async def test_format_blocks(self, mock_mcp_client): ] messages = await agent.format_blocks(blocks) - assert len(messages) == 2 - assert messages[0] == {"type": "input_text", "text": "Hello, GPT!"} - assert messages[1] == {"type": "input_text", "text": "Another message"} + assert len(messages) == 1 + msg = cast(dict[str, Any], messages[0]) + assert msg["role"] == "user" + content = cast(list[dict[str, Any]], msg["content"]) + assert len(content) == 2 + assert content[0] == {"type": "input_text", "text": "Hello, GPT!"} + assert content[1] == {"type": "input_text", "text": "Another message"} # Test with mixed content blocks = [ @@ -82,9 +87,13 @@ async def test_format_blocks(self, mock_mcp_client): ] messages = await agent.format_blocks(blocks) - assert len(messages) == 2 - assert messages[0] == {"type": "input_text", "text": "Text content"} - assert messages[1] == { + assert len(messages) == 1 + msg = cast(dict[str, Any], messages[0]) + assert msg["role"] == "user" + content = cast(list[dict[str, Any]], msg["content"]) + assert len(content) == 2 + assert content[0] == {"type": "input_text", "text": "Text content"} + assert content[1] == { "type": "input_image", "image_url": "data:image/png;base64,base64data", } @@ -113,12 +122,22 @@ async def test_format_tool_results(self, mock_mcp_client, mock_openai): messages = await agent.format_tool_results(tool_calls, tool_results) - # OpenAI's format_tool_results returns input_image with screenshot - assert len(messages) == 1 - msg = messages[0] - assert msg.get("type") == "input_image" - assert "image_url" in msg - assert msg.get("image_url") == "data:image/png;base64,base64data" + # Should return both tool results as function_call_output + assert len(messages) == 2 + # First result is text + msg0 = cast(dict[str, Any], messages[0]) + assert msg0["type"] == "function_call_output" + assert msg0["call_id"] == "call_123" + output0 = cast(list[dict[str, Any]], msg0["output"]) + assert output0[0]["type"] == "input_text" + assert output0[0]["text"] == "Success" + # Second result is image + msg1 = cast(dict[str, Any], messages[1]) + assert msg1["type"] == "function_call_output" + assert msg1["call_id"] == "call_456" + output1 = cast(list[dict[str, Any]], msg1["output"]) + assert output1[0]["type"] == "input_image" + assert output1[0]["image_url"] == "data:image/png;base64,base64data" @pytest.mark.asyncio async def test_format_tool_results_with_error(self, mock_mcp_client, mock_openai): @@ -141,8 +160,16 @@ async def test_format_tool_results_with_error(self, mock_mcp_client, mock_openai messages = await agent.format_tool_results(tool_calls, tool_results) - # Since the result has isError=True and no screenshot, returns empty list - assert len(messages) == 0 + # Error results are returned with error flag and content + assert len(messages) == 1 + msg = cast(dict[str, Any], messages[0]) + assert msg["type"] == "function_call_output" + assert msg["call_id"] == "call_error" + output = cast(list[dict[str, Any]], msg["output"]) + assert output[0]["type"] == "input_text" + assert output[0]["text"] == "[tool_error] true" + assert output[1]["type"] == "input_text" + assert output[1]["text"] == "Something went wrong" @pytest.mark.asyncio async def test_get_model_response(self, mock_mcp_client, mock_openai): From efa396f1dc28b26f51770cd43688b8fd9d29309a Mon Sep 17 00:00:00 2001 From: Jaideep Date: Sun, 23 Nov 2025 22:58:18 -0800 Subject: [PATCH 6/7] ruff ruff --- hud/agents/tests/test_openai.py | 65 ++++++++++++++++++------------- hud/agents/tests/test_operator.py | 20 +++++----- 2 files changed, 49 insertions(+), 36 deletions(-) diff --git a/hud/agents/tests/test_openai.py b/hud/agents/tests/test_openai.py index f72d840d0..a85d9f1ba 100644 --- a/hud/agents/tests/test_openai.py +++ b/hud/agents/tests/test_openai.py @@ -16,7 +16,7 @@ from openai.types.responses.response_reasoning_item import Summary from hud.agents.openai import OpenAIAgent -from hud.types import AgentResponse, MCPToolCall, MCPToolResult +from hud.types import MCPToolCall, MCPToolResult class TestOpenAIAgent: @@ -111,9 +111,9 @@ async def test_format_blocks_text_only(self, mock_mcp_client, mock_openai): messages = await agent.format_blocks(blocks) assert len(messages) == 1 - msg = cast(dict[str, Any], messages[0]) + msg = cast("dict[str, Any]", messages[0]) assert msg["role"] == "user" - content = cast(list[dict[str, Any]], msg["content"]) + content = cast("list[dict[str, Any]]", msg["content"]) assert len(content) == 2 assert content[0] == {"type": "input_text", "text": "Hello, world!"} assert content[1] == {"type": "input_text", "text": "How are you?"} @@ -134,9 +134,9 @@ async def test_format_blocks_with_image(self, mock_mcp_client, mock_openai): messages = await agent.format_blocks(blocks) assert len(messages) == 1 - msg = cast(dict[str, Any], messages[0]) + msg = cast("dict[str, Any]", messages[0]) assert msg["role"] == "user" - content = cast(list[dict[str, Any]], msg["content"]) + content = cast("list[dict[str, Any]]", msg["content"]) assert len(content) == 2 assert content[0] == {"type": "input_text", "text": "Check this out:"} assert content[1] == { @@ -157,9 +157,9 @@ async def test_format_blocks_empty(self, mock_mcp_client, mock_openai): messages = await agent.format_blocks(blocks) assert len(messages) == 1 - msg = cast(dict[str, Any], messages[0]) + msg = cast("dict[str, Any]", messages[0]) assert msg["role"] == "user" - content = cast(list[dict[str, Any]], msg["content"]) + content = cast("list[dict[str, Any]]", msg["content"]) assert len(content) == 1 assert content[0] == {"type": "input_text", "text": ""} @@ -186,10 +186,10 @@ async def test_format_tool_results_text(self, mock_mcp_client, mock_openai): messages = await agent.format_tool_results(tool_calls, tool_results) assert len(messages) == 1 - msg = cast(dict[str, Any], messages[0]) + msg = cast("dict[str, Any]", messages[0]) assert msg["type"] == "function_call_output" assert msg["call_id"] == "call_123" - output = cast(list[dict[str, Any]], msg["output"]) + output = cast("list[dict[str, Any]]", msg["output"]) assert len(output) == 1 assert output[0]["type"] == "input_text" assert output[0]["text"] == "Tool executed successfully" @@ -209,7 +209,9 @@ async def test_format_tool_results_with_image(self, mock_mcp_client, mock_openai tool_results = [ MCPToolResult( - content=[types.ImageContent(type="image", data="screenshot_data", mimeType="image/png")], + content=[ + types.ImageContent(type="image", data="screenshot_data", mimeType="image/png") + ], isError=False, ), ] @@ -217,10 +219,10 @@ async def test_format_tool_results_with_image(self, mock_mcp_client, mock_openai messages = await agent.format_tool_results(tool_calls, tool_results) assert len(messages) == 1 - msg = cast(dict[str, Any], messages[0]) + msg = cast("dict[str, Any]", messages[0]) assert msg["type"] == "function_call_output" assert msg["call_id"] == "call_456" - output = cast(list[dict[str, Any]], msg["output"]) + output = cast("list[dict[str, Any]]", msg["output"]) assert len(output) == 1 assert output[0]["type"] == "input_image" assert output[0]["image_url"] == "data:image/png;base64,screenshot_data" @@ -248,10 +250,10 @@ async def test_format_tool_results_with_error(self, mock_mcp_client, mock_openai messages = await agent.format_tool_results(tool_calls, tool_results) assert len(messages) == 1 - msg = cast(dict[str, Any], messages[0]) + msg = cast("dict[str, Any]", messages[0]) assert msg["type"] == "function_call_output" assert msg["call_id"] == "call_error" - output = cast(list[dict[str, Any]], msg["output"]) + output = cast("list[dict[str, Any]]", msg["output"]) assert len(output) == 2 assert output[0]["type"] == "input_text" assert output[0]["text"] == "[tool_error] true" @@ -282,14 +284,15 @@ async def test_format_tool_results_with_structured_content(self, mock_mcp_client messages = await agent.format_tool_results(tool_calls, tool_results) assert len(messages) == 1 - msg = cast(dict[str, Any], messages[0]) + msg = cast("dict[str, Any]", messages[0]) assert msg["type"] == "function_call_output" assert msg["call_id"] == "call_789" - output = cast(list[dict[str, Any]], msg["output"]) + output = cast("list[dict[str, Any]]", msg["output"]) assert len(output) == 1 assert output[0]["type"] == "input_text" # Structured content is JSON serialized import json + parsed = json.loads(output[0]["text"]) assert parsed == {"key": "value", "number": 42} @@ -321,9 +324,9 @@ async def test_format_tool_results_multiple(self, mock_mcp_client, mock_openai): messages = await agent.format_tool_results(tool_calls, tool_results) assert len(messages) == 2 - msg0 = cast(dict[str, Any], messages[0]) + msg0 = cast("dict[str, Any]", messages[0]) assert msg0["call_id"] == "call_1" - msg1 = cast(dict[str, Any], messages[1]) + msg1 = cast("dict[str, Any]", messages[1]) assert msg1["call_id"] == "call_2" @pytest.mark.asyncio @@ -424,7 +427,9 @@ async def test_get_response_with_tool_call(self, mock_mcp_client, mock_openai): mock_openai.responses.create = AsyncMock(return_value=mock_response) - messages = [{"role": "user", "content": [{"type": "input_text", "text": "Do something"}]}] + messages = [ + {"role": "user", "content": [{"type": "input_text", "text": "Do something"}]} + ] response = await agent.get_response(messages) assert response.done is False @@ -478,7 +483,9 @@ async def test_get_response_with_reasoning(self, mock_mcp_client, mock_openai): mock_openai.responses.create = AsyncMock(return_value=mock_response) - messages = [{"role": "user", "content": [{"type": "input_text", "text": "Hard question"}]}] + messages = [ + {"role": "user", "content": [{"type": "input_text", "text": "Hard question"}]} + ] response = await agent.get_response(messages) assert "Thinking: Let me think about this..." in response.content @@ -508,7 +515,9 @@ async def test_get_response_empty_messages(self, mock_mcp_client, mock_openai): assert response.tool_calls == [] @pytest.mark.asyncio - async def test_get_response_no_new_messages_with_previous_id(self, mock_mcp_client, mock_openai): + async def test_get_response_no_new_messages_with_previous_id( + self, mock_mcp_client, mock_openai + ): """Test getting model response when no new messages and previous response exists.""" with patch("hud.settings.settings.telemetry_enabled", False): agent = OpenAIAgent( @@ -543,11 +552,13 @@ async def test_build_request_payload(self, mock_mcp_client, mock_openai): validate_api_key=False, ) - agent._openai_tools = [cast(Any, {"type": "function", "name": "test"})] + agent._openai_tools = [cast("Any", {"type": "function", "name": "test"})] agent.system_prompt = "You are a helpful assistant" agent.last_response_id = "prev_123" - new_items = cast(Any, [{"role": "user", "content": [{"type": "input_text", "text": "Hi"}]}]) + new_items = cast( + "Any", [{"role": "user", "content": [{"type": "input_text", "text": "Hi"}]}] + ) payload = agent._build_request_payload(new_items) assert payload["model"] == "gpt-4o" @@ -570,7 +581,9 @@ async def test_build_request_payload_minimal(self, mock_mcp_client, mock_openai) validate_api_key=False, ) - new_items = cast(Any, [{"role": "user", "content": [{"type": "input_text", "text": "Hi"}]}]) + new_items = cast( + "Any", [{"role": "user", "content": [{"type": "input_text", "text": "Hi"}]}] + ) payload = agent._build_request_payload(new_items) assert payload["model"] == "gpt-5.1" # default @@ -655,7 +668,7 @@ async def test_build_openai_tools(self, mock_mcp_client, mock_openai): assert len(agent._openai_tools) == 2 assert agent._tool_name_map == {"tool1": "tool1", "tool2": "tool2"} - tool1 = cast(dict[str, Any], agent._openai_tools[0]) + tool1 = cast("dict[str, Any]", agent._openai_tools[0]) assert tool1["type"] == "function" assert tool1["name"] == "tool1" assert tool1["description"] == "First tool" @@ -692,7 +705,7 @@ async def test_build_openai_tools_skips_incomplete(self, mock_mcp_client, mock_o # Should only have the complete tool assert len(agent._openai_tools) == 1 - tool = cast(dict[str, Any], agent._openai_tools[0]) + tool = cast("dict[str, Any]", agent._openai_tools[0]) assert tool["name"] == "complete" @pytest.mark.asyncio diff --git a/hud/agents/tests/test_operator.py b/hud/agents/tests/test_operator.py index 6fddfceee..e20287006 100644 --- a/hud/agents/tests/test_operator.py +++ b/hud/agents/tests/test_operator.py @@ -73,9 +73,9 @@ async def test_format_blocks(self, mock_mcp_client): messages = await agent.format_blocks(blocks) assert len(messages) == 1 - msg = cast(dict[str, Any], messages[0]) + msg = cast("dict[str, Any]", messages[0]) assert msg["role"] == "user" - content = cast(list[dict[str, Any]], msg["content"]) + content = cast("list[dict[str, Any]]", msg["content"]) assert len(content) == 2 assert content[0] == {"type": "input_text", "text": "Hello, GPT!"} assert content[1] == {"type": "input_text", "text": "Another message"} @@ -88,9 +88,9 @@ async def test_format_blocks(self, mock_mcp_client): messages = await agent.format_blocks(blocks) assert len(messages) == 1 - msg = cast(dict[str, Any], messages[0]) + msg = cast("dict[str, Any]", messages[0]) assert msg["role"] == "user" - content = cast(list[dict[str, Any]], msg["content"]) + content = cast("list[dict[str, Any]]", msg["content"]) assert len(content) == 2 assert content[0] == {"type": "input_text", "text": "Text content"} assert content[1] == { @@ -125,17 +125,17 @@ async def test_format_tool_results(self, mock_mcp_client, mock_openai): # Should return both tool results as function_call_output assert len(messages) == 2 # First result is text - msg0 = cast(dict[str, Any], messages[0]) + msg0 = cast("dict[str, Any]", messages[0]) assert msg0["type"] == "function_call_output" assert msg0["call_id"] == "call_123" - output0 = cast(list[dict[str, Any]], msg0["output"]) + output0 = cast("list[dict[str, Any]]", msg0["output"]) assert output0[0]["type"] == "input_text" assert output0[0]["text"] == "Success" # Second result is image - msg1 = cast(dict[str, Any], messages[1]) + msg1 = cast("dict[str, Any]", messages[1]) assert msg1["type"] == "function_call_output" assert msg1["call_id"] == "call_456" - output1 = cast(list[dict[str, Any]], msg1["output"]) + output1 = cast("list[dict[str, Any]]", msg1["output"]) assert output1[0]["type"] == "input_image" assert output1[0]["image_url"] == "data:image/png;base64,base64data" @@ -162,10 +162,10 @@ async def test_format_tool_results_with_error(self, mock_mcp_client, mock_openai # Error results are returned with error flag and content assert len(messages) == 1 - msg = cast(dict[str, Any], messages[0]) + msg = cast("dict[str, Any]", messages[0]) assert msg["type"] == "function_call_output" assert msg["call_id"] == "call_error" - output = cast(list[dict[str, Any]], msg["output"]) + output = cast("list[dict[str, Any]]", msg["output"]) assert output[0]["type"] == "input_text" assert output[0]["text"] == "[tool_error] true" assert output[1]["type"] == "input_text" From e6c4b52fe975a5956d79f896900b15424b1449f5 Mon Sep 17 00:00:00 2001 From: Jaideep Date: Mon, 24 Nov 2025 00:49:57 -0800 Subject: [PATCH 7/7] fix operator reasoning defaults --- hud/agents/operator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hud/agents/operator.py b/hud/agents/operator.py index 8e8ee455e..e24becbfa 100644 --- a/hud/agents/operator.py +++ b/hud/agents/operator.py @@ -109,6 +109,7 @@ def _build_openai_tools(self) -> None: def _build_request_payload(self, new_items: ResponseInputParam) -> dict[str, Any]: payload = super()._build_request_payload(new_items) payload["truncation"] = "auto" + payload["reasoning"] = {"summary": "auto"} return payload @hud.instrument(