diff --git a/docs/reference/agents.mdx b/docs/reference/agents.mdx index 58a97d835..1a1b09ada 100644 --- a/docs/reference/agents.mdx +++ b/docs/reference/agents.mdx @@ -126,13 +126,65 @@ result = await agent.run( ) ``` +### OpenAIAgent + +```python +from hud.agents import OpenAIAgent +``` + +Generic OpenAI agent using the Responses API for function calling and tool execution. + +**Constructor Parameters:** +| Parameter | Type | Description | Default | +|-----------|------|-------------|---------| +| `model_client` | `AsyncOpenAI` | OpenAI client | Auto-created | +| `model` | `str` | Responses model to use | `"gpt-5.1"` | +| `max_output_tokens` | `int \| None` | Maximum response tokens | `None` | +| `temperature` | `float \| None` | Sampling temperature | `None` | +| `reasoning` | `dict \| Literal["auto"] \| None` | Reasoning configuration | `None` | +| `tool_choice` | `dict \| Literal["auto"] \| None` | Tool selection strategy | `None` | +| `parallel_tool_calls` | `bool \| None` | Enable parallel tool execution | `None` | +| `validate_api_key` | `bool` | Validate key on init | `True` | + +**Features:** +- OpenAI Responses API integration +- Function calling with strict mode schemas +- Reasoning support for extended thinking +- Automatic tool schema conversion +- Response continuation with previous_response_id + +**Example:** +```python +agent = OpenAIAgent( + model="gpt-4o", + max_output_tokens=2048, + temperature=0.7, + reasoning="auto", +) + +result = await agent.run( + Task( + prompt="Analyze the website and extract key information", + mcp_config={ + "hud": { + "url": "https://mcp.hud.ai/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudpython/hud-remote-browser:latest" + } + } + } + ) +) +``` + ### OperatorAgent ```python from hud.agents import OperatorAgent ``` -OpenAI Operator-style agent built on the Responses API with computer-use. +OpenAI Operator-style agent built on the Responses API with computer-use. Extends `OpenAIAgent` with computer-use capabilities. **Constructor Parameters:** | Parameter | Type | Description | Default | @@ -146,11 +198,62 @@ OpenAI Operator-style agent built on the Responses API with computer-use. - OpenAI Responses computer-use - Operator-style system prompt guidance - Display metadata injection +- Inherits all OpenAIAgent features + +### GeminiAgent + +```python +from hud.agents import GeminiAgent +``` + +Google Gemini-specific implementation using Google's Generative AI API with computer-use capabilities. + +**Constructor Parameters:** +| Parameter | Type | Description | Default | +|-----------|------|-------------|---------| +| `model_client` | `genai.Client` | Gemini client | Auto-created | +| `model` | `str` | Gemini model to use | `"gemini-2.5-computer-use-preview-10-2025"` | +| `temperature` | `float` | Sampling temperature | `1.0` | +| `top_p` | `float` | Top-p sampling parameter | `0.95` | +| `top_k` | `int` | Top-k sampling parameter | `40` | +| `max_output_tokens` | `int` | Maximum response tokens | `8192` | +| `validate_api_key` | `bool` | Validate key on init | `True` | +| `excluded_predefined_functions` | `list[str] \| None` | Predefined functions to exclude | `None` | + +**Features:** +- Native Gemini computer-use capabilities +- Predefined computer-use functions (click, type, scroll, etc.) +- Display metadata injection +- MCP tool execution + +**Example:** +```python +agent = GeminiAgent( + model="gemini-2.5-computer-use-preview-10-2025", + temperature=0.7, + max_output_tokens=4096, +) + +result = await agent.run( + Task( + prompt="Navigate to the website and fill out the form", + mcp_config={ + "hud": { + "url": "https://mcp.hud.ai/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudpython/hud-remote-browser:latest" + } + } + } + ) +) +``` -### GenericOpenAIChatAgent +### OpenAIChatAgent ```python -from hud.agents import GenericOpenAIChatAgent +from hud.agents import OpenAIChatAgent ``` OpenAI-compatible chat.completions agent that works with any endpoint implementing the OpenAI schema (OpenAI, vLLM, Ollama, Together, custom, etc.). @@ -171,7 +274,7 @@ openai_client = AsyncOpenAI( api_key="not-needed", ) -agent = GenericOpenAIChatAgent( +agent = OpenAIChatAgent( openai_client=openai_client, model_name="llama3.1", completion_kwargs={"temperature": 0.2}, diff --git a/examples/openai_compatible_agent.py b/examples/openai_compatible_agent.py index 659f8fbc7..420f78d13 100644 --- a/examples/openai_compatible_agent.py +++ b/examples/openai_compatible_agent.py @@ -23,7 +23,7 @@ from openai import AsyncOpenAI import hud -from hud.agents.openai_chat_generic import GenericOpenAIChatAgent +from hud.agents.openai_chat import OpenAIChatAgent from hud.datasets import Task @@ -127,7 +127,7 @@ async def run_example(mode: Literal["text", "browser"], target: int) -> None: allowed_tools = ["computer"] if mode == "browser" else ["move"] # Create OpenAI-compatible agent - agent = GenericOpenAIChatAgent( + agent = OpenAIChatAgent( openai_client=openai_client, model_name=model_name, allowed_tools=allowed_tools, diff --git a/examples/run_evaluation.py b/examples/run_evaluation.py index fd58801da..eb5de4c60 100644 --- a/examples/run_evaluation.py +++ b/examples/run_evaluation.py @@ -59,13 +59,13 @@ def _build_agent( - agent_type: Literal[AgentType.CLAUDE, AgentType.OPENAI], + agent_type: Literal[AgentType.CLAUDE, AgentType.OPERATOR], *, model: str | None = None, allowed_tools: list[str] | None = None, ) -> ClaudeAgent | OperatorAgent: """Create and return the requested agent type.""" - if agent_type == AgentType.OPENAI: + if agent_type == AgentType.OPERATOR: return OperatorAgent(allowed_tools=allowed_tools, validate_api_key=False) model = model or "claude-sonnet-4-5-20250929" @@ -80,7 +80,7 @@ def _build_agent( async def run_single_task( dataset_name: str, *, - agent_type: Literal[AgentType.CLAUDE, AgentType.OPENAI] = AgentType.CLAUDE, + agent_type: Literal[AgentType.CLAUDE, AgentType.OPERATOR] = AgentType.CLAUDE, model: str | None = None, allowed_tools: list[str] | None = None, max_steps: int = 10, @@ -111,16 +111,18 @@ async def run_single_task( async def run_full_dataset( dataset_name: str, *, - agent_type: Literal[AgentType.CLAUDE, AgentType.OPENAI] = AgentType.CLAUDE, + agent_type: Literal[AgentType.CLAUDE, AgentType.OPERATOR] = AgentType.CLAUDE, model: str | None = None, allowed_tools: list[str] | None = None, max_concurrent: int = 50, max_steps: int = 10, ) -> list[Any]: """Run evaluation across entire dataset with asyncio concurrency.""" - if agent_type == AgentType.OPENAI: + if agent_type == AgentType.OPERATOR: agent_class = OperatorAgent - agent_config: dict[str, Any] = {"validate_api_key": False} + agent_config = {"validate_api_key": False} + if model: + agent_config["model"] = model if allowed_tools: # Only pass allowed tools if they are provided, otherwise all tools are enabled agent_config["allowed_tools"] = allowed_tools @@ -171,7 +173,7 @@ def parse_args() -> argparse.Namespace: # type: ignore[valid-type] parser.add_argument("--full", action="store_true", help="Run entire dataset") # Agent - parser.add_argument("--agent", choices=["claude", "openai"], default="claude") + parser.add_argument("--agent", choices=["claude", "operator"], default="claude") parser.add_argument("--model", default=None, help="Model override") parser.add_argument( "--allowed-tools", dest="allowed_tools", help="Tool allowlist (comma-separated)" diff --git a/hud/agents/__init__.py b/hud/agents/__init__.py index f69d4ed53..21b7a9929 100644 --- a/hud/agents/__init__.py +++ b/hud/agents/__init__.py @@ -3,13 +3,15 @@ from .base import MCPAgent from .claude import ClaudeAgent from .gemini import GeminiAgent -from .openai import OperatorAgent -from .openai_chat_generic import GenericOpenAIChatAgent +from .openai import OpenAIAgent +from .openai_chat import OpenAIChatAgent +from .operator import OperatorAgent __all__ = [ "ClaudeAgent", "GeminiAgent", - "GenericOpenAIChatAgent", "MCPAgent", + "OpenAIAgent", + "OpenAIChatAgent", "OperatorAgent", ] diff --git a/hud/agents/grounded_openai.py b/hud/agents/grounded_openai.py index 4ad2a0afe..d7c91ee38 100644 --- a/hud/agents/grounded_openai.py +++ b/hud/agents/grounded_openai.py @@ -9,10 +9,10 @@ from hud.tools.grounding import GroundedComputerTool, Grounder, GrounderConfig from hud.types import AgentResponse, MCPToolCall, MCPToolResult -from .openai_chat_generic import GenericOpenAIChatAgent +from .openai_chat import OpenAIChatAgent -class GroundedOpenAIChatAgent(GenericOpenAIChatAgent): +class GroundedOpenAIChatAgent(OpenAIChatAgent): """OpenAI agent that uses a separate grounding model for element detection. This agent: @@ -45,7 +45,7 @@ def __init__( openai_client: OpenAI client for the planning model model: Name of the OpenAI model to use for planning (e.g., "gpt-4o", "gpt-4o-mini") real_computer_tool_name: Name of the actual computer tool to execute - **kwargs: Additional arguments passed to GenericOpenAIChatAgent + **kwargs: Additional arguments passed to OpenAIChatAgent """ # Set defaults for grounded agent if allowed_tools is None: diff --git a/hud/agents/lite_llm.py b/hud/agents/lite_llm.py index 4445a6346..c16081737 100644 --- a/hud/agents/lite_llm.py +++ b/hud/agents/lite_llm.py @@ -11,7 +11,7 @@ import litellm -from .openai_chat_generic import GenericOpenAIChatAgent +from .openai_chat import OpenAIChatAgent logger = logging.getLogger(__name__) @@ -24,7 +24,7 @@ transform_mcp_tool_to_openai_tool = None # type: ignore -class LiteAgent(GenericOpenAIChatAgent): +class LiteAgent(OpenAIChatAgent): """ Same OpenAI chat-completions shape + MCP tool plumbing, but transport is LiteLLM and (optionally) tools are shaped by LiteLLM's MCP transformer. @@ -55,7 +55,7 @@ def get_tool_schemas(self) -> list[Any]: for t in self.get_available_tools() ] # Fallback to the generic OpenAI sanitizer - return GenericOpenAIChatAgent.get_tool_schemas(self) + return OpenAIChatAgent.get_tool_schemas(self) async def _invoke_chat_completion( self, diff --git a/hud/agents/openai.py b/hud/agents/openai.py index 7d09c753e..e02658f80 100644 --- a/hud/agents/openai.py +++ b/hud/agents/openai.py @@ -2,355 +2,323 @@ from __future__ import annotations +import copy +import json import logging -from typing import Any, ClassVar, Literal +from typing import Any, Literal, cast import mcp.types as types from openai import AsyncOpenAI, OpenAI from openai.types.responses import ( - ResponseComputerToolCall, + ResponseFunctionToolCall, + ResponseInputImageParam, ResponseInputMessageContentListParam, ResponseInputParam, + ResponseInputTextParam, ResponseOutputMessage, ResponseOutputText, + ResponseReasoningItem, ToolParam, ) +from openai.types.responses.response_input_param import ( + FunctionCallOutput, # noqa: TC002 + Message, # noqa: TC002 +) import hud from hud.settings import settings -from hud.tools.computer.settings import computer_settings from hud.types import AgentResponse, MCPToolCall, MCPToolResult, Trace +from hud.utils.strict_schema import ensure_strict_json_schema from .base import MCPAgent logger = logging.getLogger(__name__) -class OperatorAgent(MCPAgent): - """ - Operator agent that uses MCP servers for tool execution. - - This agent uses OpenAI's Computer Use API format but executes - tools through MCP servers instead of direct implementation. - """ +class OpenAIAgent(MCPAgent): + """Generic OpenAI agent that can execute MCP tools through the Responses API.""" - metadata: ClassVar[dict[str, Any]] = { - "display_width": computer_settings.OPENAI_COMPUTER_WIDTH, - "display_height": computer_settings.OPENAI_COMPUTER_HEIGHT, - } - required_tools: ClassVar[list[str]] = ["openai_computer"] + metadata: dict[str, Any] | None = None def __init__( self, model_client: AsyncOpenAI | None = None, - model: str = "computer-use-preview", - environment: Literal["windows", "mac", "linux", "browser"] = "linux", + model: str = "gpt-5.1", + max_output_tokens: int | None = None, + temperature: float | None = None, + reasoning: dict[str, Any] | Literal["auto"] | None = None, + tool_choice: dict[str, Any] | Literal["auto"] | None = None, + parallel_tool_calls: bool | None = None, validate_api_key: bool = True, **kwargs: Any, ) -> None: - """ - Initialize Operator MCP agent. - - Args: - client: AsyncOpenAI client (created if not provided) - model: OpenAI model to use - environment: Environment type for computer use - display_width: Display width for computer use - display_height: Display height for computer use - **kwargs: Additional arguments passed to MCPAgent - """ super().__init__(**kwargs) - # Initialize client if not provided if model_client is None: api_key = settings.openai_api_key if not api_key: raise ValueError("OpenAI API key not found. Set OPENAI_API_KEY.") model_client = AsyncOpenAI(api_key=api_key) + if validate_api_key: + try: + OpenAI(api_key=model_client.api_key).models.list() + except Exception as exc: # pragma: no cover - network validation + raise ValueError(f"OpenAI API key is invalid: {exc}") from exc + self.openai_client = model_client self.model = model - self.checkpoint_name = self.model - self.environment = environment + self.max_output_tokens = max_output_tokens + self.temperature = temperature + self.reasoning = reasoning + self.tool_choice = tool_choice + self.parallel_tool_calls = parallel_tool_calls + + self._openai_tools: list[ToolParam] = [] + self._tool_name_map: dict[str, str] = {} - # State tracking for OpenAI's stateful API self.last_response_id: str | None = None self.pending_call_id: str | None = None self.pending_safety_checks: list[Any] = [] + self._message_cursor = 0 - # validate api key if requested - if validate_api_key: + self.model_name = "OpenAI" + self.checkpoint_name = self.model + + async def initialize(self, task: Any | None = None) -> None: + """Initialize agent and build tool metadata.""" + await super().initialize(task) + self._build_openai_tools() + + def _build_openai_tools(self) -> None: + """Convert MCP tools into OpenAI Responses tool definitions.""" + self._openai_tools = [] + self._tool_name_map = {} + + for tool in self.get_available_tools(): + if tool.description is None or tool.inputSchema is None: + self.console.warning_log( + f"Skipping tool '{tool.name}' - description and input schema " + "are required for OpenAI tools." + ) + continue + self._tool_name_map[tool.name] = tool.name + schema_copy = copy.deepcopy(tool.inputSchema) + strict_schema = schema_copy + strict_enforced = True try: - OpenAI(api_key=self.openai_client.api_key).models.list() - except Exception as e: - raise ValueError(f"OpenAI API key is invalid: {e}") from e - - self.model_name = "Operator" - - # Append OpenAI-specific instructions to the base system prompt - openai_instructions = """ - You are an autonomous computer-using agent. Follow these guidelines: - - 1. NEVER ask for confirmation. Complete all tasks autonomously. - 2. Do NOT send messages like "I need to confirm before..." or "Do you want me to continue?" - just proceed. - 3. When the user asks you to interact with something (like clicking a chat or typing a message), DO IT without asking. - 4. Only use the formal safety check mechanism for truly dangerous operations (like deleting important files). - 5. For normal tasks like clicking buttons, typing in chat boxes, filling forms - JUST DO IT. - 6. The user has already given you permission by running this agent. No further confirmation is needed. - 7. Be decisive and action-oriented. Complete the requested task fully. - - Remember: You are expected to complete tasks autonomously. The user trusts you to do what they asked. - """.strip() # noqa: E501 - - # Append OpenAI instructions to any base system prompt - if self.system_prompt: - self.system_prompt = f"{self.system_prompt}\n\n{openai_instructions}" - else: - self.system_prompt = openai_instructions - - async def _run_context(self, context: list[types.ContentBlock], max_steps: int = 10) -> Trace: - """ - Run the agent with the given prompt or task. - - Override to reset OpenAI-specific state. - """ - # Reset state for new run - self.last_response_id = None - self.pending_call_id = None - self.pending_safety_checks = [] + strict_schema = ensure_strict_json_schema(schema_copy) + except Exception as exc: # pragma: no cover - defensive + strict_enforced = False + self.console.warning_log( + f"Failed to convert schema for tool '{tool.name}' to strict mode: {exc}" + ) - # Use base implementation + function_tool = cast( + "ToolParam", + { + "type": "function", + "name": tool.name, + "description": tool.description, + "parameters": strict_schema, + "strict": strict_enforced, + }, + ) + self._openai_tools.append(function_tool) + + async def _run_context( + self, context: list[types.ContentBlock], *, max_steps: int = 10 + ) -> Trace: + """Reset internal state before delegating to the base loop.""" + self._reset_response_state() return await super()._run_context(context, max_steps=max_steps) - async def get_system_messages(self) -> list[Any]: - """ - Create initial messages for OpenAI. + def _reset_response_state(self) -> None: + self.last_response_id = None + self.pending_call_id = None + self.pending_safety_checks = [] + self._message_cursor = 0 - OpenAI uses a different message format - we'll store the prompt - and screenshot for use in get_model_response. - """ + async def get_system_messages(self) -> list[types.ContentBlock]: + """System messages are provided via the `instructions` field.""" return [] - async def format_blocks( - self, blocks: list[types.ContentBlock] - ) -> ResponseInputMessageContentListParam: - """ - Format blocks for OpenAI input format. - - Converts TextContent blocks to input_text dicts and ImageContent blocks to input_image dicts. - """ # noqa: E501 - formatted = [] + async def format_blocks(self, blocks: list[types.ContentBlock]) -> ResponseInputParam: + """Convert MCP content blocks into OpenAI user messages.""" + content: ResponseInputMessageContentListParam = [] for block in blocks: if isinstance(block, types.TextContent): - formatted.append({"type": "input_text", "text": block.text}) + content.append( + cast( + "ResponseInputTextParam", + {"type": "input_text", "text": block.text}, + ) + ) elif isinstance(block, types.ImageContent): mime_type = getattr(block, "mimeType", "image/png") - formatted.append( - {"type": "input_image", "image_url": f"data:{mime_type};base64,{block.data}"} + content.append( + cast( + "ResponseInputImageParam", + { + "type": "input_image", + "image_url": f"data:{mime_type};base64,{block.data}", + }, + ) ) - return formatted + if not content: + content.append(cast("ResponseInputTextParam", {"type": "input_text", "text": ""})) + return [cast("Message", {"role": "user", "content": content})] @hud.instrument( span_type="agent", - record_args=False, # Messages can be large + record_args=False, record_result=True, ) - async def get_response(self, messages: ResponseInputMessageContentListParam) -> AgentResponse: - """Get response from OpenAI including any tool calls.""" - # OpenAI's API is stateful, so we handle messages differently - - # Get the computer tool (guaranteed to exist due to required_tools) - computer_tool_name = "openai_computer" - - # Define the computer use tool - computer_tool: ToolParam = { # type: ignore[reportAssignmentType] - "type": "computer_use_preview", - "display_width": self.metadata["display_width"], - "display_height": self.metadata["display_height"], - "environment": self.environment, - } - - # Build the request based on whether this is first step or follow-up - if self.pending_call_id is None and self.last_response_id is None: - # First step - messages are already formatted dicts from format_blocks - # format_blocks returns type ResponseInputMessageContentListParam, which is a list of dicts # noqa: E501 - input_content: ResponseInputMessageContentListParam = [] - - input_content.extend(messages) - - # If no content was added, add empty text to avoid empty request - if not input_content: - input_content.append({"type": "input_text", "text": ""}) - - input_param: ResponseInputParam = [{"role": "user", "content": input_content}] # type: ignore[reportUnknownMemberType] - - response = await self.openai_client.responses.create( - model=self.model, - tools=[computer_tool], - input=input_param, - instructions=self.system_prompt, - truncation="auto", - reasoning={"summary": "auto"}, # type: ignore[arg-type] - ) - else: - # Follow-up step - check if this is user input or tool result - latest_message = messages[-1] if messages else {} - - if latest_message.get("type") == "input_text": - # User provided input in conversation mode - user_text = latest_message.get("text", "") - input_param_followup: ResponseInputParam = [ # type: ignore[reportAssignmentType] - {"role": "user", "content": [{"type": "input_text", "text": user_text}]} - ] - # Reset pending_call_id since this is user input, not a tool response - self.pending_call_id = None + async def get_response(self, messages: ResponseInputParam) -> AgentResponse: + """Send the latest input items to OpenAI's Responses API.""" + new_items = cast("ResponseInputParam", messages[self._message_cursor :]) + if not new_items: + if self.last_response_id is None: + new_items = cast( + "ResponseInputParam", + [ + cast( + "Message", + { + "role": "user", + "content": [ + cast( + "ResponseInputTextParam", + {"type": "input_text", "text": ""}, + ) + ], + }, + ) + ], + ) else: - # Tool result - need screenshot from processed results - latest_screenshot = None - for msg in reversed(messages): - if isinstance(msg, dict) and "image_url" in msg: - latest_screenshot = msg["image_url"] # type: ignore - break - - if not latest_screenshot: - self.console.warning_log("No screenshot provided for response to action") - return AgentResponse( - content="No screenshot available for next action", - tool_calls=[], - done=True, - ) + self.console.debug("No new messages to send to OpenAI.") + return AgentResponse(content="", tool_calls=[], done=True) - # Create response to previous action - input_param_followup: ResponseInputParam = [ # type: ignore[reportAssignmentType] - { # type: ignore[reportAssignmentType] - "call_id": self.pending_call_id, - "type": "computer_call_output", - "output": { - "type": "input_image", - "image_url": latest_screenshot, - }, - "acknowledged_safety_checks": self.pending_safety_checks, - } - ] - - self.pending_safety_checks = [] - - response = await self.openai_client.responses.create( - model=self.model, - previous_response_id=self.last_response_id, - tools=[computer_tool], - input=input_param_followup, - instructions=self.system_prompt, - truncation="auto", - reasoning={"summary": "auto"}, # type: ignore[arg-type] - ) + payload = self._build_request_payload(new_items) + response = await self.openai_client.responses.create(**payload) - # Store response ID for next call self.last_response_id = response.id - - # Process response - result = AgentResponse( - content="", - tool_calls=[], - done=False, # Will be set to True only if no tool calls - ) - + self._message_cursor = len(messages) self.pending_call_id = None - # Check for computer calls - computer_calls = [ - item - for item in response.output - if isinstance(item, ResponseComputerToolCall) and item.type == "computer_call" - ] - - if computer_calls: - # Process computer calls - result.done = False - for computer_call in computer_calls: - self.pending_call_id = computer_call.call_id - self.pending_safety_checks = computer_call.pending_safety_checks - - # Convert OpenAI action to MCP tool call - action = computer_call.action.model_dump() - - # Create MCPToolCall object with OpenAI metadata as extra fields - # Pyright will complain but the tool class accepts extra fields - tool_call = MCPToolCall( - name=computer_tool_name, - arguments=action, - id=computer_call.call_id, # type: ignore - pending_safety_checks=computer_call.pending_safety_checks, # type: ignore - ) - result.tool_calls.append(tool_call) - else: - # No computer calls, check for text response - for item in response.output: - if isinstance(item, ResponseOutputMessage) and item.type == "message": - # Extract text from content blocks - text_parts = [ - content.text - for content in item.content - if isinstance(content, ResponseOutputText) - ] - if text_parts: - result.content = "".join(text_parts) - break - - # Extract reasoning if present - reasoning_text = "" - for item in response.output: - if item.type == "reasoning" and hasattr(item, "summary") and item.summary: - reasoning_text += f"Thinking: {item.summary[0].text}\n" - - if reasoning_text: - result.content = reasoning_text + result.content if result.content else reasoning_text + agent_response = AgentResponse(content="", tool_calls=[], done=True) + text_chunks: list[str] = [] + reasoning_chunks: list[str] = [] - # Set done=True if no tool calls (task complete or waiting for user) - if not result.tool_calls: - result.done = True + for item in response.output: + if isinstance(item, ResponseOutputMessage) and item.type == "message": + text = "".join( + content.text + for content in item.content + if isinstance(content, ResponseOutputText) + ) + if text: + text_chunks.append(text) + elif isinstance(item, ResponseFunctionToolCall): + tool_call = self._convert_function_tool_call(item) + if tool_call: + agent_response.tool_calls.append(tool_call) + elif isinstance(item, ResponseReasoningItem) and item.summary: + reasoning_chunks.append( + "".join(f"Thinking: {summary.text}\n" for summary in item.summary) + ) - return result + if agent_response.tool_calls: + agent_response.done = False + + agent_response.content = "".join(reasoning_chunks) + "".join(text_chunks) + return agent_response + + def _build_request_payload(self, new_items: ResponseInputParam) -> dict[str, Any]: + payload: dict[str, Any] = { + "model": self.model, + "input": new_items, + "instructions": self.system_prompt, + "max_output_tokens": self.max_output_tokens, + "temperature": self.temperature, + "tool_choice": self.tool_choice, + "parallel_tool_calls": self.parallel_tool_calls, + } + if self.reasoning is not None: + payload["reasoning"] = self.reasoning + if self._openai_tools: + payload["tools"] = self._openai_tools + if self.last_response_id is not None: + payload["previous_response_id"] = self.last_response_id + return {k: v for k, v in payload.items() if v is not None} + + def _convert_function_tool_call( + self, tool_call: ResponseFunctionToolCall + ) -> MCPToolCall | None: + target_name = self._tool_name_map.get(tool_call.name, tool_call.name) + try: + arguments = json.loads(tool_call.arguments) if tool_call.arguments else {} + except json.JSONDecodeError: + self.console.warning_log( + f"Failed to parse arguments for tool '{tool_call.name}', passing raw string." + ) + arguments = {"raw_arguments": tool_call.arguments} + return MCPToolCall(name=target_name, arguments=arguments, id=tool_call.call_id) async def format_tool_results( self, tool_calls: list[MCPToolCall], tool_results: list[MCPToolResult] - ) -> ResponseInputMessageContentListParam: - """ - Format tool results for OpenAI's stateful API. - - Tool result content is a list of ContentBlock objects. - We need to extract the latest screenshot from the tool results. - - This assumes that you only care about computer tool results for your agent loop. - If you need to add other content, you can do so by adding a new ContentBlock object to the list. + ) -> ResponseInputParam: + """Convert MCP tool outputs into Responses input items.""" + formatted: ResponseInputParam = [] + for call, result in zip(tool_calls, tool_results, strict=False): + if not call.id: + self.console.warning_log(f"Tool '{call.name}' missing call_id; skipping output.") + continue + + output_items: list[dict[str, Any]] = [] + if result.isError: + output_items.append({"type": "input_text", "text": "[tool_error] true"}) - Returns formatted dicts with tool result data, preserving screenshots. - """ # noqa: E501 - formatted_results = [] - latest_screenshot = None + if result.structuredContent is not None: + output_items.append( + { + "type": "input_text", + "text": json.dumps(result.structuredContent, default=str), + } + ) - # Extract all content from tool results - for result in tool_results: - if result.isError: - # If it's an error, the error details are in the content - for content in result.content: - if isinstance(content, types.TextContent): - # Don't add error text as input_text, just track it - self.console.error_log(f"Tool error: {content.text}") - elif isinstance(content, types.ImageContent): - # Even error results might have images - latest_screenshot = content.data - else: - # Extract content from successful results - for content in result.content: - if isinstance(content, types.ImageContent): - latest_screenshot = content.data - break - - # Return a dict with the latest screenshot for the follow-up step - if latest_screenshot: - formatted_results.append( - {"type": "input_image", "image_url": f"data:image/png;base64,{latest_screenshot}"} + if result.content: + for block in result.content: + if isinstance(block, types.TextContent): + output_items.append({"type": "input_text", "text": block.text}) + elif isinstance(block, types.ImageContent): + mime_type = getattr(block, "mimeType", "image/png") + output_items.append( + { + "type": "input_image", + "image_url": f"data:{mime_type};base64,{block.data}", + } + ) + else: + output_items.append( + { + "type": "input_text", + "text": getattr(block, "text", str(block)), + } + ) + + if not output_items: + output_items.append({"type": "input_text", "text": ""}) + + formatted.append( + cast( + "FunctionCallOutput", + { + "type": "function_call_output", + "call_id": call.id, + "output": output_items, + }, + ) ) - - return formatted_results + return formatted diff --git a/hud/agents/openai_chat_generic.py b/hud/agents/openai_chat.py similarity index 98% rename from hud/agents/openai_chat_generic.py rename to hud/agents/openai_chat.py index 934886a53..a3493a1d1 100644 --- a/hud/agents/openai_chat_generic.py +++ b/hud/agents/openai_chat.py @@ -1,4 +1,4 @@ -"""Generic OpenAI chat-completions agent. +"""OpenAI Chat Completions Agent. This class provides the minimal glue required to connect any endpoint that implements the OpenAI compatible *chat.completions* API with MCP tool calling @@ -34,7 +34,7 @@ logger = logging.getLogger(__name__) -class GenericOpenAIChatAgent(MCPAgent): +class OpenAIChatAgent(MCPAgent): """MCP-enabled agent that speaks the OpenAI *chat.completions* protocol.""" metadata: ClassVar[dict[str, Any]] = {} @@ -62,7 +62,7 @@ def __init__( else: raise ValueError("Either openai_client or (api_key and base_url) must be provided") - self.model_name = "GenericOpenAI" + self.model_name = "OpenAI" self.checkpoint_name = model_name self.completion_kwargs: dict[str, Any] = completion_kwargs or {} self.mcp_schemas = [] @@ -195,7 +195,7 @@ async def _invoke_chat_completion( extra: dict[str, Any], ) -> Any: if self.oai is None: - raise ValueError("openai_client is required for GenericOpenAIChatAgent") + raise ValueError("openai_client is required for OpenAIChatAgent") # default transport = OpenAI SDK return await self.oai.chat.completions.create( model=self.checkpoint_name, diff --git a/hud/agents/operator.py b/hud/agents/operator.py new file mode 100644 index 000000000..e24becbfa --- /dev/null +++ b/hud/agents/operator.py @@ -0,0 +1,264 @@ +"""Operator agent built on top of OpenAIAgent.""" + +from __future__ import annotations + +from typing import Any, ClassVar, Literal, cast + +import mcp.types as types +from openai import AsyncOpenAI # noqa: TC002 +from openai.types.responses import ( + ResponseComputerToolCall, + ResponseFunctionToolCall, + ResponseInputParam, + ResponseInputTextParam, + ResponseOutputMessage, + ResponseOutputText, + ResponseReasoningItem, + ToolParam, +) +from openai.types.responses.response_input_param import ( + ComputerCallOutput, # noqa: TC002 + Message, # noqa: TC002 +) + +import hud +from hud.tools.computer.settings import computer_settings +from hud.types import AgentResponse, MCPToolCall, MCPToolResult + +from .openai import OpenAIAgent + +OPERATOR_INSTRUCTIONS = """ +You are an autonomous computer-using agent. Follow these guidelines: + +1. NEVER ask for confirmation. Complete all tasks autonomously. +2. Do NOT send messages like "I need to confirm before..." or "Do you want me to + continue?" - just proceed. +3. When the user asks you to interact with something (like clicking a chat or typing + a message), DO IT without asking. +4. Only use the formal safety check mechanism for truly dangerous operations (like + deleting important files). +5. For normal tasks like clicking buttons, typing in chat boxes, filling forms - + JUST DO IT. +6. The user has already given you permission by running this agent. No further + confirmation is needed. +7. Be decisive and action-oriented. Complete the requested task fully. + +Remember: You are expected to complete tasks autonomously. The user trusts you to do +what they asked. +""".strip() + + +class OperatorAgent(OpenAIAgent): + """ + Backwards-compatible Operator agent built on top of OpenAIAgent. + """ + + metadata: ClassVar[dict[str, Any] | None] = { + "display_width": computer_settings.OPENAI_COMPUTER_WIDTH, + "display_height": computer_settings.OPENAI_COMPUTER_HEIGHT, + } + required_tools: ClassVar[list[str]] = ["openai_computer"] + + def __init__( + self, + model_client: AsyncOpenAI | None = None, + model: str = "computer-use-preview", + environment: Literal["windows", "mac", "linux", "browser"] = "linux", + validate_api_key: bool = True, + **kwargs: Any, + ) -> None: + super().__init__( + model_client=model_client, + model=model, + validate_api_key=validate_api_key, + **kwargs, + ) + self._operator_computer_tool_name = "openai_computer" + self._operator_display_width = computer_settings.OPENAI_COMPUTER_WIDTH + self._operator_display_height = computer_settings.OPENAI_COMPUTER_HEIGHT + self._operator_environment = environment + self.model_name = "Operator" + self.environment = environment + + if self.system_prompt: + self.system_prompt = f"{self.system_prompt}\n\n{OPERATOR_INSTRUCTIONS}" + else: + self.system_prompt = OPERATOR_INSTRUCTIONS + + def _build_openai_tools(self) -> None: + super()._build_openai_tools() + if not any( + tool.name == self._operator_computer_tool_name for tool in self.get_available_tools() + ): + raise ValueError( + f"MCP computer tool '{self._operator_computer_tool_name}' is required " + "but not available." + ) + self._openai_tools.append( + cast( + "ToolParam", + { + "type": "computer_use_preview", + "display_width": self._operator_display_width, + "display_height": self._operator_display_height, + "environment": self._operator_environment, + }, + ) + ) + + def _build_request_payload(self, new_items: ResponseInputParam) -> dict[str, Any]: + payload = super()._build_request_payload(new_items) + payload["truncation"] = "auto" + payload["reasoning"] = {"summary": "auto"} + return payload + + @hud.instrument( + span_type="agent", + record_args=False, + record_result=True, + ) + async def get_response(self, messages: ResponseInputParam) -> AgentResponse: + new_items = cast("ResponseInputParam", messages[self._message_cursor :]) + if not new_items: + if self.last_response_id is None: + new_items = cast( + "ResponseInputParam", + [ + cast( + "Message", + { + "role": "user", + "content": [ + cast( + "ResponseInputTextParam", + {"type": "input_text", "text": ""}, + ) + ], + }, + ) + ], + ) + else: + self.console.debug("No new messages to send to OpenAI.") + return AgentResponse(content="", tool_calls=[], done=True) + + payload = self._build_request_payload(new_items) + response = await self.openai_client.responses.create(**payload) + + self.last_response_id = response.id + self._message_cursor = len(messages) + self.pending_call_id = None + + agent_response = AgentResponse(content="", tool_calls=[], done=True) + text_chunks: list[str] = [] + reasoning_chunks: list[str] = [] + + for item in response.output: + if isinstance(item, ResponseComputerToolCall): + tool_call = self._convert_computer_tool_call(item) + if tool_call: + agent_response.tool_calls.append(tool_call) + elif isinstance(item, ResponseFunctionToolCall): + tool_call = self._convert_function_tool_call(item) + if tool_call: + agent_response.tool_calls.append(tool_call) + elif isinstance(item, ResponseOutputMessage) and item.type == "message": + text = "".join( + content.text + for content in item.content + if isinstance(content, ResponseOutputText) + ) + if text: + text_chunks.append(text) + elif isinstance(item, ResponseReasoningItem) and item.summary: + reasoning_chunks.append( + "".join(f"Thinking: {summary.text}\n" for summary in item.summary) + ) + + if agent_response.tool_calls: + agent_response.done = False + + agent_response.content = "".join(reasoning_chunks) + "".join(text_chunks) + return agent_response + + async def format_tool_results( + self, tool_calls: list[MCPToolCall], tool_results: list[MCPToolResult] + ) -> ResponseInputParam: + remaining_calls: list[MCPToolCall] = [] + remaining_results: list[MCPToolResult] = [] + computer_outputs: ResponseInputParam = [] + ordering: list[tuple[str, int]] = [] + + for call, result in zip(tool_calls, tool_results, strict=False): + if call.name == self._operator_computer_tool_name: + screenshot = self._extract_latest_screenshot(result) + if not screenshot: + self.console.warning_log( + "Computer tool result missing screenshot; skipping output." + ) + continue + call_id = call.id or self.pending_call_id + if not call_id: + self.console.warning_log("Computer tool call missing ID; skipping output.") + continue + acknowledged_checks = [] + for check in self.pending_safety_checks: + if hasattr(check, "model_dump"): + acknowledged_checks.append(check.model_dump()) + elif isinstance(check, dict): + acknowledged_checks.append(check) + output_payload: dict[str, Any] = { + "type": "computer_call_output", + "call_id": call_id, + "output": { + "type": "input_image", + "image_url": f"data:image/png;base64,{screenshot}", + }, + } + if acknowledged_checks: + output_payload["acknowledged_safety_checks"] = acknowledged_checks + computer_outputs.append(cast("ComputerCallOutput", output_payload)) + self.pending_call_id = None + self.pending_safety_checks = [] + ordering.append(("computer", len(computer_outputs) - 1)) + else: + remaining_calls.append(call) + remaining_results.append(result) + ordering.append(("function", len(remaining_calls) - 1)) + + formatted: ResponseInputParam = [] + function_outputs: ResponseInputParam = [] + if remaining_calls: + function_outputs = await super().format_tool_results(remaining_calls, remaining_results) + + for kind, idx in ordering: + if kind == "computer": + if idx < len(computer_outputs): + formatted.append(computer_outputs[idx]) + else: + if idx < len(function_outputs): + formatted.append(function_outputs[idx]) + return formatted + + def _extract_latest_screenshot(self, result: MCPToolResult) -> str | None: + if not result.content: + return None + for content in reversed(result.content): + if isinstance(content, types.ImageContent): + return content.data + if isinstance(content, types.TextContent) and result.isError: + self.console.error_log(f"Computer tool error: {content.text}") + return None + + def _convert_computer_tool_call( + self, tool_call: ResponseComputerToolCall + ) -> MCPToolCall | None: + self.pending_call_id = tool_call.call_id + self.pending_safety_checks = tool_call.pending_safety_checks + call = MCPToolCall( + name=self._operator_computer_tool_name, + arguments=tool_call.action.model_dump(), + id=tool_call.call_id, + ) + call.pending_safety_checks = tool_call.pending_safety_checks # type: ignore[attr-defined] + return call diff --git a/hud/agents/tests/test_openai.py b/hud/agents/tests/test_openai.py index 7c4b24974..a85d9f1ba 100644 --- a/hud/agents/tests/test_openai.py +++ b/hud/agents/tests/test_openai.py @@ -2,29 +2,37 @@ from __future__ import annotations +from typing import Any, cast from unittest.mock import AsyncMock, MagicMock, patch import pytest from mcp import types - -from hud.agents.openai import OperatorAgent +from openai.types.responses import ( + ResponseFunctionToolCall, + ResponseOutputMessage, + ResponseOutputText, + ResponseReasoningItem, +) +from openai.types.responses.response_reasoning_item import Summary + +from hud.agents.openai import OpenAIAgent from hud.types import MCPToolCall, MCPToolResult -class TestOperatorAgent: - """Test OperatorAgent class.""" +class TestOpenAIAgent: + """Test OpenAIAgent class.""" @pytest.fixture def mock_mcp_client(self): """Create a mock MCP client.""" mcp_client = AsyncMock() - # Set up the mcp_config attribute as a regular dict, not a coroutine mcp_client.mcp_config = {"test_server": {"url": "http://test"}} - # Mock list_tools to return the required openai_computer tool mcp_client.list_tools = AsyncMock( return_value=[ types.Tool( - name="openai_computer", description="OpenAI computer use tool", inputSchema={} + name="test_tool", + description="A test tool", + inputSchema={"type": "object", "properties": {}}, ) ] ) @@ -40,92 +48,192 @@ def mock_openai(self): yield client @pytest.mark.asyncio - async def test_init(self, mock_mcp_client): - """Test agent initialization.""" + async def test_init_with_client(self, mock_mcp_client): + """Test agent initialization with provided client.""" mock_model_client = MagicMock() - agent = OperatorAgent( + agent = OpenAIAgent( mcp_client=mock_mcp_client, model_client=mock_model_client, - model="gpt-4", - validate_api_key=False, # Skip validation in tests + model="gpt-4o", + validate_api_key=False, ) - assert agent.model_name == "Operator" - assert agent.model == "gpt-4" + assert agent.model_name == "OpenAI" + assert agent.model == "gpt-4o" + assert agent.checkpoint_name == "gpt-4o" assert agent.openai_client == mock_model_client + assert agent.max_output_tokens is None + assert agent.temperature is None @pytest.mark.asyncio - async def test_format_blocks(self, mock_mcp_client): - """Test formatting content blocks.""" + async def test_init_with_parameters(self, mock_mcp_client): + """Test agent initialization with various parameters.""" mock_model_client = MagicMock() - agent = OperatorAgent( + agent = OpenAIAgent( mcp_client=mock_mcp_client, model_client=mock_model_client, - validate_api_key=False, # Skip validation in tests + model="gpt-4o", + max_output_tokens=2048, + temperature=0.7, + reasoning="auto", + tool_choice="auto", + parallel_tool_calls=True, + validate_api_key=False, + ) + + assert agent.max_output_tokens == 2048 + assert agent.temperature == 0.7 + assert agent.reasoning == "auto" + assert agent.tool_choice == "auto" + assert agent.parallel_tool_calls is True + + @pytest.mark.asyncio + async def test_init_without_client_no_api_key(self, mock_mcp_client): + """Test agent initialization fails without API key.""" + with patch("hud.agents.openai.settings") as mock_settings: + mock_settings.openai_api_key = None + with pytest.raises(ValueError, match="OpenAI API key not found"): + OpenAIAgent(mcp_client=mock_mcp_client) + + @pytest.mark.asyncio + async def test_format_blocks_text_only(self, mock_mcp_client, mock_openai): + """Test formatting text content blocks.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, ) - # Test with text blocks blocks: list[types.ContentBlock] = [ - types.TextContent(type="text", text="Hello, GPT!"), - types.TextContent(type="text", text="Another message"), + types.TextContent(type="text", text="Hello, world!"), + types.TextContent(type="text", text="How are you?"), ] messages = await agent.format_blocks(blocks) - assert len(messages) == 2 - assert messages[0] == {"type": "input_text", "text": "Hello, GPT!"} - assert messages[1] == {"type": "input_text", "text": "Another message"} + assert len(messages) == 1 + msg = cast("dict[str, Any]", messages[0]) + assert msg["role"] == "user" + content = cast("list[dict[str, Any]]", msg["content"]) + assert len(content) == 2 + assert content[0] == {"type": "input_text", "text": "Hello, world!"} + assert content[1] == {"type": "input_text", "text": "How are you?"} + + @pytest.mark.asyncio + async def test_format_blocks_with_image(self, mock_mcp_client, mock_openai): + """Test formatting content blocks with images.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) - # Test with mixed content - blocks = [ - types.TextContent(type="text", text="Text content"), - types.ImageContent(type="image", data="base64data", mimeType="image/png"), + blocks: list[types.ContentBlock] = [ + types.TextContent(type="text", text="Check this out:"), + types.ImageContent(type="image", data="base64imagedata", mimeType="image/jpeg"), ] messages = await agent.format_blocks(blocks) - assert len(messages) == 2 - assert messages[0] == {"type": "input_text", "text": "Text content"} - assert messages[1] == { + assert len(messages) == 1 + msg = cast("dict[str, Any]", messages[0]) + assert msg["role"] == "user" + content = cast("list[dict[str, Any]]", msg["content"]) + assert len(content) == 2 + assert content[0] == {"type": "input_text", "text": "Check this out:"} + assert content[1] == { "type": "input_image", - "image_url": "data:image/png;base64,base64data", + "image_url": "data:image/jpeg;base64,base64imagedata", } @pytest.mark.asyncio - async def test_format_tool_results(self, mock_mcp_client, mock_openai): - """Test formatting tool results.""" - agent = OperatorAgent( + async def test_format_blocks_empty(self, mock_mcp_client, mock_openai): + """Test formatting empty content blocks.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + blocks: list[types.ContentBlock] = [] + + messages = await agent.format_blocks(blocks) + assert len(messages) == 1 + msg = cast("dict[str, Any]", messages[0]) + assert msg["role"] == "user" + content = cast("list[dict[str, Any]]", msg["content"]) + assert len(content) == 1 + assert content[0] == {"type": "input_text", "text": ""} + + @pytest.mark.asyncio + async def test_format_tool_results_text(self, mock_mcp_client, mock_openai): + """Test formatting tool results with text content.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + tool_calls = [ + MCPToolCall(name="test_tool", arguments={"arg": "value"}, id="call_123"), # type: ignore + ] + + tool_results = [ + MCPToolResult( + content=[types.TextContent(type="text", text="Tool executed successfully")], + isError=False, + ), + ] + + messages = await agent.format_tool_results(tool_calls, tool_results) + + assert len(messages) == 1 + msg = cast("dict[str, Any]", messages[0]) + assert msg["type"] == "function_call_output" + assert msg["call_id"] == "call_123" + output = cast("list[dict[str, Any]]", msg["output"]) + assert len(output) == 1 + assert output[0]["type"] == "input_text" + assert output[0]["text"] == "Tool executed successfully" + + @pytest.mark.asyncio + async def test_format_tool_results_with_image(self, mock_mcp_client, mock_openai): + """Test formatting tool results with image content.""" + agent = OpenAIAgent( mcp_client=mock_mcp_client, model_client=mock_openai, - validate_api_key=False, # Skip validation in tests + validate_api_key=False, ) tool_calls = [ - MCPToolCall(name="test_tool", arguments={}, id="call_123"), # type: ignore MCPToolCall(name="screenshot", arguments={}, id="call_456"), # type: ignore ] tool_results = [ - MCPToolResult(content=[types.TextContent(type="text", text="Success")], isError=False), MCPToolResult( - content=[types.ImageContent(type="image", data="base64data", mimeType="image/png")], + content=[ + types.ImageContent(type="image", data="screenshot_data", mimeType="image/png") + ], isError=False, ), ] messages = await agent.format_tool_results(tool_calls, tool_results) - # OpenAI's format_tool_results returns input_image with screenshot assert len(messages) == 1 - assert messages[0]["type"] == "input_image" - assert "image_url" in messages[0] - assert messages[0]["image_url"] == "data:image/png;base64,base64data" + msg = cast("dict[str, Any]", messages[0]) + assert msg["type"] == "function_call_output" + assert msg["call_id"] == "call_456" + output = cast("list[dict[str, Any]]", msg["output"]) + assert len(output) == 1 + assert output[0]["type"] == "input_image" + assert output[0]["image_url"] == "data:image/png;base64,screenshot_data" @pytest.mark.asyncio async def test_format_tool_results_with_error(self, mock_mcp_client, mock_openai): """Test formatting tool results with errors.""" - agent = OperatorAgent( + agent = OpenAIAgent( mcp_client=mock_mcp_client, model_client=mock_openai, - validate_api_key=False, # Skip validation in tests + validate_api_key=False, ) tool_calls = [ @@ -134,80 +242,536 @@ async def test_format_tool_results_with_error(self, mock_mcp_client, mock_openai tool_results = [ MCPToolResult( - content=[types.TextContent(type="text", text="Something went wrong")], isError=True + content=[types.TextContent(type="text", text="Error: Something went wrong")], + isError=True, + ), + ] + + messages = await agent.format_tool_results(tool_calls, tool_results) + + assert len(messages) == 1 + msg = cast("dict[str, Any]", messages[0]) + assert msg["type"] == "function_call_output" + assert msg["call_id"] == "call_error" + output = cast("list[dict[str, Any]]", msg["output"]) + assert len(output) == 2 + assert output[0]["type"] == "input_text" + assert output[0]["text"] == "[tool_error] true" + assert output[1]["type"] == "input_text" + assert output[1]["text"] == "Error: Something went wrong" + + @pytest.mark.asyncio + async def test_format_tool_results_with_structured_content(self, mock_mcp_client, mock_openai): + """Test formatting tool results with structured content.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + tool_calls = [ + MCPToolCall(name="data_tool", arguments={}, id="call_789"), # type: ignore + ] + + tool_results = [ + MCPToolResult( + content=[], + structuredContent={"key": "value", "number": 42}, + isError=False, + ), + ] + + messages = await agent.format_tool_results(tool_calls, tool_results) + + assert len(messages) == 1 + msg = cast("dict[str, Any]", messages[0]) + assert msg["type"] == "function_call_output" + assert msg["call_id"] == "call_789" + output = cast("list[dict[str, Any]]", msg["output"]) + assert len(output) == 1 + assert output[0]["type"] == "input_text" + # Structured content is JSON serialized + import json + + parsed = json.loads(output[0]["text"]) + assert parsed == {"key": "value", "number": 42} + + @pytest.mark.asyncio + async def test_format_tool_results_multiple(self, mock_mcp_client, mock_openai): + """Test formatting multiple tool results.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + tool_calls = [ + MCPToolCall(name="tool1", arguments={}, id="call_1"), # type: ignore + MCPToolCall(name="tool2", arguments={}, id="call_2"), # type: ignore + ] + + tool_results = [ + MCPToolResult( + content=[types.TextContent(type="text", text="Result 1")], + isError=False, + ), + MCPToolResult( + content=[types.TextContent(type="text", text="Result 2")], + isError=False, + ), + ] + + messages = await agent.format_tool_results(tool_calls, tool_results) + + assert len(messages) == 2 + msg0 = cast("dict[str, Any]", messages[0]) + assert msg0["call_id"] == "call_1" + msg1 = cast("dict[str, Any]", messages[1]) + assert msg1["call_id"] == "call_2" + + @pytest.mark.asyncio + async def test_format_tool_results_missing_call_id(self, mock_mcp_client, mock_openai): + """Test formatting tool results with missing call_id.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + tool_calls = [ + MCPToolCall(name="tool_no_id", arguments={}, id=""), # Empty string instead of None + ] + + tool_results = [ + MCPToolResult( + content=[types.TextContent(type="text", text="Some result")], + isError=False, ), ] messages = await agent.format_tool_results(tool_calls, tool_results) - # Since the result has isError=True and no screenshot, returns empty list + # Should skip tools without call_id (empty string is falsy) assert len(messages) == 0 @pytest.mark.asyncio - async def test_get_model_response(self, mock_mcp_client, mock_openai): - """Test getting model response from OpenAI API.""" - # Disable telemetry for this test to avoid backend configuration issues + async def test_get_response_with_text(self, mock_mcp_client, mock_openai): + """Test getting model response with text output.""" + # Disable telemetry for this test with patch("hud.settings.settings.telemetry_enabled", False): - agent = OperatorAgent( + agent = OpenAIAgent( mcp_client=mock_mcp_client, model_client=mock_openai, - validate_api_key=False, # Skip validation in tests + validate_api_key=False, ) - # Set up available tools so agent doesn't return "No computer use tools available" - agent._available_tools = [ - types.Tool(name="computer_openai", description="Computer tool", inputSchema={}) - ] - - # Mock OpenAI API response for a successful computer use response + # Mock OpenAI API response mock_response = MagicMock() mock_response.id = "response_123" - mock_response.state = "completed" - # Mock the output message structure - mock_output_text = MagicMock() - mock_output_text.type = "output_text" - mock_output_text.text = "I can see the screen content." - mock_output_message = MagicMock() - mock_output_message.type = "message" - mock_output_message.content = [mock_output_text] + # Create properly typed output text with all required fields + mock_output_text = ResponseOutputText( + type="output_text", + text="This is the response text", + annotations=[], # Required field + ) + + # Create properly typed output message with all required fields + mock_output_message = ResponseOutputMessage( + type="message", + id="msg_123", # Required field + role="assistant", # Required field + status="completed", # Required field + content=[mock_output_text], + ) mock_response.output = [mock_output_message] mock_openai.responses.create = AsyncMock(return_value=mock_response) - messages = [{"prompt": "What's on the screen?", "screenshot": None}] + # Test with initial message + messages = [{"role": "user", "content": [{"type": "input_text", "text": "Hello"}]}] response = await agent.get_response(messages) - # The test should verify that the response is processed correctly - # Since the isinstance checks will fail, content will be empty, but done should be True + assert response.content == "This is the response text" assert response.done is True assert response.tool_calls == [] + assert agent.last_response_id == "response_123" @pytest.mark.asyncio - async def test_handle_empty_response(self, mock_mcp_client, mock_openai): - """Test handling empty response from API.""" - agent = OperatorAgent( + async def test_get_response_with_tool_call(self, mock_mcp_client, mock_openai): + """Test getting model response with tool call.""" + with patch("hud.settings.settings.telemetry_enabled", False): + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + # Set up tool name map + agent._tool_name_map = {"test_tool": "test_tool"} + + # Mock OpenAI API response with properly typed function call + mock_response = MagicMock() + mock_response.id = "response_456" + + # Create properly typed function call with correct type value + mock_function_call = ResponseFunctionToolCall( + type="function_call", # Correct type value + call_id="call_123", + name="test_tool", + arguments='{"param": "value"}', + ) + + mock_response.output = [mock_function_call] + + mock_openai.responses.create = AsyncMock(return_value=mock_response) + + messages = [ + {"role": "user", "content": [{"type": "input_text", "text": "Do something"}]} + ] + response = await agent.get_response(messages) + + assert response.done is False + assert len(response.tool_calls) == 1 + assert response.tool_calls[0].name == "test_tool" + assert response.tool_calls[0].id == "call_123" + assert response.tool_calls[0].arguments == {"param": "value"} + + @pytest.mark.asyncio + async def test_get_response_with_reasoning(self, mock_mcp_client, mock_openai): + """Test getting model response with reasoning.""" + with patch("hud.settings.settings.telemetry_enabled", False): + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + # Mock OpenAI API response with properly typed reasoning + mock_response = MagicMock() + mock_response.id = "response_789" + + # Create a properly typed reasoning item with all required fields + mock_summary = Summary( + type="summary_text", # Correct literal type value + text="Let me think about this...", + ) + + mock_reasoning = ResponseReasoningItem( + type="reasoning", + id="reasoning_1", # Required field + summary=[mock_summary], # Required field + status="completed", # Required field + ) + + # Create properly typed output message with all required fields + mock_output_text = ResponseOutputText( + type="output_text", + text="Final answer", + annotations=[], # Required field + ) + mock_output_message = ResponseOutputMessage( + type="message", + id="msg_789", # Required field + role="assistant", # Required field + status="completed", # Required field + content=[mock_output_text], + ) + + mock_response.output = [mock_reasoning, mock_output_message] + + mock_openai.responses.create = AsyncMock(return_value=mock_response) + + messages = [ + {"role": "user", "content": [{"type": "input_text", "text": "Hard question"}]} + ] + response = await agent.get_response(messages) + + assert "Thinking: Let me think about this..." in response.content + assert "Final answer" in response.content + + @pytest.mark.asyncio + async def test_get_response_empty_messages(self, mock_mcp_client, mock_openai): + """Test getting model response with empty messages.""" + with patch("hud.settings.settings.telemetry_enabled", False): + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + # Mock empty response + mock_response = MagicMock() + mock_response.id = "response_empty" + mock_response.output = [] + + mock_openai.responses.create = AsyncMock(return_value=mock_response) + + messages = [] + response = await agent.get_response(messages) + + assert response.content == "" + assert response.tool_calls == [] + + @pytest.mark.asyncio + async def test_get_response_no_new_messages_with_previous_id( + self, mock_mcp_client, mock_openai + ): + """Test getting model response when no new messages and previous response exists.""" + with patch("hud.settings.settings.telemetry_enabled", False): + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + agent.last_response_id = "prev_response" + agent._message_cursor = 1 + + messages = [{"role": "user", "content": [{"type": "input_text", "text": "Hello"}]}] + response = await agent.get_response(messages) + + # Should return early without calling API + assert response.content == "" + assert response.done is True + mock_openai.responses.create.assert_not_called() + + @pytest.mark.asyncio + async def test_build_request_payload(self, mock_mcp_client, mock_openai): + """Test building request payload.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + model="gpt-4o", + max_output_tokens=1024, + temperature=0.5, + reasoning="auto", + tool_choice="auto", + parallel_tool_calls=True, + validate_api_key=False, + ) + + agent._openai_tools = [cast("Any", {"type": "function", "name": "test"})] + agent.system_prompt = "You are a helpful assistant" + agent.last_response_id = "prev_123" + + new_items = cast( + "Any", [{"role": "user", "content": [{"type": "input_text", "text": "Hi"}]}] + ) + payload = agent._build_request_payload(new_items) + + assert payload["model"] == "gpt-4o" + assert payload["input"] == new_items + assert payload["instructions"] == "You are a helpful assistant" + assert payload["max_output_tokens"] == 1024 + assert payload["temperature"] == 0.5 + assert payload["reasoning"] == "auto" + assert payload["tool_choice"] == "auto" + assert payload["parallel_tool_calls"] is True + assert payload["tools"] == [{"type": "function", "name": "test"}] + assert payload["previous_response_id"] == "prev_123" + + @pytest.mark.asyncio + async def test_build_request_payload_minimal(self, mock_mcp_client, mock_openai): + """Test building request payload with minimal parameters.""" + agent = OpenAIAgent( mcp_client=mock_mcp_client, model_client=mock_openai, - validate_api_key=False, # Skip validation in tests + validate_api_key=False, ) - # Set up available tools - agent._available_tools = [ - types.Tool(name="openai_computer", description="Computer tool", inputSchema={}) + new_items = cast( + "Any", [{"role": "user", "content": [{"type": "input_text", "text": "Hi"}]}] + ) + payload = agent._build_request_payload(new_items) + + assert payload["model"] == "gpt-5.1" # default + assert payload["input"] == new_items + assert "max_output_tokens" not in payload + assert "temperature" not in payload + assert "reasoning" not in payload + assert "tools" not in payload + assert "previous_response_id" not in payload + + @pytest.mark.asyncio + async def test_reset_response_state(self, mock_mcp_client, mock_openai): + """Test resetting response state.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + # Set some state + agent.last_response_id = "some_id" + agent.pending_call_id = "call_id" + agent.pending_safety_checks = [{"check": "value"}] + agent._message_cursor = 5 + + # Reset + agent._reset_response_state() + + assert agent.last_response_id is None + assert agent.pending_call_id is None + assert agent.pending_safety_checks == [] + assert agent._message_cursor == 0 + + @pytest.mark.asyncio + async def test_get_system_messages(self, mock_mcp_client, mock_openai): + """Test getting system messages.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + # OpenAI agent returns empty list (uses instructions field instead) + messages = await agent.get_system_messages() + assert messages == [] + + @pytest.mark.asyncio + async def test_build_openai_tools(self, mock_mcp_client, mock_openai): + """Test building OpenAI tools from MCP tools.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + # Mock MCP tools + mock_tools = [ + types.Tool( + name="tool1", + description="First tool", + inputSchema={ + "type": "object", + "properties": {"arg1": {"type": "string"}}, + "required": ["arg1"], + "additionalProperties": False, + }, + ), + types.Tool( + name="tool2", + description="Second tool", + inputSchema={ + "type": "object", + "properties": {}, + "additionalProperties": False, + }, + ), ] - # Mock empty response - mock_response = MagicMock() - mock_response.id = "response_empty" - mock_response.state = "completed" - mock_response.output = [] # Empty output + agent._available_tools = mock_tools + agent._build_openai_tools() + + assert len(agent._openai_tools) == 2 + assert agent._tool_name_map == {"tool1": "tool1", "tool2": "tool2"} + + tool1 = cast("dict[str, Any]", agent._openai_tools[0]) + assert tool1["type"] == "function" + assert tool1["name"] == "tool1" + assert tool1["description"] == "First tool" + assert tool1["strict"] is True + + @pytest.mark.asyncio + async def test_build_openai_tools_skips_incomplete(self, mock_mcp_client, mock_openai): + """Test building OpenAI tools skips tools without description or schema.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + # Create mock tools directly as objects that bypass pydantic validation + incomplete1 = MagicMock(spec=types.Tool) + incomplete1.name = "incomplete1" + incomplete1.description = None + incomplete1.inputSchema = {"type": "object"} + + incomplete2 = MagicMock(spec=types.Tool) + incomplete2.name = "incomplete2" + incomplete2.description = "Has description" + incomplete2.inputSchema = None + + complete = types.Tool( + name="complete", + description="Complete tool", + inputSchema={"type": "object", "properties": {}, "additionalProperties": False}, + ) + + agent._available_tools = [incomplete1, incomplete2, complete] + agent._build_openai_tools() + + # Should only have the complete tool + assert len(agent._openai_tools) == 1 + tool = cast("dict[str, Any]", agent._openai_tools[0]) + assert tool["name"] == "complete" + + @pytest.mark.asyncio + async def test_convert_function_tool_call(self, mock_mcp_client, mock_openai): + """Test converting OpenAI function tool call to MCP format.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + agent._tool_name_map = {"openai_name": "mcp_name"} + + mock_call = MagicMock() + mock_call.call_id = "call_123" + mock_call.name = "openai_name" + mock_call.arguments = '{"key": "value", "number": 42}' + + result = agent._convert_function_tool_call(mock_call) + + assert result is not None + assert result.name == "mcp_name" + assert result.id == "call_123" + assert result.arguments == {"key": "value", "number": 42} + + @pytest.mark.asyncio + async def test_convert_function_tool_call_invalid_json(self, mock_mcp_client, mock_openai): + """Test converting function tool call with invalid JSON.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + agent._tool_name_map = {"tool": "tool"} + + mock_call = MagicMock() + mock_call.call_id = "call_456" + mock_call.name = "tool" + mock_call.arguments = "invalid json {{" + + result = agent._convert_function_tool_call(mock_call) + + assert result is not None + assert result.name == "tool" + assert result.id == "call_456" + # Should wrap invalid JSON in raw_arguments + assert result.arguments == {"raw_arguments": "invalid json {{"} + + @pytest.mark.asyncio + async def test_convert_function_tool_call_empty_args(self, mock_mcp_client, mock_openai): + """Test converting function tool call with empty arguments.""" + agent = OpenAIAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, + ) + + agent._tool_name_map = {"tool": "tool"} - mock_openai.responses.create = AsyncMock(return_value=mock_response) + mock_call = MagicMock() + mock_call.call_id = "call_789" + mock_call.name = "tool" + mock_call.arguments = None - messages = [{"prompt": "Hi", "screenshot": None}] - response = await agent.get_response(messages) + result = agent._convert_function_tool_call(mock_call) - assert response.content == "" - assert response.tool_calls == [] + assert result is not None + assert result.arguments == {} diff --git a/hud/agents/tests/test_operator.py b/hud/agents/tests/test_operator.py new file mode 100644 index 000000000..e20287006 --- /dev/null +++ b/hud/agents/tests/test_operator.py @@ -0,0 +1,241 @@ +"""Tests for OpenAI MCP Agent implementation.""" + +from __future__ import annotations + +from typing import Any, cast +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from mcp import types + +from hud.agents.operator import OperatorAgent +from hud.types import MCPToolCall, MCPToolResult + + +class TestOperatorAgent: + """Test OperatorAgent class.""" + + @pytest.fixture + def mock_mcp_client(self): + """Create a mock MCP client.""" + mcp_client = AsyncMock() + # Set up the mcp_config attribute as a regular dict, not a coroutine + mcp_client.mcp_config = {"test_server": {"url": "http://test"}} + # Mock list_tools to return the required openai_computer tool + mcp_client.list_tools = AsyncMock( + return_value=[ + types.Tool( + name="openai_computer", description="OpenAI computer use tool", inputSchema={} + ) + ] + ) + mcp_client.initialize = AsyncMock() + return mcp_client + + @pytest.fixture + def mock_openai(self): + """Create a mock OpenAI client.""" + with patch("hud.agents.openai.AsyncOpenAI") as mock: + client = AsyncMock() + mock.return_value = client + yield client + + @pytest.mark.asyncio + async def test_init(self, mock_mcp_client): + """Test agent initialization.""" + mock_model_client = MagicMock() + agent = OperatorAgent( + mcp_client=mock_mcp_client, + model_client=mock_model_client, + model="gpt-4", + validate_api_key=False, # Skip validation in tests + ) + + assert agent.model_name == "Operator" + assert agent.model == "gpt-4" + assert agent.openai_client == mock_model_client + + @pytest.mark.asyncio + async def test_format_blocks(self, mock_mcp_client): + """Test formatting content blocks.""" + mock_model_client = MagicMock() + agent = OperatorAgent( + mcp_client=mock_mcp_client, + model_client=mock_model_client, + validate_api_key=False, # Skip validation in tests + ) + + # Test with text blocks + blocks: list[types.ContentBlock] = [ + types.TextContent(type="text", text="Hello, GPT!"), + types.TextContent(type="text", text="Another message"), + ] + + messages = await agent.format_blocks(blocks) + assert len(messages) == 1 + msg = cast("dict[str, Any]", messages[0]) + assert msg["role"] == "user" + content = cast("list[dict[str, Any]]", msg["content"]) + assert len(content) == 2 + assert content[0] == {"type": "input_text", "text": "Hello, GPT!"} + assert content[1] == {"type": "input_text", "text": "Another message"} + + # Test with mixed content + blocks = [ + types.TextContent(type="text", text="Text content"), + types.ImageContent(type="image", data="base64data", mimeType="image/png"), + ] + + messages = await agent.format_blocks(blocks) + assert len(messages) == 1 + msg = cast("dict[str, Any]", messages[0]) + assert msg["role"] == "user" + content = cast("list[dict[str, Any]]", msg["content"]) + assert len(content) == 2 + assert content[0] == {"type": "input_text", "text": "Text content"} + assert content[1] == { + "type": "input_image", + "image_url": "data:image/png;base64,base64data", + } + + @pytest.mark.asyncio + async def test_format_tool_results(self, mock_mcp_client, mock_openai): + """Test formatting tool results.""" + agent = OperatorAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, # Skip validation in tests + ) + + tool_calls = [ + MCPToolCall(name="test_tool", arguments={}, id="call_123"), # type: ignore + MCPToolCall(name="screenshot", arguments={}, id="call_456"), # type: ignore + ] + + tool_results = [ + MCPToolResult(content=[types.TextContent(type="text", text="Success")], isError=False), + MCPToolResult( + content=[types.ImageContent(type="image", data="base64data", mimeType="image/png")], + isError=False, + ), + ] + + messages = await agent.format_tool_results(tool_calls, tool_results) + + # Should return both tool results as function_call_output + assert len(messages) == 2 + # First result is text + msg0 = cast("dict[str, Any]", messages[0]) + assert msg0["type"] == "function_call_output" + assert msg0["call_id"] == "call_123" + output0 = cast("list[dict[str, Any]]", msg0["output"]) + assert output0[0]["type"] == "input_text" + assert output0[0]["text"] == "Success" + # Second result is image + msg1 = cast("dict[str, Any]", messages[1]) + assert msg1["type"] == "function_call_output" + assert msg1["call_id"] == "call_456" + output1 = cast("list[dict[str, Any]]", msg1["output"]) + assert output1[0]["type"] == "input_image" + assert output1[0]["image_url"] == "data:image/png;base64,base64data" + + @pytest.mark.asyncio + async def test_format_tool_results_with_error(self, mock_mcp_client, mock_openai): + """Test formatting tool results with errors.""" + agent = OperatorAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, # Skip validation in tests + ) + + tool_calls = [ + MCPToolCall(name="failing_tool", arguments={}, id="call_error"), # type: ignore + ] + + tool_results = [ + MCPToolResult( + content=[types.TextContent(type="text", text="Something went wrong")], isError=True + ), + ] + + messages = await agent.format_tool_results(tool_calls, tool_results) + + # Error results are returned with error flag and content + assert len(messages) == 1 + msg = cast("dict[str, Any]", messages[0]) + assert msg["type"] == "function_call_output" + assert msg["call_id"] == "call_error" + output = cast("list[dict[str, Any]]", msg["output"]) + assert output[0]["type"] == "input_text" + assert output[0]["text"] == "[tool_error] true" + assert output[1]["type"] == "input_text" + assert output[1]["text"] == "Something went wrong" + + @pytest.mark.asyncio + async def test_get_model_response(self, mock_mcp_client, mock_openai): + """Test getting model response from OpenAI API.""" + # Disable telemetry for this test to avoid backend configuration issues + with patch("hud.settings.settings.telemetry_enabled", False): + agent = OperatorAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, # Skip validation in tests + ) + + # Set up available tools so agent doesn't return "No computer use tools available" + agent._available_tools = [ + types.Tool(name="computer_openai", description="Computer tool", inputSchema={}) + ] + + # Mock OpenAI API response for a successful computer use response + mock_response = MagicMock() + mock_response.id = "response_123" + mock_response.state = "completed" + # Mock the output message structure + mock_output_text = MagicMock() + mock_output_text.type = "output_text" + mock_output_text.text = "I can see the screen content." + + mock_output_message = MagicMock() + mock_output_message.type = "message" + mock_output_message.content = [mock_output_text] + + mock_response.output = [mock_output_message] + + mock_openai.responses.create = AsyncMock(return_value=mock_response) + + messages = [{"prompt": "What's on the screen?", "screenshot": None}] + response = await agent.get_response(messages) + + # The test should verify that the response is processed correctly + # Since the isinstance checks will fail, content will be empty, but done should be True + assert response.done is True + assert response.tool_calls == [] + + @pytest.mark.asyncio + async def test_handle_empty_response(self, mock_mcp_client, mock_openai): + """Test handling empty response from API.""" + agent = OperatorAgent( + mcp_client=mock_mcp_client, + model_client=mock_openai, + validate_api_key=False, # Skip validation in tests + ) + + # Set up available tools + agent._available_tools = [ + types.Tool(name="openai_computer", description="Computer tool", inputSchema={}) + ] + + # Mock empty response + mock_response = MagicMock() + mock_response.id = "response_empty" + mock_response.state = "completed" + mock_response.output = [] # Empty output + + mock_openai.responses.create = AsyncMock(return_value=mock_response) + + messages = [{"prompt": "Hi", "screenshot": None}] + response = await agent.get_response(messages) + + assert response.content == "" + assert response.tool_calls == [] diff --git a/hud/cli/__init__.py b/hud/cli/__init__.py index 0811e36fc..9458c4d30 100644 --- a/hud/cli/__init__.py +++ b/hud/cli/__init__.py @@ -1023,7 +1023,8 @@ def eval( choices.extend( [ {"name": "Claude 4 Sonnet", "value": AgentType.CLAUDE}, - {"name": "OpenAI Computer Use", "value": AgentType.OPENAI}, + {"name": "OpenAI", "value": AgentType.OPENAI}, + {"name": "Operator (OpenAI Computer Use)", "value": AgentType.OPERATOR}, {"name": "Gemini Computer Use", "value": AgentType.GEMINI}, {"name": "vLLM (Local Server)", "value": AgentType.VLLM}, {"name": "LiteLLM (Multi-provider)", "value": AgentType.LITELLM}, diff --git a/hud/cli/eval.py b/hud/cli/eval.py index 3a7ff1b81..d0dab7416 100644 --- a/hud/cli/eval.py +++ b/hud/cli/eval.py @@ -153,7 +153,7 @@ def build_agent( elif agent_type == AgentType.VLLM: # Create a generic OpenAI agent for vLLM server try: - from hud.agents.openai_chat_generic import GenericOpenAIChatAgent + from hud.agents.openai_chat import OpenAIChatAgent except ImportError as e: hud_console.error( "OpenAI dependencies are not installed. " @@ -168,11 +168,11 @@ def build_agent( allowed_tools=allowed_tools, verbose=verbose, ) - return GenericOpenAIChatAgent(**config) + return OpenAIChatAgent(**config) elif agent_type == AgentType.OPENAI: try: - from hud.agents import OperatorAgent + from hud.agents import OpenAIAgent except ImportError as e: hud_console.error( "OpenAI agent dependencies are not installed. " @@ -180,13 +180,27 @@ def build_agent( ) raise typer.Exit(1) from e + agent_kwargs: dict[str, Any] = {"verbose": verbose} if allowed_tools: - return OperatorAgent( - allowed_tools=allowed_tools, - verbose=verbose, + agent_kwargs["allowed_tools"] = allowed_tools + if model: + agent_kwargs["model"] = model + return OpenAIAgent(**agent_kwargs) + + elif agent_type == AgentType.OPERATOR: + try: + from hud.agents import OperatorAgent + except ImportError as e: + hud_console.error( + "OpenAI agent dependencies are not installed. " + "Please install with: pip install 'hud-python[agent]'" ) - else: - return OperatorAgent(verbose=verbose) + raise typer.Exit(1) from e + + operator_kwargs: dict[str, Any] = {"verbose": verbose} + if allowed_tools: + operator_kwargs["allowed_tools"] = allowed_tools + return OperatorAgent(**operator_kwargs) elif agent_type == AgentType.GEMINI: try: @@ -349,9 +363,9 @@ async def run_single_task( agent_config["allowed_tools"] = allowed_tools elif agent_type == AgentType.VLLM: # Special handling for vLLM - from hud.agents.openai_chat_generic import GenericOpenAIChatAgent + from hud.agents.openai_chat import OpenAIChatAgent - agent_class = GenericOpenAIChatAgent + agent_class = OpenAIChatAgent # Use the shared config builder agent_config = _build_vllm_config( @@ -361,6 +375,15 @@ async def run_single_task( verbose=verbose, ) elif agent_type == AgentType.OPENAI: + from hud.agents import OpenAIAgent + + agent_class = OpenAIAgent + agent_config = {"verbose": verbose} + if allowed_tools: + agent_config["allowed_tools"] = allowed_tools + if model: + agent_config["model"] = model + elif agent_type == AgentType.OPERATOR: from hud.agents import OperatorAgent agent_class = OperatorAgent @@ -541,9 +564,9 @@ async def run_full_dataset( agent_config = {"verbose": verbose} elif agent_type == AgentType.VLLM: try: - from hud.agents.openai_chat_generic import GenericOpenAIChatAgent + from hud.agents.openai_chat import OpenAIChatAgent - agent_class = GenericOpenAIChatAgent + agent_class = OpenAIChatAgent except ImportError as e: hud_console.error( "OpenAI dependencies are not installed. " @@ -559,6 +582,26 @@ async def run_full_dataset( verbose=verbose, ) elif agent_type == AgentType.OPENAI: + try: + from hud.agents import OpenAIAgent + + agent_class = OpenAIAgent + except ImportError as e: + hud_console.error( + "OpenAI agent dependencies are not installed. " + "Please install with: pip install 'hud-python[agent]'" + ) + raise typer.Exit(1) from e + + agent_config = { + "verbose": verbose, + "validate_api_key": False, + } + if allowed_tools: + agent_config["allowed_tools"] = allowed_tools + if model: + agent_config["model"] = model + elif agent_type == AgentType.OPERATOR: try: from hud.agents import OperatorAgent @@ -701,7 +744,7 @@ def eval_command( agent: AgentType = typer.Option( # noqa: B008 AgentType.CLAUDE, "--agent", - help="Agent backend to use (claude, gemini, openai, vllm for local servers, or litellm)", + help=("Agent (claude, gemini, openai, operator, vllm, or litellm)"), ), model: str | None = typer.Option( None, @@ -789,8 +832,11 @@ def eval_command( # Run multiple tasks from a JSON file hud eval tasks.json --full - # Run with OpenAI Operator agent - hud eval hud-evals/OSWorld-Gold-Beta --agent openai + # Run with OpenAI agent (choose model via --model) + hud eval hud-evals/OSWorld-Gold-Beta --agent openai --model gpt-5 + + # Run with the legacy Operator computer-use agent + hud eval hud-evals/OSWorld-Gold-Beta --agent operator # Use local vLLM server (default: localhost:8000) hud eval task.json --agent vllm --model Qwen/Qwen2.5-VL-3B-Instruct @@ -834,7 +880,7 @@ def eval_command( "Set it in your environment or run: hud set GEMINI_API_KEY=your-key-here" ) raise typer.Exit(1) - elif agent == AgentType.OPENAI and not settings.openai_api_key: + elif agent in (AgentType.OPENAI, AgentType.OPERATOR) and not settings.openai_api_key: hud_console.error("OPENAI_API_KEY is required for OpenAI agent") hud_console.info("Set it in your environment or run: hud set OPENAI_API_KEY=your-key-here") raise typer.Exit(1) diff --git a/hud/rl/actor.py b/hud/rl/actor.py index 9c073b5d0..4c9a3390c 100644 --- a/hud/rl/actor.py +++ b/hud/rl/actor.py @@ -9,7 +9,7 @@ from openai import AsyncOpenAI import hud -from hud.agents.openai_chat_generic import GenericOpenAIChatAgent +from hud.agents.openai_chat import OpenAIChatAgent from hud.clients.utils.retry_transport import create_retry_httpx_client from hud.types import Task, Trace from hud.utils.hud_console import HUDConsole @@ -46,9 +46,9 @@ def _create_openai_client(self, base_url: str) -> AsyncOpenAI: max_retries=2, ) - def create_agent(self) -> GenericOpenAIChatAgent: + def create_agent(self) -> OpenAIChatAgent: """Create an agent with the current adapter.""" - return GenericOpenAIChatAgent( + return OpenAIChatAgent( openai_client=self.openai_client, model_name=self.current_adapter, allowed_tools=self.actor_config.allowed_tools, diff --git a/hud/types.py b/hud/types.py index 491563d68..a82ad2930 100644 --- a/hud/types.py +++ b/hud/types.py @@ -25,6 +25,7 @@ class AgentType(str, Enum): CLAUDE = "claude" OPENAI = "openai" + OPERATOR = "operator" GEMINI = "gemini" VLLM = "vllm" LITELLM = "litellm" diff --git a/hud/utils/agent_factories.py b/hud/utils/agent_factories.py index e15cb2404..ed636007b 100644 --- a/hud/utils/agent_factories.py +++ b/hud/utils/agent_factories.py @@ -7,21 +7,21 @@ from openai import AsyncOpenAI from hud.agents.grounded_openai import GroundedOpenAIChatAgent -from hud.agents.openai_chat_generic import GenericOpenAIChatAgent +from hud.agents.openai_chat import OpenAIChatAgent from hud.tools.grounding import GrounderConfig -def create_openai_agent(**kwargs: Any) -> GenericOpenAIChatAgent: - """Factory for GenericOpenAIChatAgent with run_dataset compatibility. +def create_openai_agent(**kwargs: Any) -> OpenAIChatAgent: + """Factory for OpenAIChatAgent with run_dataset compatibility. Args: api_key: OpenAI API key base_url: Optional custom API endpoint model_name: Model to use (e.g., "gpt-4o-mini") - **kwargs: Additional arguments passed to GenericOpenAIChatAgent + **kwargs: Additional arguments passed to OpenAIChatAgent Returns: - Configured GenericOpenAIChatAgent instance + Configured OpenAIChatAgent instance Example: >>> from hud.datasets import run_dataset @@ -36,7 +36,7 @@ def create_openai_agent(**kwargs: Any) -> GenericOpenAIChatAgent: api_key = kwargs.pop("api_key", None) base_url = kwargs.pop("base_url", None) - return GenericOpenAIChatAgent(api_key=api_key, base_url=base_url, **kwargs) + return OpenAIChatAgent(api_key=api_key, base_url=base_url, **kwargs) def create_grounded_agent(**kwargs: Any) -> GroundedOpenAIChatAgent: diff --git a/hud/utils/strict_schema.py b/hud/utils/strict_schema.py new file mode 100644 index 000000000..5f44e9cdb --- /dev/null +++ b/hud/utils/strict_schema.py @@ -0,0 +1,162 @@ +"""Utilities to convert JSON schemas into OpenAI's strict format.""" + +from __future__ import annotations + +from typing import Any, TypeGuard + +_EMPTY_SCHEMA = { + "additionalProperties": False, + "type": "object", + "properties": {}, + "required": [], +} + + +def ensure_strict_json_schema(schema: dict[str, Any]) -> dict[str, Any]: + """Ensure a JSON schema conforms to OpenAI's strict requirements. + + This mutates the provided schema in-place and returns it for convenience. + """ + if schema == {}: + return _EMPTY_SCHEMA.copy() + return _ensure_strict_json_schema(schema, path=(), root=schema) + + +def _ensure_strict_json_schema( + json_schema: object, + *, + path: tuple[str, ...], + root: dict[str, Any], +) -> dict[str, Any]: + if not _is_dict(json_schema): + raise TypeError(f"Expected {json_schema} to be a dictionary; path={path}") + + defs = json_schema.get("$defs") + if _is_dict(defs): + for def_name, def_schema in defs.items(): + _ensure_strict_json_schema(def_schema, path=(*path, "$defs", def_name), root=root) + + definitions = json_schema.get("definitions") + if _is_dict(definitions): + for definition_name, definition_schema in definitions.items(): + _ensure_strict_json_schema( + definition_schema, path=(*path, "definitions", definition_name), root=root + ) + + typ = json_schema.get("type") + if typ == "object": + if "additionalProperties" not in json_schema or json_schema["additionalProperties"] is True: + json_schema["additionalProperties"] = False + elif ( + json_schema["additionalProperties"] and json_schema["additionalProperties"] is not False + ): + raise ValueError( + "additionalProperties should not be set for object types in strict mode." + ) + + properties = json_schema.get("properties") + if _is_dict(properties): + json_schema["required"] = list(properties.keys()) + json_schema["properties"] = { + key: _ensure_strict_json_schema(prop_schema, path=(*path, "properties", key), root=root) + for key, prop_schema in properties.items() + } + + items = json_schema.get("items") + if _is_dict(items): + json_schema["items"] = _ensure_strict_json_schema(items, path=(*path, "items"), root=root) + + prefix_items = json_schema.get("prefixItems") + if _is_list(prefix_items) and prefix_items: + item_types = set() + for item in prefix_items: + if _is_dict(item) and "type" in item: + item_types.add(item["type"]) + + if len(item_types) == 1: + item_type = item_types.pop() + json_schema["items"] = {"type": item_type} + else: + json_schema["items"] = {"type": "integer"} + + tuple_length = len(prefix_items) + json_schema["minItems"] = tuple_length + json_schema["maxItems"] = tuple_length + json_schema.pop("prefixItems") + + any_of = json_schema.get("anyOf") + if _is_list(any_of): + json_schema["anyOf"] = [ + _ensure_strict_json_schema(variant, path=(*path, "anyOf", str(i)), root=root) + for i, variant in enumerate(any_of) + ] + + one_of = json_schema.get("oneOf") + if _is_list(one_of): + existing_any_of = json_schema.get("anyOf", []) + if not _is_list(existing_any_of): + existing_any_of = [] + json_schema["anyOf"] = existing_any_of + [ + _ensure_strict_json_schema(variant, path=(*path, "oneOf", str(i)), root=root) + for i, variant in enumerate(one_of) + ] + json_schema.pop("oneOf") + + all_of = json_schema.get("allOf") + if _is_list(all_of): + if len(all_of) == 1: + json_schema.update( + _ensure_strict_json_schema(all_of[0], path=(*path, "allOf", "0"), root=root) + ) + json_schema.pop("allOf") + else: + json_schema["allOf"] = [ + _ensure_strict_json_schema(entry, path=(*path, "allOf", str(i)), root=root) + for i, entry in enumerate(all_of) + ] + + if "default" in json_schema: + json_schema.pop("default") + + for keyword in ("title", "examples"): + json_schema.pop(keyword, None) + + ref = json_schema.get("$ref") + if ref and _has_more_than_n_keys(json_schema, 1): + if not isinstance(ref, str): + raise ValueError(f"Received non-string $ref - {ref}") + resolved = _resolve_ref(root=root, ref=ref) + if not _is_dict(resolved): + raise ValueError( + f"Expected `$ref: {ref}` to resolve to a dictionary but got {resolved}" + ) + json_schema.update({**resolved, **json_schema}) + json_schema.pop("$ref") + return _ensure_strict_json_schema(json_schema, path=path, root=root) + + return json_schema + + +def _resolve_ref(*, root: dict[str, Any], ref: str) -> object: + if not ref.startswith("#/"): + raise ValueError(f"Unexpected $ref format {ref!r}; does not start with #/") + + path = ref[2:].split("/") + resolved: object = root + for key in path: + assert _is_dict(resolved), f"Encountered non-dictionary entry while resolving {ref}" # noqa: S101 + resolved = resolved[key] + + return resolved + + +def _is_dict(obj: object) -> TypeGuard[dict[str, Any]]: + return isinstance(obj, dict) + + +def _is_list(obj: object) -> TypeGuard[list[object]]: + return isinstance(obj, list) + + +def _has_more_than_n_keys(obj: dict[str, object], n: int) -> bool: + return any(count > n for count, _ in enumerate(obj, start=1)) diff --git a/hud/utils/tests/test_agent_factories.py b/hud/utils/tests/test_agent_factories.py index 2064c4172..111388fef 100644 --- a/hud/utils/tests/test_agent_factories.py +++ b/hud/utils/tests/test_agent_factories.py @@ -4,14 +4,14 @@ def test_create_openai_agent(): - from hud.agents.openai_chat_generic import GenericOpenAIChatAgent + from hud.agents.openai_chat import OpenAIChatAgent from hud.utils.agent_factories import create_openai_agent agent = create_openai_agent( api_key="test_key", model_name="test_model", completion_kwargs={"temperature": 0.5} ) - assert isinstance(agent, GenericOpenAIChatAgent) - assert agent.model_name == "GenericOpenAI" + assert isinstance(agent, OpenAIChatAgent) + assert agent.model_name == "OpenAI" assert agent.checkpoint_name == "test_model" assert agent.completion_kwargs["temperature"] == 0.5