OpenAI Agent using Responses API

docs/reference/agents.mdx

@@ -126,13 +126,65 @@ result = await agent.run(
 )
 ```
 
+### OpenAIAgent
+
+```python
+from hud.agents import OpenAIAgent
+```
+
+Generic OpenAI agent using the Responses API for function calling and tool execution.
+
+**Constructor Parameters:**
+| Parameter | Type | Description | Default |
+|-----------|------|-------------|---------|
+| `model_client` | `AsyncOpenAI` | OpenAI client | Auto-created |
+| `model` | `str` | Responses model to use | `"gpt-5.1"` |
+| `max_output_tokens` | `int \| None` | Maximum response tokens | `None` |
+| `temperature` | `float \| None` | Sampling temperature | `None` |
+| `reasoning` | `dict \| Literal["auto"] \| None` | Reasoning configuration | `None` |
+| `tool_choice` | `dict \| Literal["auto"] \| None` | Tool selection strategy | `None` |
+| `parallel_tool_calls` | `bool \| None` | Enable parallel tool execution | `None` |
+| `validate_api_key` | `bool` | Validate key on init | `True` |
+
+**Features:**
+- OpenAI Responses API integration
+- Function calling with strict mode schemas
+- Reasoning support for extended thinking
+- Automatic tool schema conversion
+- Response continuation with previous_response_id
+
+**Example:**
+```python
+agent = OpenAIAgent(
+    model="gpt-4o",
+    max_output_tokens=2048,
+    temperature=0.7,
+    reasoning="auto",
+)
+
+result = await agent.run(
+    Task(
+        prompt="Analyze the website and extract key information",
+        mcp_config={
+            "hud": {
+                "url": "https://mcp.hud.ai/v3/mcp",
+                "headers": {
+                    "Authorization": "Bearer ${HUD_API_KEY}",
+                    "Mcp-Image": "hudpython/hud-remote-browser:latest"
+                }
+            }
+        }
+    )
+)
+```
+
 ### OperatorAgent
 
 ```python
 from hud.agents import OperatorAgent
 ```
 
-OpenAI Operator-style agent built on the Responses API with computer-use.
+OpenAI Operator-style agent built on the Responses API with computer-use. Extends `OpenAIAgent` with computer-use capabilities.
 
 **Constructor Parameters:**
 | Parameter | Type | Description | Default |
@@ -146,11 +198,62 @@ OpenAI Operator-style agent built on the Responses API with computer-use.
 - OpenAI Responses computer-use
 - Operator-style system prompt guidance
 - Display metadata injection
+- Inherits all OpenAIAgent features
+
+### GeminiAgent
+
+```python
+from hud.agents import GeminiAgent
+```
+
+Google Gemini-specific implementation using Google's Generative AI API with computer-use capabilities.
+
+**Constructor Parameters:**
+| Parameter | Type | Description | Default |
+|-----------|------|-------------|---------|
+| `model_client` | `genai.Client` | Gemini client | Auto-created |
+| `model` | `str` | Gemini model to use | `"gemini-2.5-computer-use-preview-10-2025"` |
+| `temperature` | `float` | Sampling temperature | `1.0` |
+| `top_p` | `float` | Top-p sampling parameter | `0.95` |
+| `top_k` | `int` | Top-k sampling parameter | `40` |
+| `max_output_tokens` | `int` | Maximum response tokens | `8192` |
+| `validate_api_key` | `bool` | Validate key on init | `True` |
+| `excluded_predefined_functions` | `list[str] \| None` | Predefined functions to exclude | `None` |
+
+**Features:**
+- Native Gemini computer-use capabilities
+- Predefined computer-use functions (click, type, scroll, etc.)
+- Display metadata injection
+- MCP tool execution
+
+**Example:**
+```python
+agent = GeminiAgent(
+    model="gemini-2.5-computer-use-preview-10-2025",
+    temperature=0.7,
+    max_output_tokens=4096,
+)
+
+result = await agent.run(
+    Task(
+        prompt="Navigate to the website and fill out the form",
+        mcp_config={
+            "hud": {
+                "url": "https://mcp.hud.ai/v3/mcp",
+                "headers": {
+                    "Authorization": "Bearer ${HUD_API_KEY}",
+                    "Mcp-Image": "hudpython/hud-remote-browser:latest"
+                }
+            }
+        }
+    )
+)
+```
 
-### GenericOpenAIChatAgent
+### OpenAIChatAgent
 
 ```python
-from hud.agents import GenericOpenAIChatAgent
+from hud.agents import OpenAIChatAgent
 ```
 
 OpenAI-compatible chat.completions agent that works with any endpoint implementing the OpenAI schema (OpenAI, vLLM, Ollama, Together, custom, etc.).
@@ -171,7 +274,7 @@ openai_client = AsyncOpenAI(
     api_key="not-needed",
 )
 
-agent = GenericOpenAIChatAgent(
+agent = OpenAIChatAgent(
     openai_client=openai_client,
     model_name="llama3.1",
     completion_kwargs={"temperature": 0.2},

examples/openai_compatible_agent.py

@@ -23,7 +23,7 @@
 from openai import AsyncOpenAI
 
 import hud
-from hud.agents.openai_chat_generic import GenericOpenAIChatAgent
+from hud.agents.openai_chat import OpenAIChatAgent
 from hud.datasets import Task
 
 
@@ -127,7 +127,7 @@ async def run_example(mode: Literal["text", "browser"], target: int) -> None:
     allowed_tools = ["computer"] if mode == "browser" else ["move"]
 
     # Create OpenAI-compatible agent
-    agent = GenericOpenAIChatAgent(
+    agent = OpenAIChatAgent(
         openai_client=openai_client,
         model_name=model_name,
         allowed_tools=allowed_tools,

examples/run_evaluation.py

@@ -59,13 +59,13 @@
 
 
 def _build_agent(
-    agent_type: Literal[AgentType.CLAUDE, AgentType.OPENAI],
+    agent_type: Literal[AgentType.CLAUDE, AgentType.OPERATOR],
     *,
     model: str | None = None,
     allowed_tools: list[str] | None = None,
 ) -> ClaudeAgent | OperatorAgent:
     """Create and return the requested agent type."""
-    if agent_type == AgentType.OPENAI:
+    if agent_type == AgentType.OPERATOR:
         return OperatorAgent(allowed_tools=allowed_tools, validate_api_key=False)
 
     model = model or "claude-sonnet-4-5-20250929"
@@ -80,7 +80,7 @@ def _build_agent(
 async def run_single_task(
     dataset_name: str,
     *,
-    agent_type: Literal[AgentType.CLAUDE, AgentType.OPENAI] = AgentType.CLAUDE,
+    agent_type: Literal[AgentType.CLAUDE, AgentType.OPERATOR] = AgentType.CLAUDE,
     model: str | None = None,
     allowed_tools: list[str] | None = None,
     max_steps: int = 10,
@@ -111,16 +111,18 @@ async def run_single_task(
 async def run_full_dataset(
     dataset_name: str,
     *,
-    agent_type: Literal[AgentType.CLAUDE, AgentType.OPENAI] = AgentType.CLAUDE,
+    agent_type: Literal[AgentType.CLAUDE, AgentType.OPERATOR] = AgentType.CLAUDE,
     model: str | None = None,
     allowed_tools: list[str] | None = None,
     max_concurrent: int = 50,
     max_steps: int = 10,
 ) -> list[Any]:
     """Run evaluation across entire dataset with asyncio concurrency."""
-    if agent_type == AgentType.OPENAI:
+    if agent_type == AgentType.OPERATOR:
         agent_class = OperatorAgent
-        agent_config: dict[str, Any] = {"validate_api_key": False}
+        agent_config = {"validate_api_key": False}
+        if model:
+            agent_config["model"] = model
         if allowed_tools:
             # Only pass allowed tools if they are provided, otherwise all tools are enabled
             agent_config["allowed_tools"] = allowed_tools
@@ -171,7 +173,7 @@ def parse_args() -> argparse.Namespace:  # type: ignore[valid-type]
     parser.add_argument("--full", action="store_true", help="Run entire dataset")
 
     # Agent
-    parser.add_argument("--agent", choices=["claude", "openai"], default="claude")
+    parser.add_argument("--agent", choices=["claude", "operator"], default="claude")
     parser.add_argument("--model", default=None, help="Model override")
     parser.add_argument(
         "--allowed-tools", dest="allowed_tools", help="Tool allowlist (comma-separated)"

hud/agents/__init__.py

@@ -3,13 +3,15 @@
 from .base import MCPAgent
 from .claude import ClaudeAgent
 from .gemini import GeminiAgent
-from .openai import OperatorAgent
-from .openai_chat_generic import GenericOpenAIChatAgent
+from .openai import OpenAIAgent
+from .openai_chat import OpenAIChatAgent
+from .operator import OperatorAgent
 
 __all__ = [
     "ClaudeAgent",
     "GeminiAgent",
-    "GenericOpenAIChatAgent",
     "MCPAgent",
+    "OpenAIAgent",
+    "OpenAIChatAgent",
     "OperatorAgent",
 ]

hud/agents/grounded_openai.py

@@ -9,10 +9,10 @@
 from hud.tools.grounding import GroundedComputerTool, Grounder, GrounderConfig
 from hud.types import AgentResponse, MCPToolCall, MCPToolResult
 
-from .openai_chat_generic import GenericOpenAIChatAgent
+from .openai_chat import OpenAIChatAgent
 
 
-class GroundedOpenAIChatAgent(GenericOpenAIChatAgent):
+class GroundedOpenAIChatAgent(OpenAIChatAgent):
     """OpenAI agent that uses a separate grounding model for element detection.
 
     This agent:
@@ -45,7 +45,7 @@ def __init__(
             openai_client: OpenAI client for the planning model
             model: Name of the OpenAI model to use for planning (e.g., "gpt-4o", "gpt-4o-mini")
             real_computer_tool_name: Name of the actual computer tool to execute
-            **kwargs: Additional arguments passed to GenericOpenAIChatAgent
+            **kwargs: Additional arguments passed to OpenAIChatAgent
         """
         # Set defaults for grounded agent
         if allowed_tools is None:

hud/agents/lite_llm.py

@@ -11,7 +11,7 @@
 
 import litellm
 
-from .openai_chat_generic import GenericOpenAIChatAgent
+from .openai_chat import OpenAIChatAgent
 
 logger = logging.getLogger(__name__)
 
@@ -24,7 +24,7 @@
     transform_mcp_tool_to_openai_tool = None  # type: ignore
 
 
-class LiteAgent(GenericOpenAIChatAgent):
+class LiteAgent(OpenAIChatAgent):
     """
     Same OpenAI chat-completions shape + MCP tool plumbing,
     but transport is LiteLLM and (optionally) tools are shaped by LiteLLM's MCP transformer.
@@ -55,7 +55,7 @@ def get_tool_schemas(self) -> list[Any]:
                 for t in self.get_available_tools()
             ]
         # Fallback to the generic OpenAI sanitizer
-        return GenericOpenAIChatAgent.get_tool_schemas(self)
+        return OpenAIChatAgent.get_tool_schemas(self)
 
     async def _invoke_chat_completion(
         self,