From d9fd842d6ae7642567b690c9d4b645a00210072b Mon Sep 17 00:00:00 2001 From: Asadullo Ganiev <62354884+solvemproblr@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:33:28 +0500 Subject: [PATCH 1/9] feat(cli): add project placement --- README.md | 10 ++ hud/cli/__init__.py | 2 + hud/cli/deploy.py | 23 +++- hud/cli/project.py | 169 ++++++++++++++++++++++++ hud/cli/sync.py | 29 ++++- hud/cli/tests/test_deploy.py | 92 +++++++++++++ hud/cli/utils/project.py | 194 ++++++++++++++++++++++++++++ hud/cli/utils/source.py | 6 + hud/cli/utils/tests/test_project.py | 178 +++++++++++++++++++++++++ hud/eval/sync.py | 4 + hud/eval/tests/test_sync.py | 35 +++++ hud/settings.py | 7 + 12 files changed, 744 insertions(+), 5 deletions(-) create mode 100644 hud/cli/project.py create mode 100644 hud/cli/utils/project.py create mode 100644 hud/cli/utils/tests/test_project.py diff --git a/README.md b/README.md index f154e6089..a90faf086 100644 --- a/README.md +++ b/README.md @@ -148,6 +148,16 @@ A **capability** is a connection the environment exposes; a **harness** attaches From the [platform UI](https://hud.ai) you can run batches, compare models on the same taskset, and inspect every trace. +A **project** holds the environments and tasksets a team creates and decides who can see them. Deploying without one uses your team's default project, so nothing here is required to get started. To put an environment somewhere else, pin the directory once and every later `hud deploy` and `hud sync tasks` follows it: + +```bash +hud project list # projects you can use +hud project use browser-evals # writes projectId to .hud/config.json +hud project # where does a deploy here land? +``` + +Both commands also take `--project ` to override the resolved Project. A successful deploy or sync pins that Project to the directory; `hud set HUD_PROJECT=` instead sets a machine-wide default for directories you have not pinned. Precedence is the flag, then the directory's `.hud/config.json`, then `HUD_PROJECT`, then your team default. An environment or taskset that already exists stays where it is; naming a different project fails rather than moving it. + → [Run & deploy](https://docs.hud.ai/v6/reference/runtime) ## Train on rewards diff --git a/hud/cli/__init__.py b/hud/cli/__init__.py index 628bc2898..ba676c935 100644 --- a/hud/cli/__init__.py +++ b/hud/cli/__init__.py @@ -37,6 +37,7 @@ from .init import init_command # noqa: E402 from .jobs import jobs_app # noqa: E402 from .models import models_app # noqa: E402 +from .project import project_app # noqa: E402 from .qa import qa_app # noqa: E402 from .serve import serve_command # noqa: E402 from .sync import sync_app # noqa: E402 @@ -52,6 +53,7 @@ app.add_typer(jobs_app, name="jobs") app.add_typer(trace_app, name="trace") app.add_typer(qa_app, name="qa") +app.add_typer(project_app, name="project") @app.command(name="set") diff --git a/hud/cli/deploy.py b/hud/cli/deploy.py index dc9e26d6c..2649a9118 100644 --- a/hud/cli/deploy.py +++ b/hud/cli/deploy.py @@ -19,6 +19,7 @@ from hud.cli.utils.build_logs import poll_build_status, stream_build_logs from hud.cli.utils.config import parse_env_file, parse_key_value from hud.cli.utils.context import create_build_context_tarball, format_size +from hud.cli.utils.project import Placement, resolve_writable_placement from hud.cli.utils.registry import get_registry_environment from hud.cli.utils.source import EnvironmentSource from hud.eval.runtime import ComposeProject, RuntimeConfig @@ -41,6 +42,7 @@ class _DeployPlan: name: str registry_id: str | None + placement: Placement runtime: str | None runtime_config: RuntimeConfig | None env_vars: dict[str, str] @@ -375,6 +377,7 @@ def _prepare_deploy_plan( env_file: str | None, no_env: bool, registry_id: str | None, + project: str | None, build_args: list[str] | None, build_secrets: list[str] | None, runtime: str | None, @@ -390,6 +393,7 @@ def _prepare_deploy_plan( platform, console, ) + placement = resolve_writable_placement(platform, env_source, flag=project, console=console) skip_dotenv = _skip_dotenv( env_source, env_dir, @@ -437,6 +441,7 @@ def _prepare_deploy_plan( return _DeployPlan( name=resolved_name, registry_id=registry_id, + placement=placement, runtime=normalized_runtime, runtime_config=loaded_runtime_config, env_vars=env_vars, @@ -453,6 +458,7 @@ def deploy_environment( no_cache: bool = False, verbose: bool = False, registry_id: str | None = None, + project: str | None = None, build_args: list[str] | None = None, build_secrets: list[str] | None = None, runtime: str | None = None, @@ -491,6 +497,7 @@ def deploy_environment( env_file=env_file, no_env=no_env, registry_id=registry_id, + project=project, build_args=build_args, build_secrets=build_secrets, runtime=runtime, @@ -555,6 +562,7 @@ async def _trigger_build( key: value for key, value in ( ("registry_id", plan.registry_id), + ("project_id", plan.placement.project_id), ("runtime_provider", plan.runtime), ( "runtime_config", @@ -634,7 +642,7 @@ async def _deploy_async( # Save immediately after trigger so rebuilds work even if streaming crashes. if env_dir and registry_id: - _save_deploy_link(env_dir, registry_id, console, env_name=plan.name) + _save_deploy_link(env_dir, registry_id, console, env_name=plan.name, plan=plan) console.success(f"Build triggered [{time.time() - step_start:.1f}s]") console.info(f"Build ID: {build_id}") @@ -682,12 +690,15 @@ def _save_deploy_link( registry_id: str, console: HUDConsole, env_name: str | None = None, + plan: _DeployPlan | None = None, ) -> None: """Save deploy linking info to .hud/config.json.""" try: config_data: dict[str, Any] = {"registryId": registry_id} if env_name: config_data["registryName"] = env_name + if plan is not None and plan.placement.project is not None: + config_data["projectId"] = plan.placement.project.id changed = EnvironmentSource.open(env_dir).save_config(config_data) console.success(f"Linked to environment: {registry_id[:8]}...") if changed: @@ -715,6 +726,7 @@ def deploy_all( no_env: bool = False, no_cache: bool = False, verbose: bool = False, + project: str | None = None, build_args: list[str] | None = None, build_secrets: list[str] | None = None, runtime: str | None = None, @@ -755,6 +767,7 @@ def deploy_all( no_cache=no_cache, verbose=verbose, registry_id=None, + project=project, build_args=build_args, build_secrets=build_secrets, runtime=runtime, @@ -833,6 +846,12 @@ def deploy_command( help="Existing registry ID for rebuilds (advanced)", hidden=True, ), + project: str | None = typer.Option( + None, + "--project", + help="Project to create this environment in (name or ID). Defaults to the " + "directory's saved project, then HUD_PROJECT, then your team default.", + ), runtime: str | None = typer.Option( None, "--runtime", @@ -859,6 +878,7 @@ def deploy_command( no_env=no_env, no_cache=no_cache, verbose=verbose, + project=project, build_args=build_args, build_secrets=secrets, runtime=runtime, @@ -874,6 +894,7 @@ def deploy_command( no_cache=no_cache, verbose=verbose, registry_id=registry_id, + project=project, build_args=build_args, build_secrets=secrets, runtime=runtime, diff --git a/hud/cli/project.py b/hud/cli/project.py new file mode 100644 index 000000000..3189d9ffb --- /dev/null +++ b/hud/cli/project.py @@ -0,0 +1,169 @@ +"""``hud project`` — see and choose the Project new environments land in.""" + +from __future__ import annotations + +import httpx +import typer + +from hud.cli.utils.api import require_api_key +from hud.cli.utils.project import ( + Project, + ProjectNotFound, + ProjectNotWritable, + list_projects, + report_project_error, + resolve_placement, + resolve_project, +) +from hud.cli.utils.source import EnvironmentSource +from hud.utils.exceptions import HudRequestError +from hud.utils.hud_console import HUDConsole +from hud.utils.platform import PlatformClient + +project_app = typer.Typer( + name="project", + help="Show and choose the HUD Project new environments and tasksets land in", + add_completion=False, + rich_markup_mode="rich", +) + + +@project_app.command("list") +def list_command() -> None: + """List the Projects you can see. + + [not dim]Examples: + hud project list[/not dim] + """ + console = HUDConsole() + require_api_key("list projects") + + try: + projects = list_projects(PlatformClient.from_settings()) + except HudRequestError as e: + raise report_project_error(console, e) from e + + if not projects: + console.warning("No projects found") + console.hint("Create one with: hud project create ") + return + + console.info("Your projects:") + for project in sorted(projects, key=lambda p: (not p.is_default, p.name)): + tags: list[str] = [] + if project.is_default: + tags.append("default") + if not project.can_create: + tags.append("read-only") + suffix = f" [{', '.join(tags)}]" if tags else "" + console.info(f" {project.name} ({project.short_id}...){suffix}") + + +@project_app.command("create") +def create_command( + name: str = typer.Argument(..., help="Name for the new project"), + description: str | None = typer.Option(None, "--description", help="What the project holds"), + directory: str = typer.Option(".", "--directory", "-C", help="Directory to pin it to"), + no_use: bool = typer.Option(False, "--no-use", help="Create without pinning this directory"), +) -> None: + """Create a Project and pin this directory to it. + + [not dim]Only team admins can create projects. + + Examples: + hud project create browser-evals + hud project create browser-evals --no-use[/not dim] + """ + console = HUDConsole() + require_api_key("create a project") + + platform = PlatformClient.from_settings() + payload: dict[str, str] = {"name": name} + if description: + payload["description"] = description + + try: + created = Project.from_record(platform.post("/projects", json=payload)) + except HudRequestError as e: + if e.status_code == httpx.codes.CONFLICT: + console.error(f"A project named '{name}' already exists") + console.hint(f"Pin this directory to it with: hud project use {name}") + raise typer.Exit(1) from e + if e.status_code == httpx.codes.FORBIDDEN: + console.error("Only team admins can create projects") + raise typer.Exit(1) from e + raise report_project_error(console, e) from e + + console.success(f"Created project: {created.name} ({created.short_id}...)") + if not no_use: + _pin(created, directory, console) + + +@project_app.command("use") +def use_command( + ref: str = typer.Argument(..., help="Project name or ID"), + directory: str = typer.Option(".", "--directory", "-C", help="Directory to pin"), +) -> None: + """Pin a directory to a Project. + + [not dim]Writes projectId to .hud/config.json, so teammates deploying this + environment place it in the same project. Set a machine-wide fallback for + unpinned directories with: hud set HUD_PROJECT= + + Examples: + hud project use browser-evals + hud project use browser-evals -C ./envs/browser[/not dim] + """ + console = HUDConsole() + require_api_key("select a project") + + try: + project = resolve_project(PlatformClient.from_settings(), ref) + except (ProjectNotFound, HudRequestError) as e: + raise report_project_error(console, e) from e + if not project.can_create: + raise report_project_error(console, ProjectNotWritable(project)) + _pin(project, directory, console) + + +@project_app.callback(invoke_without_command=True) +def project_callback( + ctx: typer.Context, + directory: str = typer.Option(".", "--directory", "-C", help="Directory to report on"), +) -> None: + """Show the Project this directory places new environments and tasksets in. + + [not dim]Examples: + hud project # where does a deploy here land? + hud project list # projects you can see + hud project use browser-evals # pin this directory[/not dim] + """ + if ctx.invoked_subcommand is not None: + return + + console = HUDConsole() + require_api_key("resolve the current project") + try: + placement = resolve_placement( + PlatformClient.from_settings(), + EnvironmentSource.open(directory), + flag=None, + ) + except (ProjectNotFound, HudRequestError) as e: + raise report_project_error(console, e) from e + + console.info(f"Project: {placement.label}") + if placement.project is None: + console.hint("Pin a different one with: hud project use ") + elif not placement.project.can_create: + console.warning("You do not have create access to this Project") + + +def _pin(project: Project, directory: str, console: HUDConsole) -> None: + changed = EnvironmentSource.open(directory).save_config({"projectId": project.id}) + console.success(f"Using project: {project.name} ({project.short_id}...)") + if changed: + console.dim_info("Config saved to:", ".hud/config.json") + + +__all__ = ["project_app"] diff --git a/hud/cli/sync.py b/hud/cli/sync.py index fab060487..26f3e9d24 100644 --- a/hud/cli/sync.py +++ b/hud/cli/sync.py @@ -11,6 +11,7 @@ import typer from hud.cli.utils.api import require_api_key +from hud.cli.utils.project import Placement, resolve_writable_placement from hud.cli.utils.registry import ( RegistryEnvironment, get_registry_environment, @@ -220,11 +221,14 @@ def _show_upload_error(error: HudRequestError, console: HUDConsole) -> None: console.error(f"Upload failed ({error.status_code}): {detail or error}") -def _save_taskset_id(result: dict[str, object], console: HUDConsole) -> None: +def _save_taskset_id(result: dict[str, object], placement: Placement, console: HUDConsole) -> None: returned_id = result.get("taskset_id") if not isinstance(returned_id, str) or not returned_id: return - changed = EnvironmentSource.open().save_config({"tasksetId": returned_id}) + config: dict[str, object] = {"tasksetId": returned_id} + if placement.project is not None: + config["projectId"] = placement.project.id + changed = EnvironmentSource.open().save_config(config) if changed: console.dim_info("Taskset ID saved to:", ".hud/config.json") from hud.settings import settings @@ -247,6 +251,12 @@ def sync_tasks_command( "--id", help="Taskset ID directly (skip name resolution)", ), + project: str | None = typer.Option( + None, + "--project", + help="Project to create this taskset in (name or ID). Defaults to the " + "directory's saved project, then HUD_PROJECT, then your team default.", + ), task_filter: str | None = typer.Option( None, "--task", @@ -318,6 +328,12 @@ def sync_tasks_command( # Creating a new taskset is only allowed when targeting an explicit name # (not an --id or a stored id, which must already exist). allow_create = taskset is not None and taskset_id is None + placement = resolve_writable_placement( + platform, + EnvironmentSource.open(), + flag=project, + console=hud_console, + ) try: remote_taskset = _fetch_remote_taskset( @@ -355,7 +371,12 @@ def sync_tasks_command( # Upload tasks; the platform validates referenced environments. hud_console.progress_message("Uploading tasks...") try: - result = upload_taskset(platform, plan.taskset_name, plan.to_apply) + result = upload_taskset( + platform, + plan.taskset_name, + plan.to_apply, + project_id=placement.project_id, + ) except HudRequestError as e: _show_upload_error(e, hud_console) return @@ -365,7 +386,7 @@ def sync_tasks_command( hud_console.success("Sync complete") hud_console.info(f" + {created} created, ~ {updated} updated") - _save_taskset_id(result, hud_console) + _save_taskset_id(result, placement, hud_console) @sync_app.command("env") diff --git a/hud/cli/tests/test_deploy.py b/hud/cli/tests/test_deploy.py index e915092fe..61fe384fd 100644 --- a/hud/cli/tests/test_deploy.py +++ b/hud/cli/tests/test_deploy.py @@ -10,11 +10,15 @@ import typer from hud.cli.deploy import _resolve_environment_name +from hud.cli.utils.project import Placement, Project, ProjectSource from hud.cli.utils.registry import RegistryEnvironment from hud.cli.utils.source import EnvironmentSource from hud.utils.hud_console import HUDConsole from hud.utils.platform import PlatformClient +# Deploys that accept the team's default Project send no project_id. +_UNPLACED = Placement(project=None, source=ProjectSource.TEAM_DEFAULT) + @pytest.mark.parametrize(("value", "expected"), [("HUD", "hud"), ("modal", "modal")]) def test_normalize_runtime_uses_public_runtime_names(value: str, expected: str) -> None: @@ -232,6 +236,7 @@ def test_prepare_deploy_uses_context_recipe( env_file=None, no_env=True, registry_id=None, + project=None, build_args=None, build_secrets=None, runtime=None, @@ -331,6 +336,7 @@ def test_prepare_deploy_rejects_image_config_for_compose_context( env_file=None, no_env=True, registry_id=None, + project=None, build_args=None, build_secrets=None, runtime=None, @@ -450,6 +456,7 @@ async def test_upload_url_failure(self) -> None: plan=_DeployPlan( name="test-env", registry_id=None, + placement=_UNPLACED, runtime=None, runtime_config=None, env_vars={}, @@ -481,6 +488,7 @@ async def test_upload_url_network_error(self) -> None: plan=_DeployPlan( name="test-env", registry_id=None, + placement=_UNPLACED, runtime=None, runtime_config=None, env_vars={}, @@ -493,6 +501,90 @@ async def test_upload_url_network_error(self) -> None: assert result.success is False + @pytest.mark.asyncio + async def test_trigger_build_sends_resolved_project(self) -> None: + """A resolved placement reaches the platform as project_id.""" + from hud.cli.deploy import _DeployPlan, _trigger_build + from hud.utils.platform import PlatformClient + + class FakePlatform(PlatformClient): + payload: dict[str, object] | None = None + + async def apost( + self, + path: str, + *, + json: object | None = None, + ) -> dict[str, object]: + object.__setattr__(self, "payload", json) + return {"id": "build-1", "registry_id": "registry-1"} + + platform = FakePlatform("https://api.example", "key") + await _trigger_build( + platform, + build_id="build-1", + plan=_DeployPlan( + name="test-env", + registry_id=None, + placement=Placement( + project=Project( + id="project-1", + name="browser-evals", + is_default=False, + can_create=True, + ), + source=ProjectSource.FLAG, + ), + runtime=None, + runtime_config=None, + env_vars={}, + build_args={}, + build_secrets={}, + ), + no_cache=False, + ) + + assert platform.payload is not None + assert platform.payload["project_id"] == "project-1" + + @pytest.mark.asyncio + async def test_trigger_build_omits_project_for_the_team_default(self) -> None: + """The zero-config deploy stays byte-identical to before projects existed.""" + from hud.cli.deploy import _DeployPlan, _trigger_build + from hud.utils.platform import PlatformClient + + class FakePlatform(PlatformClient): + payload: dict[str, object] | None = None + + async def apost( + self, + path: str, + *, + json: object | None = None, + ) -> dict[str, object]: + object.__setattr__(self, "payload", json) + return {"id": "build-1", "registry_id": "registry-1"} + + platform = FakePlatform("https://api.example", "key") + await _trigger_build( + platform, + build_id="build-1", + plan=_DeployPlan( + name="test-env", + registry_id=None, + placement=_UNPLACED, + runtime=None, + runtime_config=None, + env_vars={}, + build_args={}, + build_secrets={}, + ), + no_cache=False, + ) + + assert platform.payload is not None + assert "project_id" not in platform.payload + class TestSaveDeployLink: """Tests for _save_deploy_link function.""" diff --git a/hud/cli/utils/project.py b/hud/cli/utils/project.py new file mode 100644 index 000000000..b2484b63a --- /dev/null +++ b/hud/cli/utils/project.py @@ -0,0 +1,194 @@ +"""Project lookup and placement resolution for the CLI.""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass +from enum import Enum +from typing import TYPE_CHECKING, Any + +import typer + +from hud.utils.naming import normalize_environment_name + +if TYPE_CHECKING: + from hud.cli.utils.source import EnvironmentSource + from hud.utils.hud_console import HUDConsole + from hud.utils.platform import PlatformClient + + +class ProjectSource(Enum): + """Where a resolved Project came from, most specific first.""" + + FLAG = "--project" + CONFIG = ".hud/config.json" + SETTINGS = "HUD_PROJECT" + TEAM_DEFAULT = "team default" + + +@dataclass(frozen=True) +class Project: + id: str + name: str + is_default: bool + can_create: bool + + @classmethod + def from_record(cls, data: dict[str, Any]) -> Project: + capabilities = data.get("capabilities") + return cls( + id=str(data["id"]), + name=str(data.get("name") or "unnamed"), + is_default=bool(data.get("is_default")), + can_create=bool(capabilities.get("create")) + if isinstance(capabilities, dict) + else False, + ) + + @property + def short_id(self) -> str: + return self.id[:8] + + +@dataclass(frozen=True) +class Placement: + """The Project selected for the current directory.""" + + project: Project | None + source: ProjectSource + + @property + def project_id(self) -> str | None: + """The id to send to the platform, or None to accept the team default.""" + return self.project.id if self.project else None + + @property + def label(self) -> str: + if self.project is None: + return "team default Project" + return f"{self.project.name} (via {self.source.value})" + + +class ProjectNotFound(LookupError): + """No visible Project matches the given reference.""" + + def __init__(self, ref: str, available: list[Project]) -> None: + self.ref = ref + self.available = available + super().__init__(f"No project found matching '{ref}'") + + +class ProjectNotWritable(PermissionError): + """The caller may see the Project but may not create resources in it.""" + + def __init__(self, project: Project) -> None: + self.project = project + super().__init__( + f"You do not have permission to create environments or tasksets in " + f"project '{project.name}'" + ) + + +def list_projects(platform: PlatformClient) -> list[Project]: + """Every Project visible to the caller.""" + data = platform.get("/projects") + records = data.get("projects") if isinstance(data, dict) else None + if not isinstance(records, list): + return [] + return [Project.from_record(item) for item in records if isinstance(item, dict)] + + +def resolve_project(platform: PlatformClient, ref: str) -> Project: + """Map a Project name or id to the Project itself. + + Names are normalized the same way the platform normalizes them on create, + so `My Project` and `my-project` resolve to the same row. + """ + projects = list_projects(platform) + try: + project_id = str(uuid.UUID(ref)) + except ValueError: + project_id = None + + match = next((p for p in projects if p.id == project_id), None) + if match is None: + name = normalize_environment_name(ref, default="") + match = next((p for p in projects if p.name == name), None) + + if match is None: + raise ProjectNotFound(ref, projects) + return match + + +def resolve_placement( + platform: PlatformClient, + env_source: EnvironmentSource, + *, + flag: str | None, +) -> Placement: + """Resolve the configured Project.""" + from hud.settings import settings + + for ref, source in ( + (flag, ProjectSource.FLAG), + (env_source.project_id, ProjectSource.CONFIG), + (settings.project, ProjectSource.SETTINGS), + ): + if ref: + project = resolve_project(platform, ref) + return Placement(project=project, source=source) + + return Placement(project=None, source=ProjectSource.TEAM_DEFAULT) + + +def report_project_error(console: HUDConsole, error: Exception) -> typer.Exit: + """Explain why a Project could not be used, and return the exit to raise.""" + if isinstance(error, ProjectNotFound): + console.error(str(error)) + if error.available: + console.info("Projects you can see:") + for candidate in error.available: + console.info(f" {candidate.name} ({candidate.short_id}...)") + else: + console.hint("Create one with: hud project create ") + elif isinstance(error, ProjectNotWritable): + console.error(str(error)) + console.hint("Ask a project manager for 'create' scope, or pick another project") + else: + console.error(f"Failed to reach the HUD platform: {error}") + return typer.Exit(1) + + +def resolve_writable_placement( + platform: PlatformClient, + env_source: EnvironmentSource, + *, + flag: str | None, + console: HUDConsole, +) -> Placement: + """Resolve and announce a Project that accepts new resources.""" + from hud.utils.exceptions import HudRequestError + + try: + placement = resolve_placement(platform, env_source, flag=flag) + if placement.project is not None and not placement.project.can_create: + raise ProjectNotWritable(placement.project) + except (ProjectNotFound, ProjectNotWritable, HudRequestError) as e: + raise report_project_error(console, e) from e + + console.info(f"Project: {placement.label}") + return placement + + +__all__ = [ + "Placement", + "Project", + "ProjectNotFound", + "ProjectNotWritable", + "ProjectSource", + "list_projects", + "report_project_error", + "resolve_placement", + "resolve_project", + "resolve_writable_placement", +] diff --git a/hud/cli/utils/source.py b/hud/cli/utils/source.py index 33fcbaac3..a7638828a 100644 --- a/hud/cli/utils/source.py +++ b/hud/cli/utils/source.py @@ -210,6 +210,12 @@ def taskset_id(self) -> str | None: value = self.load_config().get("tasksetId") return value if isinstance(value, str) else None + @property + def project_id(self) -> str | None: + """The Project this directory's environment and taskset belong to.""" + value = self.load_config().get("projectId") + return value if isinstance(value, str) else None + def iter_source_files(self) -> Iterator[Path]: for name in self.SOURCE_INCLUDE_FILES: path = self.root / name diff --git a/hud/cli/utils/tests/test_project.py b/hud/cli/utils/tests/test_project.py new file mode 100644 index 000000000..2eb6cc1bd --- /dev/null +++ b/hud/cli/utils/tests/test_project.py @@ -0,0 +1,178 @@ +"""Project lookup and placement precedence for CLI create-and-link flows.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +import pytest +import typer + +from hud.cli.utils.project import ( + Project, + ProjectNotFound, + ProjectSource, + resolve_placement, + resolve_project, + resolve_writable_placement, +) +from hud.cli.utils.source import EnvironmentSource +from hud.utils.hud_console import HUDConsole +from hud.utils.platform import PlatformClient + +if TYPE_CHECKING: + from pathlib import Path + +_DEFAULT_ID = "11111111-1111-4111-8111-111111111111" +_BROWSER_ID = "22222222-2222-4222-8222-222222222222" +_READONLY_ID = "33333333-3333-4333-8333-333333333333" + + +def _record( + project_id: str, name: str, *, is_default: bool = False, create: bool = True +) -> dict[str, Any]: + return { + "id": project_id, + "name": name, + "is_default": is_default, + "capabilities": {"view": True, "create": create, "manage": False}, + } + + +@pytest.fixture +def calls() -> list[str]: + """URLs the fake platform transport was asked for.""" + return [] + + +@pytest.fixture +def platform(monkeypatch: pytest.MonkeyPatch, calls: list[str]) -> PlatformClient: + """A client whose ``GET /projects`` returns a fixed three-project team.""" + + def fake_request(method: str, url: str, **kwargs: Any) -> dict[str, Any]: + calls.append(url) + return { + "projects": [ + _record(_DEFAULT_ID, "default", is_default=True), + _record(_BROWSER_ID, "browser-evals"), + _record(_READONLY_ID, "locked-down", create=False), + ] + } + + monkeypatch.setattr("hud.utils.platform.make_request_sync", fake_request) + return PlatformClient("https://api.example", "key") + + +def _no_settings_project(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("hud.settings.settings.project", None) + + +def test_resolve_matches_a_normalized_name(platform: PlatformClient) -> None: + """A human-typed name resolves through the same normalization the platform applies.""" + assert resolve_project(platform, "Browser Evals").id == _BROWSER_ID + assert resolve_project(platform, "browser-evals").id == _BROWSER_ID + + +def test_resolve_matches_an_id(platform: PlatformClient) -> None: + assert resolve_project(platform, _BROWSER_ID).name == "browser-evals" + assert resolve_project(platform, _BROWSER_ID.upper()).name == "browser-evals" + + +def test_resolve_reports_the_visible_alternatives(platform: PlatformClient) -> None: + with pytest.raises(ProjectNotFound) as excinfo: + resolve_project(platform, "nope") + + assert [p.name for p in excinfo.value.available] == [ + "default", + "browser-evals", + "locked-down", + ] + + +def test_flag_outranks_directory_config( + platform: PlatformClient, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _no_settings_project(monkeypatch) + source = EnvironmentSource.open(tmp_path) + source.save_config({"projectId": _DEFAULT_ID}) + + placement = resolve_placement(platform, source, flag="browser-evals") + + assert placement.project is not None + assert placement.project.id == _BROWSER_ID + assert placement.source is ProjectSource.FLAG + + +def test_directory_config_outranks_the_machine_default( + platform: PlatformClient, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Placement is a property of the environment, not of who deploys it.""" + monkeypatch.setattr("hud.settings.settings.project", "default") + source = EnvironmentSource.open(tmp_path) + source.save_config({"projectId": _BROWSER_ID}) + + placement = resolve_placement(platform, source, flag=None) + + assert placement.project is not None + assert placement.project.id == _BROWSER_ID + assert placement.source is ProjectSource.CONFIG + + +def test_machine_default_applies_to_an_unpinned_directory( + platform: PlatformClient, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr("hud.settings.settings.project", "browser-evals") + + placement = resolve_placement(platform, EnvironmentSource.open(tmp_path), flag=None) + + assert placement.project is not None + assert placement.project.id == _BROWSER_ID + assert placement.source is ProjectSource.SETTINGS + + +def test_unconfigured_placement_sends_no_project_and_makes_no_call( + platform: PlatformClient, + calls: list[str], + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The zero-config path stays free: no project on the wire, no lookup.""" + _no_settings_project(monkeypatch) + + placement = resolve_placement(platform, EnvironmentSource.open(tmp_path), flag=None) + + assert placement.project_id is None + assert placement.source is ProjectSource.TEAM_DEFAULT + assert placement.label == "team default Project" + assert calls == [] + + +def test_placement_resolves_a_project_the_caller_cannot_create_in( + platform: PlatformClient, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _no_settings_project(monkeypatch) + + source = EnvironmentSource.open(tmp_path) + placement = resolve_placement(platform, source, flag="locked-down") + assert placement.project is not None + assert placement.project.id == _READONLY_ID + + with pytest.raises(typer.Exit): + resolve_writable_placement( + platform, + source, + flag="locked-down", + console=HUDConsole(), + ) + + +def test_from_record_defaults_capabilities_to_read_only() -> None: + """A response without capabilities is not assumed writable.""" + assert Project.from_record({"id": "x", "name": "y"}).can_create is False diff --git a/hud/eval/sync.py b/hud/eval/sync.py index 67a588334..c90e7438e 100644 --- a/hud/eval/sync.py +++ b/hud/eval/sync.py @@ -135,12 +135,16 @@ def upload_taskset( platform: PlatformClient, name: str, tasks: list[Task], + *, + project_id: str | None = None, ) -> dict[str, Any]: """Upload tasks to a platform taskset, creating it if needed.""" payload: dict[str, Any] = { "taskset_name": name, "tasks": [task_upload_payload(task) for task in tasks], } + if project_id: + payload["project_id"] = project_id data = platform.post("/tasks/upload", json=payload) return data if isinstance(data, dict) else {} diff --git a/hud/eval/tests/test_sync.py b/hud/eval/tests/test_sync.py index 0b43e4a9c..267a80b5f 100644 --- a/hud/eval/tests/test_sync.py +++ b/hud/eval/tests/test_sync.py @@ -135,6 +135,41 @@ def fake_request( } +def test_upload_taskset_places_a_new_taskset(monkeypatch: pytest.MonkeyPatch) -> None: + """`project_id` reaches the platform so a created taskset lands in that Project.""" + posted: dict[str, Any] = {} + + def fake_request(method: str, url: str, json: object = None, **kwargs: Any) -> dict[str, Any]: + posted.update(json=json) + return {} + + monkeypatch.setattr("hud.utils.platform.make_request_sync", fake_request) + + upload_taskset( + PlatformClient("https://api.example", "token"), + "demo", + [], + project_id="project-1", + ) + + assert posted["json"]["project_id"] == "project-1" + + +def test_upload_taskset_omits_project_when_unset(monkeypatch: pytest.MonkeyPatch) -> None: + """Without a project the payload is unchanged, so the default Project applies.""" + posted: dict[str, Any] = {} + + def fake_request(method: str, url: str, json: object = None, **kwargs: Any) -> dict[str, Any]: + posted.update(json=json) + return {} + + monkeypatch.setattr("hud.utils.platform.make_request_sync", fake_request) + + upload_taskset(PlatformClient("https://api.example", "token"), "demo", []) + + assert "project_id" not in posted["json"] + + def test_task_upload_payload_sends_env_and_bare_task_id() -> None: payload = task_upload_payload(Task(env="e", id="solve", args={"n": 1})) diff --git a/hud/settings.py b/hud/settings.py index 27d896b32..f2ce7068a 100644 --- a/hud/settings.py +++ b/hud/settings.py @@ -92,6 +92,13 @@ def settings_customise_sources( validation_alias="HUD_API_KEY", ) + project: str | None = Field( + default=None, + description="Default HUD Project (name or id) for environments and tasksets this " + "machine creates. A directory's .hud/config.json takes precedence.", + validation_alias="HUD_PROJECT", + ) + anthropic_api_key: str | None = Field( default=None, description="API key for Anthropic models", From d0f336651941f8ff95fd0ea8170908118ade0049 Mon Sep 17 00:00:00 2001 From: Asadullo Ganiev <62354884+solvemproblr@users.noreply.github.com> Date: Sat, 5 Sep 2026 17:40:41 +0800 Subject: [PATCH 2/9] docs(projects): document project CLI workflows --- README.md | 2 +- docs/docs.json | 2 +- docs/v6/guides/creating-an-environment.mdx | 13 +++++ docs/v6/reference/cli.mdx | 59 ++++++++++++++++++++-- docs/v6/reference/projects.mdx | 37 ++++++++++++++ docs/v6/reference/tasks.mdx | 13 +++++ 6 files changed, 121 insertions(+), 5 deletions(-) create mode 100644 docs/v6/reference/projects.mdx diff --git a/README.md b/README.md index a90faf086..660da3ac4 100644 --- a/README.md +++ b/README.md @@ -158,7 +158,7 @@ hud project # where does a deploy here land? Both commands also take `--project ` to override the resolved Project. A successful deploy or sync pins that Project to the directory; `hud set HUD_PROJECT=` instead sets a machine-wide default for directories you have not pinned. Precedence is the flag, then the directory's `.hud/config.json`, then `HUD_PROJECT`, then your team default. An environment or taskset that already exists stays where it is; naming a different project fails rather than moving it. -→ [Run & deploy](https://docs.hud.ai/v6/reference/runtime) +→ [Projects](https://docs.hud.ai/v6/reference/projects) · [Run & deploy](https://docs.hud.ai/v6/reference/runtime) ## Train on rewards diff --git a/docs/docs.json b/docs/docs.json index 483742155..24febc806 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -64,7 +64,7 @@ "groups": [ { "group": "Start here", "pages": ["v6/start/index", "v6/start/quickstart", "v6/start/overview"] }, { "group": "Guides", "pages": ["v6/guides/creating-an-environment", "v6/guides/running-an-eval", "v6/guides/training-agents"] }, - { "group": "Reference", "pages": ["v6/reference/environment", "v6/reference/tasks", "v6/reference/capabilities", "v6/reference/agents", "v6/reference/runtime", "v6/reference/graders", "v6/reference/advice", "v6/reference/training", "v6/reference/types", "v6/reference/cli", "v6/reference/telemetry"] }, + { "group": "Reference", "pages": ["v6/reference/environment", "v6/reference/tasks", "v6/reference/projects", "v6/reference/capabilities", "v6/reference/agents", "v6/reference/runtime", "v6/reference/graders", "v6/reference/advice", "v6/reference/training", "v6/reference/types", "v6/reference/cli", "v6/reference/telemetry"] }, { "group": "Advanced", "pages": [ { "group": "Advanced", "expanded": false, "pages": ["v6/advanced/extending", "v6/advanced/robots"] } ] }, diff --git a/docs/v6/guides/creating-an-environment.mdx b/docs/v6/guides/creating-an-environment.mdx index d971cc6c5..9d6ddb4d0 100644 --- a/docs/v6/guides/creating-an-environment.mdx +++ b/docs/v6/guides/creating-an-environment.mdx @@ -289,10 +289,23 @@ hud set HUD_API_KEY=your-key # get one at hud.ai hud deploy ``` +The environment is created in the directory's resolved [HUD Project](/v6/reference/projects), or in +the team default Project when none is configured. Select another Project persistently or on this +deploy: + +```bash +hud project use browser-evals +hud deploy + +# Or select it directly; a successful deploy saves the resolved Project +hud deploy --project browser-evals +``` + To run a taskset from the platform, publish it too. `hud sync` uploads a taskset and only what changed: ```bash hud sync tasks my-tasks # publish tasks.py as a named taskset +hud sync tasks my-tasks --project browser-evals hud sync env # sync environment metadata ``` diff --git a/docs/v6/reference/cli.mdx b/docs/v6/reference/cli.mdx index d9d4a2627..bc1b2aab0 100644 --- a/docs/v6/reference/cli.mdx +++ b/docs/v6/reference/cli.mdx @@ -1,6 +1,6 @@ --- title: "CLI" -description: "Reference for the hud command-line interface: init, dev, run, eval, models, and other commands that span the HUD environment and agent lifecycle." +description: "Reference for the hud command-line interface: init, deploy, projects, sync, eval, and other commands across the HUD lifecycle." icon: "terminal" --- @@ -64,6 +64,7 @@ rebuilds that environment. ```bash hud deploy +hud deploy --project browser-evals ``` | Option | Description | @@ -71,6 +72,7 @@ hud deploy | `--all`, `-a` | Deploy all environments in the directory. | | `--env`, `-e` | Env var `KEY=VALUE` (repeatable). | | `--env-file` | Path to a `.env` file. | +| `--project` | Project name or ID for placement. Falls back to the directory Project, `HUD_PROJECT`, then the team default. | ## Evaluate @@ -140,11 +142,62 @@ hud task grade fix_bug --source tasks.py --url tcp://127.0.0.1:8765 --answer ".. ## Platform +### `hud project` + +Show and choose the [HUD Project](/v6/reference/projects) where new environments and tasksets are +created. + ```bash -hud sync tasks my-taskset # publish tasks as a named taskset -hud sync env # sync environment metadata +hud project # resolved Project for the current directory +hud project list # visible Projects and access +hud project create browser-evals # create and use a Project (team admins) +hud project create browser-evals --no-use +hud project use browser-evals # save projectId to .hud/config.json +hud project use browser-evals -C ./my-env ``` +| Command or option | Description | +|-------------------|-------------| +| `hud project [--directory PATH]` | Show where new resources from a directory will be created. `-C` is an alias. | +| `hud project list` | List visible Projects, marking the team default and read-only entries. | +| `hud project create ` | Create and pin a Project. Accepts `--description`, `--directory`/`-C`, and `--no-use`. | +| `hud project use ` | Pin a directory to an existing writable Project. Accepts `--directory`/`-C`. | + +#### Project selection + +`hud deploy` and `hud sync tasks` resolve their destination in this order: + +| Priority | Source | Scope | +|----------|--------|-------| +| 1 | `--project ` | Current command | +| 2 | `.hud/config.json` → `projectId` | Current directory | +| 3 | `HUD_PROJECT` | Machine or process | +| 4 | Team default Project | Zero-configuration fallback | + +Set the machine-wide fallback with `hud set HUD_PROJECT=`. When a named Project is +resolved, a successful deploy or task sync saves its ID to the directory's `.hud/config.json`. + +`hud project list` marks the team default and read-only Projects. Pinning a directory, deploying, or +syncing to a Project requires create access. Only team admins can create Projects. + +An existing environment or taskset stays in its original Project. Selecting a different Project for +an existing resource fails rather than moving it; use its current Project or create a separately named +resource in the new one. + +### `hud sync` + +Publish task definitions or link a local directory to a deployed environment: + +```bash +hud sync tasks my-taskset # publish using resolved Project +hud sync tasks my-taskset --project browser-evals # override by Project name or ID +hud sync env # link environment metadata +``` + +`hud sync tasks --project` follows the same resolution and persistence rules as +`hud deploy --project`. Existing resources remain in their original Project. See +[Projects](/v6/reference/projects) for the conceptual overview. + External benchmark formats can be adapted into runnable `Taskset`s through the experimental [Harbor integration](/v6/experimental/harbor). diff --git a/docs/v6/reference/projects.mdx b/docs/v6/reference/projects.mdx new file mode 100644 index 000000000..1f8ddf98e --- /dev/null +++ b/docs/v6/reference/projects.mdx @@ -0,0 +1,37 @@ +--- +title: "Projects" +description: "Understand how HUD Projects organize environments and tasksets." +icon: "folder-tree" +--- + +A HUD **Project** is the workspace boundary for a related set of environments and tasksets. Projects +also control who can see those resources and who can create new ones. + +## Default Project + +Every team has a default Project. If you deploy an environment or publish a taskset without choosing +another Project, HUD places it in the team default. This keeps the normal workflow zero-configuration. + +## Project selection + +A local environment directory can be associated with a Project. Deploys and task syncs from that +directory then use the same destination, keeping an environment and its tasksets together. + +For automation or exceptional cases, a command can select a different destination explicitly. HUD +also supports a machine-wide fallback for directories without an association. + +Project selection only controls where a **new** resource is created. It does not move resources +between Projects. + +## Resource ownership + +Each deployed environment or taskset belongs to one Project. If you need the same resource in another +Project, create it there under a separate name rather than changing the Project of the existing +resource. + +Projects may be writable or read-only for a given user. Only team admins can create Projects. + + + + + diff --git a/docs/v6/reference/tasks.mdx b/docs/v6/reference/tasks.mdx index 7a21ec070..43fb6a4c6 100644 --- a/docs/v6/reference/tasks.mdx +++ b/docs/v6/reference/tasks.mdx @@ -186,6 +186,19 @@ uploads a taskset and only what changed, a workflow covered in [creating an environment](/v6/guides/creating-an-environment#deploying-to-the-platform). In code, `diff(local, remote)` returns a `SyncPlan` describing the comparison: +To create the taskset outside the team default Project, select a directory Project first or pass an +explicit destination: + +```bash +hud project use browser-evals +hud sync tasks my-taskset + +# Or select it on the sync command +hud sync tasks my-taskset --project browser-evals +``` + +See [Projects](/v6/reference/projects) for placement precedence and existing-resource behavior. + ```python from hud.eval.sync import diff From b0c0ca6b111890811638e2159392e8f47a4889da Mon Sep 17 00:00:00 2001 From: Asadullo Ganiev <62354884+solvemproblr@users.noreply.github.com> Date: Sat, 5 Sep 2026 17:53:46 +0800 Subject: [PATCH 3/9] fix(projects): allow read-only sync previews --- hud/cli/sync.py | 10 +++- hud/cli/tests/test_sync_projects.py | 81 +++++++++++++++++++++++++++++ hud/cli/utils/project.py | 26 +++++++-- 3 files changed, 112 insertions(+), 5 deletions(-) create mode 100644 hud/cli/tests/test_sync_projects.py diff --git a/hud/cli/sync.py b/hud/cli/sync.py index 26f3e9d24..b02d42837 100644 --- a/hud/cli/sync.py +++ b/hud/cli/sync.py @@ -11,7 +11,11 @@ import typer from hud.cli.utils.api import require_api_key -from hud.cli.utils.project import Placement, resolve_writable_placement +from hud.cli.utils.project import ( + Placement, + require_writable_placement, + resolve_placement_or_exit, +) from hud.cli.utils.registry import ( RegistryEnvironment, get_registry_environment, @@ -328,7 +332,7 @@ def sync_tasks_command( # Creating a new taskset is only allowed when targeting an explicit name # (not an --id or a stored id, which must already exist). allow_create = taskset is not None and taskset_id is None - placement = resolve_writable_placement( + placement = resolve_placement_or_exit( platform, EnvironmentSource.open(), flag=project, @@ -364,6 +368,8 @@ def sync_tasks_command( hud_console.info("\n --dry-run: no changes made") return + require_writable_placement(placement, hud_console) + if not yes and not hud_console.confirm("Proceed?", default=False): hud_console.info("Aborted.") return diff --git a/hud/cli/tests/test_sync_projects.py b/hud/cli/tests/test_sync_projects.py new file mode 100644 index 000000000..992175da1 --- /dev/null +++ b/hud/cli/tests/test_sync_projects.py @@ -0,0 +1,81 @@ +"""Project placement behavior for ``hud sync tasks``.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +import pytest + +import hud.cli.sync as sync_module +from hud.eval import Task, Taskset + +if TYPE_CHECKING: + from pathlib import Path + + +class _ReadOnlyPlatform: + def get(self, url: str) -> dict[str, Any]: + assert url == "/projects" + return { + "projects": [ + { + "id": "33333333-3333-4333-8333-333333333333", + "name": "locked-down", + "capabilities": {"create": False}, + } + ] + } + + +def _run_sync( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + remote: Taskset, + *, + dry_run: bool, +) -> None: + task = Task(env="example", id="solve", slug="one") + local = Taskset("demo", [task]) + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(sync_module, "require_api_key", lambda _: None) + monkeypatch.setattr( + sync_module.PlatformClient, + "from_settings", + lambda: _ReadOnlyPlatform(), + ) + monkeypatch.setattr(sync_module, "_load_local_taskset", lambda *args, **kwargs: local) + monkeypatch.setattr(sync_module, "_fetch_remote_taskset", lambda *args, **kwargs: remote) + monkeypatch.setattr( + sync_module, + "upload_taskset", + lambda *args, **kwargs: pytest.fail("read-only no-op must not upload"), + ) + + sync_module.sync_tasks_command( + taskset="demo", + source=".", + taskset_id=None, + project="locked-down", + task_filter=None, + exclude=None, + yes=True, + dry_run=dry_run, + force=False, + export=None, + ) + + +def test_read_only_project_allows_up_to_date_sync( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + task = Task(env="example", id="solve", slug="one") + _run_sync(monkeypatch, tmp_path, Taskset("demo", [task]), dry_run=False) + + +def test_read_only_project_allows_dry_run( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _run_sync(monkeypatch, tmp_path, Taskset("demo", []), dry_run=True) diff --git a/hud/cli/utils/project.py b/hud/cli/utils/project.py index b2484b63a..888da79fb 100644 --- a/hud/cli/utils/project.py +++ b/hud/cli/utils/project.py @@ -167,19 +167,37 @@ def resolve_writable_placement( console: HUDConsole, ) -> Placement: """Resolve and announce a Project that accepts new resources.""" + placement = resolve_placement_or_exit(platform, env_source, flag=flag, console=console) + require_writable_placement(placement, console) + return placement + + +def resolve_placement_or_exit( + platform: PlatformClient, + env_source: EnvironmentSource, + *, + flag: str | None, + console: HUDConsole, +) -> Placement: + """Resolve and announce a Project without requiring create access.""" from hud.utils.exceptions import HudRequestError try: placement = resolve_placement(platform, env_source, flag=flag) - if placement.project is not None and not placement.project.can_create: - raise ProjectNotWritable(placement.project) - except (ProjectNotFound, ProjectNotWritable, HudRequestError) as e: + except (ProjectNotFound, HudRequestError) as e: raise report_project_error(console, e) from e console.info(f"Project: {placement.label}") return placement +def require_writable_placement(placement: Placement, console: HUDConsole) -> None: + """Exit when an operation would write to a read-only Project.""" + if placement.project is not None and not placement.project.can_create: + error = ProjectNotWritable(placement.project) + raise report_project_error(console, error) from error + + __all__ = [ "Placement", "Project", @@ -188,7 +206,9 @@ def resolve_writable_placement( "ProjectSource", "list_projects", "report_project_error", + "require_writable_placement", "resolve_placement", + "resolve_placement_or_exit", "resolve_project", "resolve_writable_placement", ] From 0a156df25c03db227d851e93fecd90878a01cdd6 Mon Sep 17 00:00:00 2001 From: Asadullo Ganiev <62354884+solvemproblr@users.noreply.github.com> Date: Wed, 9 Sep 2026 17:54:17 +0500 Subject: [PATCH 4/9] refactor(projects): simplify project selection --- README.md | 4 +- docs/v6/guides/creating-an-environment.mdx | 2 +- docs/v6/reference/cli.mdx | 13 ++--- docs/v6/reference/projects.mdx | 5 +- hud/cli/deploy.py | 10 ++-- hud/cli/project.py | 6 +-- hud/cli/sync.py | 14 ++--- hud/cli/tests/test_deploy.py | 1 + hud/cli/tests/test_sync_projects.py | 59 ++++++++++++++++++++++ hud/cli/utils/project.py | 12 ++++- hud/cli/utils/tests/test_project.py | 22 ++++---- hud/settings.py | 7 ++- hud/tests/test_settings.py | 6 +++ 13 files changed, 113 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 660da3ac4..d302802ee 100644 --- a/README.md +++ b/README.md @@ -151,12 +151,12 @@ From the [platform UI](https://hud.ai) you can run batches, compare models on th A **project** holds the environments and tasksets a team creates and decides who can see them. Deploying without one uses your team's default project, so nothing here is required to get started. To put an environment somewhere else, pin the directory once and every later `hud deploy` and `hud sync tasks` follows it: ```bash -hud project list # projects you can use +hud project list # projects you can see hud project use browser-evals # writes projectId to .hud/config.json hud project # where does a deploy here land? ``` -Both commands also take `--project ` to override the resolved Project. A successful deploy or sync pins that Project to the directory; `hud set HUD_PROJECT=` instead sets a machine-wide default for directories you have not pinned. Precedence is the flag, then the directory's `.hud/config.json`, then `HUD_PROJECT`, then your team default. An environment or taskset that already exists stays where it is; naming a different project fails rather than moving it. +`hud deploy` and `hud sync tasks` also take `--project ` as a one-command override; it does not change the directory configuration. Use `hud set HUD_DEFAULT_PROJECT=` for a machine-wide fallback. Precedence is the flag, then the directory's `.hud/config.json`, then `HUD_DEFAULT_PROJECT`, then your team default. An environment or taskset that already exists stays where it is; naming a different project fails rather than moving it. → [Projects](https://docs.hud.ai/v6/reference/projects) · [Run & deploy](https://docs.hud.ai/v6/reference/runtime) diff --git a/docs/v6/guides/creating-an-environment.mdx b/docs/v6/guides/creating-an-environment.mdx index 9d6ddb4d0..18a073f3a 100644 --- a/docs/v6/guides/creating-an-environment.mdx +++ b/docs/v6/guides/creating-an-environment.mdx @@ -297,7 +297,7 @@ deploy: hud project use browser-evals hud deploy -# Or select it directly; a successful deploy saves the resolved Project +# Or override this deploy without changing the directory Project hud deploy --project browser-evals ``` diff --git a/docs/v6/reference/cli.mdx b/docs/v6/reference/cli.mdx index bc1b2aab0..510a5b9b6 100644 --- a/docs/v6/reference/cli.mdx +++ b/docs/v6/reference/cli.mdx @@ -72,7 +72,7 @@ hud deploy --project browser-evals | `--all`, `-a` | Deploy all environments in the directory. | | `--env`, `-e` | Env var `KEY=VALUE` (repeatable). | | `--env-file` | Path to a `.env` file. | -| `--project` | Project name or ID for placement. Falls back to the directory Project, `HUD_PROJECT`, then the team default. | +| `--project` | One-command Project override by name or ID. Falls back to the directory Project, `HUD_DEFAULT_PROJECT`, then the team default. | ## Evaluate @@ -171,11 +171,12 @@ hud project use browser-evals -C ./my-env |----------|--------|-------| | 1 | `--project ` | Current command | | 2 | `.hud/config.json` → `projectId` | Current directory | -| 3 | `HUD_PROJECT` | Machine or process | +| 3 | `HUD_DEFAULT_PROJECT` | Machine-wide fallback | | 4 | Team default Project | Zero-configuration fallback | -Set the machine-wide fallback with `hud set HUD_PROJECT=`. When a named Project is -resolved, a successful deploy or task sync saves its ID to the directory's `.hud/config.json`. +`--project` does not change `.hud/config.json`. Use `hud project use ` when later deploys +and task syncs from the directory should use that Project. Set a fallback for unconfigured +directories with `hud set HUD_DEFAULT_PROJECT=`. `hud project list` marks the team default and read-only Projects. Pinning a directory, deploying, or syncing to a Project requires create access. Only team admins can create Projects. @@ -194,8 +195,8 @@ hud sync tasks my-taskset --project browser-evals # override by Project name or hud sync env # link environment metadata ``` -`hud sync tasks --project` follows the same resolution and persistence rules as -`hud deploy --project`. Existing resources remain in their original Project. See +`hud sync tasks --project` is a one-command override, like `hud deploy --project`. Existing resources +remain in their original Project. See [Projects](/v6/reference/projects) for the conceptual overview. External benchmark formats can be adapted into runnable `Taskset`s through the diff --git a/docs/v6/reference/projects.mdx b/docs/v6/reference/projects.mdx index 1f8ddf98e..dee1afdf0 100644 --- a/docs/v6/reference/projects.mdx +++ b/docs/v6/reference/projects.mdx @@ -17,8 +17,9 @@ another Project, HUD places it in the team default. This keeps the normal workfl A local environment directory can be associated with a Project. Deploys and task syncs from that directory then use the same destination, keeping an environment and its tasksets together. -For automation or exceptional cases, a command can select a different destination explicitly. HUD -also supports a machine-wide fallback for directories without an association. +For automation or exceptional cases, a command can select a different destination explicitly without +changing the directory association. A machine-wide preference can provide a fallback for directories +without an association. Project selection only controls where a **new** resource is created. It does not move resources between Projects. diff --git a/hud/cli/deploy.py b/hud/cli/deploy.py index 2649a9118..8981812a1 100644 --- a/hud/cli/deploy.py +++ b/hud/cli/deploy.py @@ -19,7 +19,7 @@ from hud.cli.utils.build_logs import poll_build_status, stream_build_logs from hud.cli.utils.config import parse_env_file, parse_key_value from hud.cli.utils.context import create_build_context_tarball, format_size -from hud.cli.utils.project import Placement, resolve_writable_placement +from hud.cli.utils.project import PROJECT_OPTION_HELP, Placement, resolve_writable_placement from hud.cli.utils.registry import get_registry_environment from hud.cli.utils.source import EnvironmentSource from hud.eval.runtime import ComposeProject, RuntimeConfig @@ -642,7 +642,7 @@ async def _deploy_async( # Save immediately after trigger so rebuilds work even if streaming crashes. if env_dir and registry_id: - _save_deploy_link(env_dir, registry_id, console, env_name=plan.name, plan=plan) + _save_deploy_link(env_dir, registry_id, console, env_name=plan.name) console.success(f"Build triggered [{time.time() - step_start:.1f}s]") console.info(f"Build ID: {build_id}") @@ -690,15 +690,12 @@ def _save_deploy_link( registry_id: str, console: HUDConsole, env_name: str | None = None, - plan: _DeployPlan | None = None, ) -> None: """Save deploy linking info to .hud/config.json.""" try: config_data: dict[str, Any] = {"registryId": registry_id} if env_name: config_data["registryName"] = env_name - if plan is not None and plan.placement.project is not None: - config_data["projectId"] = plan.placement.project.id changed = EnvironmentSource.open(env_dir).save_config(config_data) console.success(f"Linked to environment: {registry_id[:8]}...") if changed: @@ -849,8 +846,7 @@ def deploy_command( project: str | None = typer.Option( None, "--project", - help="Project to create this environment in (name or ID). Defaults to the " - "directory's saved project, then HUD_PROJECT, then your team default.", + help=PROJECT_OPTION_HELP, ), runtime: str | None = typer.Option( None, diff --git a/hud/cli/project.py b/hud/cli/project.py index 3189d9ffb..63e9c60ac 100644 --- a/hud/cli/project.py +++ b/hud/cli/project.py @@ -106,9 +106,9 @@ def use_command( ) -> None: """Pin a directory to a Project. - [not dim]Writes projectId to .hud/config.json, so teammates deploying this - environment place it in the same project. Set a machine-wide fallback for - unpinned directories with: hud set HUD_PROJECT= + [not dim]Writes projectId to .hud/config.json, so later deploys and task + syncs from this directory use the same project. Set a machine-wide fallback + with: hud set HUD_DEFAULT_PROJECT= Examples: hud project use browser-evals diff --git a/hud/cli/sync.py b/hud/cli/sync.py index b02d42837..b8097a6ae 100644 --- a/hud/cli/sync.py +++ b/hud/cli/sync.py @@ -12,7 +12,7 @@ from hud.cli.utils.api import require_api_key from hud.cli.utils.project import ( - Placement, + PROJECT_OPTION_HELP, require_writable_placement, resolve_placement_or_exit, ) @@ -225,14 +225,11 @@ def _show_upload_error(error: HudRequestError, console: HUDConsole) -> None: console.error(f"Upload failed ({error.status_code}): {detail or error}") -def _save_taskset_id(result: dict[str, object], placement: Placement, console: HUDConsole) -> None: +def _save_taskset_id(result: dict[str, object], console: HUDConsole) -> None: returned_id = result.get("taskset_id") if not isinstance(returned_id, str) or not returned_id: return - config: dict[str, object] = {"tasksetId": returned_id} - if placement.project is not None: - config["projectId"] = placement.project.id - changed = EnvironmentSource.open().save_config(config) + changed = EnvironmentSource.open().save_config({"tasksetId": returned_id}) if changed: console.dim_info("Taskset ID saved to:", ".hud/config.json") from hud.settings import settings @@ -258,8 +255,7 @@ def sync_tasks_command( project: str | None = typer.Option( None, "--project", - help="Project to create this taskset in (name or ID). Defaults to the " - "directory's saved project, then HUD_PROJECT, then your team default.", + help=PROJECT_OPTION_HELP, ), task_filter: str | None = typer.Option( None, @@ -392,7 +388,7 @@ def sync_tasks_command( hud_console.success("Sync complete") hud_console.info(f" + {created} created, ~ {updated} updated") - _save_taskset_id(result, placement, hud_console) + _save_taskset_id(result, hud_console) @sync_app.command("env") diff --git a/hud/cli/tests/test_deploy.py b/hud/cli/tests/test_deploy.py index 61fe384fd..e65d6d0a0 100644 --- a/hud/cli/tests/test_deploy.py +++ b/hud/cli/tests/test_deploy.py @@ -605,6 +605,7 @@ def test_saves_deploy_link(self, tmp_path: Path) -> None: saved = json.load(f) assert saved["registryId"] == "test-registry-id-12345" + assert "projectId" not in saved def test_creates_hud_directory(self, tmp_path: Path) -> None: """Test that .hud directory is created if missing.""" diff --git a/hud/cli/tests/test_sync_projects.py b/hud/cli/tests/test_sync_projects.py index 992175da1..83e45dc51 100644 --- a/hud/cli/tests/test_sync_projects.py +++ b/hud/cli/tests/test_sync_projects.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json from typing import TYPE_CHECKING, Any import pytest @@ -27,6 +28,20 @@ def get(self, url: str) -> dict[str, Any]: } +class _WritablePlatform: + def get(self, url: str) -> dict[str, Any]: + assert url == "/projects" + return { + "projects": [ + { + "id": "22222222-2222-4222-8222-222222222222", + "name": "browser-evals", + "capabilities": {"create": True}, + } + ] + } + + def _run_sync( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -79,3 +94,47 @@ def test_read_only_project_allows_dry_run( tmp_path: Path, ) -> None: _run_sync(monkeypatch, tmp_path, Taskset("demo", []), dry_run=True) + + +def test_project_override_does_not_pin_directory( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + task = Task(env="example", id="solve", slug="one") + local = Taskset("demo", [task]) + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(sync_module, "require_api_key", lambda _: None) + monkeypatch.setattr( + sync_module.PlatformClient, + "from_settings", + lambda: _WritablePlatform(), + ) + monkeypatch.setattr(sync_module, "_load_local_taskset", lambda *args, **kwargs: local) + monkeypatch.setattr( + sync_module, + "_fetch_remote_taskset", + lambda *args, **kwargs: Taskset("demo", []), + ) + + def upload(*args: Any, **kwargs: Any) -> dict[str, Any]: + assert kwargs["project_id"] == "22222222-2222-4222-8222-222222222222" + return {"taskset_id": "taskset-1", "tasks_created": 1} + + monkeypatch.setattr(sync_module, "upload_taskset", upload) + + sync_module.sync_tasks_command( + taskset="demo", + source=".", + taskset_id=None, + project="browser-evals", + task_filter=None, + exclude=None, + yes=True, + dry_run=False, + force=False, + export=None, + ) + + config = json.loads((tmp_path / ".hud" / "config.json").read_text()) + assert config == {"tasksetId": "taskset-1"} diff --git a/hud/cli/utils/project.py b/hud/cli/utils/project.py index 888da79fb..9dea22c15 100644 --- a/hud/cli/utils/project.py +++ b/hud/cli/utils/project.py @@ -22,10 +22,17 @@ class ProjectSource(Enum): FLAG = "--project" CONFIG = ".hud/config.json" - SETTINGS = "HUD_PROJECT" + GLOBAL_DEFAULT = "HUD_DEFAULT_PROJECT" TEAM_DEFAULT = "team default" +PROJECT_OPTION_HELP = ( + "Project for this command (name or ID). Defaults to the directory's saved " + "project, HUD_DEFAULT_PROJECT, then your team default. Does not change " + "directory configuration." +) + + @dataclass(frozen=True) class Project: id: str @@ -132,7 +139,7 @@ def resolve_placement( for ref, source in ( (flag, ProjectSource.FLAG), (env_source.project_id, ProjectSource.CONFIG), - (settings.project, ProjectSource.SETTINGS), + (settings.default_project, ProjectSource.GLOBAL_DEFAULT), ): if ref: project = resolve_project(platform, ref) @@ -199,6 +206,7 @@ def require_writable_placement(placement: Placement, console: HUDConsole) -> Non __all__ = [ + "PROJECT_OPTION_HELP", "Placement", "Project", "ProjectNotFound", diff --git a/hud/cli/utils/tests/test_project.py b/hud/cli/utils/tests/test_project.py index 2eb6cc1bd..5a8c60eaf 100644 --- a/hud/cli/utils/tests/test_project.py +++ b/hud/cli/utils/tests/test_project.py @@ -62,8 +62,8 @@ def fake_request(method: str, url: str, **kwargs: Any) -> dict[str, Any]: return PlatformClient("https://api.example", "key") -def _no_settings_project(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr("hud.settings.settings.project", None) +def _no_global_default(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("hud.settings.settings.default_project", None) def test_resolve_matches_a_normalized_name(platform: PlatformClient) -> None: @@ -93,7 +93,7 @@ def test_flag_outranks_directory_config( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - _no_settings_project(monkeypatch) + monkeypatch.setattr("hud.settings.settings.default_project", "locked-down") source = EnvironmentSource.open(tmp_path) source.save_config({"projectId": _DEFAULT_ID}) @@ -104,13 +104,13 @@ def test_flag_outranks_directory_config( assert placement.source is ProjectSource.FLAG -def test_directory_config_outranks_the_machine_default( +def test_directory_config_applies_without_a_flag( platform: PlatformClient, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: """Placement is a property of the environment, not of who deploys it.""" - monkeypatch.setattr("hud.settings.settings.project", "default") + monkeypatch.setattr("hud.settings.settings.default_project", "default") source = EnvironmentSource.open(tmp_path) source.save_config({"projectId": _BROWSER_ID}) @@ -121,18 +121,18 @@ def test_directory_config_outranks_the_machine_default( assert placement.source is ProjectSource.CONFIG -def test_machine_default_applies_to_an_unpinned_directory( +def test_global_default_applies_to_an_unpinned_directory( platform: PlatformClient, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr("hud.settings.settings.project", "browser-evals") + monkeypatch.setattr("hud.settings.settings.default_project", "browser-evals") placement = resolve_placement(platform, EnvironmentSource.open(tmp_path), flag=None) assert placement.project is not None assert placement.project.id == _BROWSER_ID - assert placement.source is ProjectSource.SETTINGS + assert placement.source is ProjectSource.GLOBAL_DEFAULT def test_unconfigured_placement_sends_no_project_and_makes_no_call( @@ -142,8 +142,7 @@ def test_unconfigured_placement_sends_no_project_and_makes_no_call( monkeypatch: pytest.MonkeyPatch, ) -> None: """The zero-config path stays free: no project on the wire, no lookup.""" - _no_settings_project(monkeypatch) - + _no_global_default(monkeypatch) placement = resolve_placement(platform, EnvironmentSource.open(tmp_path), flag=None) assert placement.project_id is None @@ -157,8 +156,7 @@ def test_placement_resolves_a_project_the_caller_cannot_create_in( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - _no_settings_project(monkeypatch) - + _no_global_default(monkeypatch) source = EnvironmentSource.open(tmp_path) placement = resolve_placement(platform, source, flag="locked-down") assert placement.project is not None diff --git a/hud/settings.py b/hud/settings.py index f2ce7068a..1b7426fc2 100644 --- a/hud/settings.py +++ b/hud/settings.py @@ -92,11 +92,10 @@ def settings_customise_sources( validation_alias="HUD_API_KEY", ) - project: str | None = Field( + default_project: str | None = Field( default=None, - description="Default HUD Project (name or id) for environments and tasksets this " - "machine creates. A directory's .hud/config.json takes precedence.", - validation_alias="HUD_PROJECT", + description="Default HUD Project name or ID for directories without a saved Project", + validation_alias="HUD_DEFAULT_PROJECT", ) anthropic_api_key: str | None = Field( diff --git a/hud/tests/test_settings.py b/hud/tests/test_settings.py index 25db843d0..08828791e 100644 --- a/hud/tests/test_settings.py +++ b/hud/tests/test_settings.py @@ -36,6 +36,12 @@ def test_file_tracking_can_be_disabled_by_env(monkeypatch): assert Settings().file_tracking_enabled is False +def test_default_project_accepts_a_name_or_id(monkeypatch): + monkeypatch.setenv("HUD_DEFAULT_PROJECT", "browser-evals") + + assert Settings().default_project == "browser-evals" + + def test_cli_analytics_is_independent_of_trace_telemetry(monkeypatch): monkeypatch.setenv("HUD_TELEMETRY_ENABLED", "true") monkeypatch.setenv("HUD_CLI_ANALYTICS_ENABLED", "false") From 290fc45ccde947b85bd2947ef5f2eb8fab720f8e Mon Sep 17 00:00:00 2001 From: Asadullo Ganiev <62354884+solvemproblr@users.noreply.github.com> Date: Wed, 9 Sep 2026 18:34:40 +0500 Subject: [PATCH 5/9] fix(projects): match paginated project API --- hud/cli/tests/test_sync_projects.py | 8 ++++---- hud/cli/utils/project.py | 25 ++++++++++++++++--------- hud/cli/utils/tests/test_project.py | 28 +++++++++++++++++++++------- 3 files changed, 41 insertions(+), 20 deletions(-) diff --git a/hud/cli/tests/test_sync_projects.py b/hud/cli/tests/test_sync_projects.py index 83e45dc51..e85ed5cd2 100644 --- a/hud/cli/tests/test_sync_projects.py +++ b/hud/cli/tests/test_sync_projects.py @@ -15,10 +15,10 @@ class _ReadOnlyPlatform: - def get(self, url: str) -> dict[str, Any]: + def get(self, url: str, *, params: dict[str, Any] | None = None) -> dict[str, Any]: assert url == "/projects" return { - "projects": [ + "items": [ { "id": "33333333-3333-4333-8333-333333333333", "name": "locked-down", @@ -29,10 +29,10 @@ def get(self, url: str) -> dict[str, Any]: class _WritablePlatform: - def get(self, url: str) -> dict[str, Any]: + def get(self, url: str, *, params: dict[str, Any] | None = None) -> dict[str, Any]: assert url == "/projects" return { - "projects": [ + "items": [ { "id": "22222222-2222-4222-8222-222222222222", "name": "browser-evals", diff --git a/hud/cli/utils/project.py b/hud/cli/utils/project.py index 9dea22c15..d05a50584 100644 --- a/hud/cli/utils/project.py +++ b/hud/cli/utils/project.py @@ -9,6 +9,7 @@ import typer +from hud.utils.exceptions import HudRequestError from hud.utils.naming import normalize_environment_name if TYPE_CHECKING: @@ -99,7 +100,12 @@ def __init__(self, project: Project) -> None: def list_projects(platform: PlatformClient) -> list[Project]: """Every Project visible to the caller.""" data = platform.get("/projects") - records = data.get("projects") if isinstance(data, dict) else None + return _projects_from_page(data) + + +def _projects_from_page(data: Any) -> list[Project]: + """Parse the platform's paginated Project response.""" + records = data.get("items") if isinstance(data, dict) else None if not isinstance(records, list): return [] return [Project.from_record(item) for item in records if isinstance(item, dict)] @@ -111,19 +117,22 @@ def resolve_project(platform: PlatformClient, ref: str) -> Project: Names are normalized the same way the platform normalizes them on create, so `My Project` and `my-project` resolve to the same row. """ - projects = list_projects(platform) try: project_id = str(uuid.UUID(ref)) except ValueError: - project_id = None - - match = next((p for p in projects if p.id == project_id), None) - if match is None: name = normalize_environment_name(ref, default="") + projects = _projects_from_page(platform.get("/projects", params={"search": name})) match = next((p for p in projects if p.name == name), None) + else: + try: + return Project.from_record(platform.get(f"/projects/{project_id}")) + except HudRequestError as e: + if e.status_code != 404: + raise + match = None if match is None: - raise ProjectNotFound(ref, projects) + raise ProjectNotFound(ref, list_projects(platform)) return match @@ -187,8 +196,6 @@ def resolve_placement_or_exit( console: HUDConsole, ) -> Placement: """Resolve and announce a Project without requiring create access.""" - from hud.utils.exceptions import HudRequestError - try: placement = resolve_placement(platform, env_source, flag=flag) except (ProjectNotFound, HudRequestError) as e: diff --git a/hud/cli/utils/tests/test_project.py b/hud/cli/utils/tests/test_project.py index 5a8c60eaf..429760529 100644 --- a/hud/cli/utils/tests/test_project.py +++ b/hud/cli/utils/tests/test_project.py @@ -11,6 +11,7 @@ Project, ProjectNotFound, ProjectSource, + list_projects, resolve_placement, resolve_project, resolve_writable_placement, @@ -50,13 +51,18 @@ def platform(monkeypatch: pytest.MonkeyPatch, calls: list[str]) -> PlatformClien def fake_request(method: str, url: str, **kwargs: Any) -> dict[str, Any]: calls.append(url) - return { - "projects": [ - _record(_DEFAULT_ID, "default", is_default=True), - _record(_BROWSER_ID, "browser-evals"), - _record(_READONLY_ID, "locked-down", create=False), - ] - } + records = [ + _record(_DEFAULT_ID, "default", is_default=True), + _record(_BROWSER_ID, "browser-evals"), + _record(_READONLY_ID, "locked-down", create=False), + ] + project_id = url.rsplit("/", 1)[-1] + if project_id in {_DEFAULT_ID, _BROWSER_ID, _READONLY_ID}: + return next(record for record in records if record["id"] == project_id) + search = (kwargs.get("params") or {}).get("search") + if search: + records = [record for record in records if search in record["name"]] + return {"items": records, "total": len(records), "limit": 50, "offset": 0} monkeypatch.setattr("hud.utils.platform.make_request_sync", fake_request) return PlatformClient("https://api.example", "key") @@ -66,6 +72,14 @@ def _no_global_default(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr("hud.settings.settings.default_project", None) +def test_list_reads_paginated_items(platform: PlatformClient) -> None: + assert [project.name for project in list_projects(platform)] == [ + "default", + "browser-evals", + "locked-down", + ] + + def test_resolve_matches_a_normalized_name(platform: PlatformClient) -> None: """A human-typed name resolves through the same normalization the platform applies.""" assert resolve_project(platform, "Browser Evals").id == _BROWSER_ID From 182abe2ad28c292605c66c2de9d30f8577573e36 Mon Sep 17 00:00:00 2001 From: Asadullo Ganiev <62354884+solvemproblr@users.noreply.github.com> Date: Wed, 9 Sep 2026 18:50:04 +0500 Subject: [PATCH 6/9] fix(projects): inherit group directory option --- hud/cli/project.py | 31 +++++++-- hud/cli/tests/test_project_command.py | 95 +++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 4 deletions(-) create mode 100644 hud/cli/tests/test_project_command.py diff --git a/hud/cli/project.py b/hud/cli/project.py index 63e9c60ac..dbb9999d1 100644 --- a/hud/cli/project.py +++ b/hud/cli/project.py @@ -27,6 +27,8 @@ rich_markup_mode="rich", ) +_DIRECTORY_META_KEY = "hud_project_directory" + @project_app.command("list") def list_command() -> None: @@ -61,9 +63,15 @@ def list_command() -> None: @project_app.command("create") def create_command( + ctx: typer.Context, name: str = typer.Argument(..., help="Name for the new project"), description: str | None = typer.Option(None, "--description", help="What the project holds"), - directory: str = typer.Option(".", "--directory", "-C", help="Directory to pin it to"), + directory: str | None = typer.Option( + None, + "--directory", + "-C", + help="Directory to pin it to (defaults to the group -C or current directory)", + ), no_use: bool = typer.Option(False, "--no-use", help="Create without pinning this directory"), ) -> None: """Create a Project and pin this directory to it. @@ -96,13 +104,19 @@ def create_command( console.success(f"Created project: {created.name} ({created.short_id}...)") if not no_use: - _pin(created, directory, console) + _pin(created, _command_directory(ctx, directory), console) @project_app.command("use") def use_command( + ctx: typer.Context, ref: str = typer.Argument(..., help="Project name or ID"), - directory: str = typer.Option(".", "--directory", "-C", help="Directory to pin"), + directory: str | None = typer.Option( + None, + "--directory", + "-C", + help="Directory to pin (defaults to the group -C or current directory)", + ), ) -> None: """Pin a directory to a Project. @@ -123,7 +137,7 @@ def use_command( raise report_project_error(console, e) from e if not project.can_create: raise report_project_error(console, ProjectNotWritable(project)) - _pin(project, directory, console) + _pin(project, _command_directory(ctx, directory), console) @project_app.callback(invoke_without_command=True) @@ -138,6 +152,7 @@ def project_callback( hud project list # projects you can see hud project use browser-evals # pin this directory[/not dim] """ + ctx.meta[_DIRECTORY_META_KEY] = directory if ctx.invoked_subcommand is not None: return @@ -159,6 +174,14 @@ def project_callback( console.warning("You do not have create access to this Project") +def _command_directory(ctx: typer.Context, directory: str | None) -> str: + """Let subcommand -C override the group-level project -C option.""" + if directory is not None: + return directory + inherited = ctx.meta.get(_DIRECTORY_META_KEY) + return inherited if isinstance(inherited, str) else "." + + def _pin(project: Project, directory: str, console: HUDConsole) -> None: changed = EnvironmentSource.open(directory).save_config({"projectId": project.id}) console.success(f"Using project: {project.name} ({project.short_id}...)") diff --git a/hud/cli/tests/test_project_command.py b/hud/cli/tests/test_project_command.py new file mode 100644 index 000000000..205f00238 --- /dev/null +++ b/hud/cli/tests/test_project_command.py @@ -0,0 +1,95 @@ +"""CLI parsing for Project commands.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING +from unittest.mock import MagicMock + +import pytest +from typer.testing import CliRunner + +from hud.cli import project +from hud.cli.utils.project import Project + +if TYPE_CHECKING: + from pathlib import Path + + +@pytest.fixture +def project_record() -> Project: + return Project( + id="22222222-2222-4222-8222-222222222222", + name="browser-evals", + is_default=False, + can_create=True, + ) + + +def test_group_directory_is_inherited_by_use( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + project_record: Project, +) -> None: + pinned: list[str] = [] + monkeypatch.setattr(project, "require_api_key", lambda _: None) + monkeypatch.setattr(project, "resolve_project", lambda _platform, _ref: project_record) + monkeypatch.setattr( + project, "_pin", lambda _project, directory, _console: pinned.append(directory) + ) + + result = CliRunner().invoke( + project.project_app, + ["-C", str(tmp_path), "use", "browser-evals"], + ) + + assert result.exit_code == 0 + assert pinned == [str(tmp_path)] + + +def test_subcommand_directory_overrides_group_directory( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + project_record: Project, +) -> None: + pinned: list[str] = [] + override = tmp_path / "override" + monkeypatch.setattr(project, "require_api_key", lambda _: None) + monkeypatch.setattr(project, "resolve_project", lambda _platform, _ref: project_record) + monkeypatch.setattr( + project, "_pin", lambda _project, directory, _console: pinned.append(directory) + ) + + result = CliRunner().invoke( + project.project_app, + ["-C", str(tmp_path), "use", "browser-evals", "-C", str(override)], + ) + + assert result.exit_code == 0 + assert pinned == [str(override)] + + +def test_group_directory_is_inherited_by_create( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + project_record: Project, +) -> None: + pinned: list[str] = [] + platform = MagicMock() + platform.post.return_value = { + "id": project_record.id, + "name": project_record.name, + "capabilities": {"create": True}, + } + monkeypatch.setattr(project, "require_api_key", lambda _: None) + monkeypatch.setattr(project.PlatformClient, "from_settings", lambda: platform) + monkeypatch.setattr( + project, "_pin", lambda _project, directory, _console: pinned.append(directory) + ) + + result = CliRunner().invoke( + project.project_app, + ["-C", str(tmp_path), "create", "browser-evals"], + ) + + assert result.exit_code == 0 + assert pinned == [str(tmp_path)] From 55103667413b72194a517161e6e51966ea179d02 Mon Sep 17 00:00:00 2001 From: Asadullo Ganiev <62354884+solvemproblr@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:21:25 +0500 Subject: [PATCH 7/9] fix(projects): explain feature-gated access --- docs/v6/reference/projects.mdx | 4 +++ hud/cli/project.py | 8 +++++- hud/cli/tests/test_project_command.py | 38 +++++++++++++++++++++++++++ hud/cli/utils/project.py | 25 +++++++++++++++++- hud/cli/utils/tests/test_project.py | 26 ++++++++++++++++++ 5 files changed, 99 insertions(+), 2 deletions(-) diff --git a/docs/v6/reference/projects.mdx b/docs/v6/reference/projects.mdx index dee1afdf0..7b87d1fae 100644 --- a/docs/v6/reference/projects.mdx +++ b/docs/v6/reference/projects.mdx @@ -7,6 +7,10 @@ icon: "folder-tree" A HUD **Project** is the workspace boundary for a related set of environments and tasksets. Projects also control who can see those resources and who can create new ones. +Projects is currently a limited beta. If it is not enabled for your team, Project-specific CLI +commands stop with a message explaining how to request access; normal deploy and task sync workflows +continue to use the team default Project. + ## Default Project Every team has a default Project. If you deploy an environment or publish a taskset without choosing diff --git a/hud/cli/project.py b/hud/cli/project.py index dbb9999d1..f4010cc33 100644 --- a/hud/cli/project.py +++ b/hud/cli/project.py @@ -11,7 +11,9 @@ ProjectNotFound, ProjectNotWritable, list_projects, + projects_not_enabled, report_project_error, + require_projects_enabled, resolve_placement, resolve_project, ) @@ -93,6 +95,8 @@ def create_command( try: created = Project.from_record(platform.post("/projects", json=payload)) except HudRequestError as e: + if projects_not_enabled(e): + raise report_project_error(console, e) from e if e.status_code == httpx.codes.CONFLICT: console.error(f"A project named '{name}' already exists") console.hint(f"Pin this directory to it with: hud project use {name}") @@ -158,9 +162,11 @@ def project_callback( console = HUDConsole() require_api_key("resolve the current project") + platform = PlatformClient.from_settings() try: + require_projects_enabled(platform) placement = resolve_placement( - PlatformClient.from_settings(), + platform, EnvironmentSource.open(directory), flag=None, ) diff --git a/hud/cli/tests/test_project_command.py b/hud/cli/tests/test_project_command.py index 205f00238..fa9bb624f 100644 --- a/hud/cli/tests/test_project_command.py +++ b/hud/cli/tests/test_project_command.py @@ -10,6 +10,7 @@ from hud.cli import project from hud.cli.utils.project import Project +from hud.utils.exceptions import HudRequestError if TYPE_CHECKING: from pathlib import Path @@ -25,6 +26,43 @@ def project_record() -> Project: ) +def _projects_disabled() -> HudRequestError: + return HudRequestError( + "Request failed: Projects are not enabled", + status_code=403, + response_json={"error": "forbidden", "detail": "Projects are not enabled"}, + ) + + +def test_bare_project_reports_disabled_feature(monkeypatch: pytest.MonkeyPatch) -> None: + platform = MagicMock() + platform.get.side_effect = _projects_disabled() + monkeypatch.setattr(project, "require_api_key", lambda _: None) + monkeypatch.setattr(project.PlatformClient, "from_settings", lambda: platform) + + result = CliRunner().invoke(project.project_app) + + assert result.exit_code == 1 + assert "Projects are not enabled for your team" in result.output + assert "Failed to reach" not in result.output + platform.get.assert_called_once_with("/projects", params={"limit": 1}) + + +def test_create_distinguishes_disabled_feature_from_admin_access( + monkeypatch: pytest.MonkeyPatch, +) -> None: + platform = MagicMock() + platform.post.side_effect = _projects_disabled() + monkeypatch.setattr(project, "require_api_key", lambda _: None) + monkeypatch.setattr(project.PlatformClient, "from_settings", lambda: platform) + + result = CliRunner().invoke(project.project_app, ["create", "browser-evals"]) + + assert result.exit_code == 1 + assert "Projects are not enabled for your team" in result.output + assert "Only team admins" not in result.output + + def test_group_directory_is_inherited_by_use( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/hud/cli/utils/project.py b/hud/cli/utils/project.py index d05a50584..45c120597 100644 --- a/hud/cli/utils/project.py +++ b/hud/cli/utils/project.py @@ -33,6 +33,9 @@ class ProjectSource(Enum): "directory configuration." ) +_PROJECTS_DISABLED_DETAIL = "projects are not enabled" +_PROJECTS_DISABLED_ERROR = "projects_not_enabled" + @dataclass(frozen=True) class Project: @@ -103,6 +106,11 @@ def list_projects(platform: PlatformClient) -> list[Project]: return _projects_from_page(data) +def require_projects_enabled(platform: PlatformClient) -> None: + """Check access to the feature-gated Projects API.""" + platform.get("/projects", params={"limit": 1}) + + def _projects_from_page(data: Any) -> list[Project]: """Parse the platform's paginated Project response.""" records = data.get("items") if isinstance(data, dict) else None @@ -159,7 +167,10 @@ def resolve_placement( def report_project_error(console: HUDConsole, error: Exception) -> typer.Exit: """Explain why a Project could not be used, and return the exit to raise.""" - if isinstance(error, ProjectNotFound): + if isinstance(error, HudRequestError) and projects_not_enabled(error): + console.error("Projects are not enabled for your team") + console.hint("Contact HUD to enable the Projects beta for your team") + elif isinstance(error, ProjectNotFound): console.error(str(error)) if error.available: console.info("Projects you can see:") @@ -175,6 +186,16 @@ def report_project_error(console: HUDConsole, error: Exception) -> typer.Exit: return typer.Exit(1) +def projects_not_enabled(error: HudRequestError) -> bool: + """Whether the Projects API rejected a caller at its feature gate.""" + if error.status_code != 403 or not isinstance(error.response_json, dict): + return False + if error.response_json.get("error") == _PROJECTS_DISABLED_ERROR: + return True + detail = error.response_json.get("detail") + return isinstance(detail, str) and detail.casefold() == _PROJECTS_DISABLED_DETAIL + + def resolve_writable_placement( platform: PlatformClient, env_source: EnvironmentSource, @@ -220,7 +241,9 @@ def require_writable_placement(placement: Placement, console: HUDConsole) -> Non "ProjectNotWritable", "ProjectSource", "list_projects", + "projects_not_enabled", "report_project_error", + "require_projects_enabled", "require_writable_placement", "resolve_placement", "resolve_placement_or_exit", diff --git a/hud/cli/utils/tests/test_project.py b/hud/cli/utils/tests/test_project.py index 429760529..8834adbce 100644 --- a/hud/cli/utils/tests/test_project.py +++ b/hud/cli/utils/tests/test_project.py @@ -12,11 +12,13 @@ ProjectNotFound, ProjectSource, list_projects, + projects_not_enabled, resolve_placement, resolve_project, resolve_writable_placement, ) from hud.cli.utils.source import EnvironmentSource +from hud.utils.exceptions import HudRequestError from hud.utils.hud_console import HUDConsole from hud.utils.platform import PlatformClient @@ -80,6 +82,30 @@ def test_list_reads_paginated_items(platform: PlatformClient) -> None: ] +def test_projects_not_enabled_matches_only_the_feature_gate() -> None: + assert projects_not_enabled( + HudRequestError( + "forbidden", + status_code=403, + response_json={"error": "projects_not_enabled", "detail": "Projects disabled"}, + ) + ) + assert projects_not_enabled( + HudRequestError( + "forbidden", + status_code=403, + response_json={"error": "forbidden", "detail": "Projects are not enabled"}, + ) + ) + assert not projects_not_enabled( + HudRequestError( + "forbidden", + status_code=403, + response_json={"error": "forbidden", "detail": "Missing create scope"}, + ) + ) + + def test_resolve_matches_a_normalized_name(platform: PlatformClient) -> None: """A human-typed name resolves through the same normalization the platform applies.""" assert resolve_project(platform, "Browser Evals").id == _BROWSER_ID From ab6dcc8a6da17886bbeb7bf2b8aac98d056acf56 Mon Sep 17 00:00:00 2001 From: Asadullo Ganiev <62354884+solvemproblr@users.noreply.github.com> Date: Sun, 13 Sep 2026 18:29:37 +0500 Subject: [PATCH 8/9] fix(cli): paginate project listing and name lookup --- hud/cli/utils/project.py | 24 +++++++-- hud/cli/utils/tests/test_project.py | 84 ++++++++++++++++++++++++----- 2 files changed, 93 insertions(+), 15 deletions(-) diff --git a/hud/cli/utils/project.py b/hud/cli/utils/project.py index 45c120597..c12a71e9c 100644 --- a/hud/cli/utils/project.py +++ b/hud/cli/utils/project.py @@ -13,6 +13,8 @@ from hud.utils.naming import normalize_environment_name if TYPE_CHECKING: + from collections.abc import Iterator + from hud.cli.utils.source import EnvironmentSource from hud.utils.hud_console import HUDConsole from hud.utils.platform import PlatformClient @@ -102,8 +104,24 @@ def __init__(self, project: Project) -> None: def list_projects(platform: PlatformClient) -> list[Project]: """Every Project visible to the caller.""" - data = platform.get("/projects") - return _projects_from_page(data) + return list(_iter_projects(platform)) + + +def _iter_projects(platform: PlatformClient, *, search: str | None = None) -> Iterator[Project]: + params: dict[str, str | int] = {"limit": 50, "offset": 0} + if search is not None: + params["search"] = search + offset = 0 + while True: + params["offset"] = offset + data = platform.get("/projects", params=params) + projects = _projects_from_page(data) + yield from projects + if not projects: + return + offset += len(data["items"]) + if offset >= data["total"]: + return def require_projects_enabled(platform: PlatformClient) -> None: @@ -129,7 +147,7 @@ def resolve_project(platform: PlatformClient, ref: str) -> Project: project_id = str(uuid.UUID(ref)) except ValueError: name = normalize_environment_name(ref, default="") - projects = _projects_from_page(platform.get("/projects", params={"search": name})) + projects = _iter_projects(platform, search=name) match = next((p for p in projects if p.name == name), None) else: try: diff --git a/hud/cli/utils/tests/test_project.py b/hud/cli/utils/tests/test_project.py index 8834adbce..54f16cf23 100644 --- a/hud/cli/utils/tests/test_project.py +++ b/hud/cli/utils/tests/test_project.py @@ -3,6 +3,7 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any +from urllib.parse import parse_qs, urlsplit import pytest import typer @@ -48,23 +49,41 @@ def calls() -> list[str]: @pytest.fixture -def platform(monkeypatch: pytest.MonkeyPatch, calls: list[str]) -> PlatformClient: - """A client whose ``GET /projects`` returns a fixed three-project team.""" +def records() -> list[dict[str, Any]]: + return [ + _record(_DEFAULT_ID, "default", is_default=True), + _record(_BROWSER_ID, "browser-evals"), + _record(_READONLY_ID, "locked-down", create=False), + ] + + +@pytest.fixture +def platform( + monkeypatch: pytest.MonkeyPatch, calls: list[str], records: list[dict[str, Any]] +) -> PlatformClient: + """A client backed by a paginated, searchable Projects API.""" def fake_request(method: str, url: str, **kwargs: Any) -> dict[str, Any]: calls.append(url) - records = [ - _record(_DEFAULT_ID, "default", is_default=True), - _record(_BROWSER_ID, "browser-evals"), - _record(_READONLY_ID, "locked-down", create=False), - ] - project_id = url.rsplit("/", 1)[-1] + parsed = urlsplit(url) + params = parse_qs(parsed.query) + project_id = parsed.path.rsplit("/", 1)[-1] if project_id in {_DEFAULT_ID, _BROWSER_ID, _READONLY_ID}: return next(record for record in records if record["id"] == project_id) - search = (kwargs.get("params") or {}).get("search") - if search: - records = [record for record in records if search in record["name"]] - return {"items": records, "total": len(records), "limit": 50, "offset": 0} + search = params.get("search", [""])[0] + matches = [ + record + for record in records + if search in record["name"] or search in (record.get("description") or "") + ] + limit = int(params.get("limit", ["50"])[0]) + offset = int(params.get("offset", ["0"])[0]) + return { + "items": matches[offset : offset + limit], + "total": len(matches), + "limit": limit, + "offset": offset, + } monkeypatch.setattr("hud.utils.platform.make_request_sync", fake_request) return PlatformClient("https://api.example", "key") @@ -82,6 +101,47 @@ def test_list_reads_paginated_items(platform: PlatformClient) -> None: ] +@pytest.mark.parametrize("count", [0, 50, 51, 101]) +def test_list_returns_every_page( + platform: PlatformClient, records: list[dict[str, Any]], calls: list[str], count: int +) -> None: + records[:] = [_record(str(i), f"project-{i}") for i in range(count)] + + assert [project.id for project in list_projects(platform)] == [r["id"] for r in records] + assert len(calls) == max(1, (count + 49) // 50) + + +@pytest.mark.parametrize("match_index", [0, 50, 100]) +def test_resolve_searches_until_the_exact_name_is_found( + platform: PlatformClient, + records: list[dict[str, Any]], + calls: list[str], + match_index: int, +) -> None: + records[:] = [_record(str(i), f"browser-evals-{i}") for i in range(101)] + records[match_index] = _record(_BROWSER_ID, "browser-evals") + + assert resolve_project(platform, "Browser Evals").id == _BROWSER_ID + assert len(calls) == match_index // 50 + 1 + assert all(parse_qs(urlsplit(url).query)["search"] == ["browser-evals"] for url in calls) + + +def test_resolve_exhausts_search_and_lists_all_alternatives_when_no_exact_name_exists( + platform: PlatformClient, records: list[dict[str, Any]], calls: list[str] +) -> None: + records[:] = [ + {**_record(str(i), f"project-{i}"), "description": "browser-evals"} for i in range(51) + ] + + with pytest.raises(ProjectNotFound) as excinfo: + resolve_project(platform, "browser-evals") + + assert [p.id for p in excinfo.value.available] == [r["id"] for r in records] + queries = [parse_qs(urlsplit(url).query) for url in calls] + assert [q.get("search") for q in queries] == [["browser-evals"], ["browser-evals"], None, None] + assert [q["offset"] for q in queries] == [["0"], ["50"], ["0"], ["50"]] + + def test_projects_not_enabled_matches_only_the_feature_gate() -> None: assert projects_not_enabled( HudRequestError( From 6637b63d5fb74f04115e9aca29e20031bc50bca2 Mon Sep 17 00:00:00 2001 From: Asadullo Ganiev <62354884+solvemproblr@users.noreply.github.com> Date: Mon, 14 Sep 2026 11:33:52 +0500 Subject: [PATCH 9/9] fix(cli): fail task sync on rejected uploads --- hud/cli/sync.py | 2 +- hud/cli/tests/test_sync_projects.py | 36 +++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/hud/cli/sync.py b/hud/cli/sync.py index b8097a6ae..929fe9adc 100644 --- a/hud/cli/sync.py +++ b/hud/cli/sync.py @@ -381,7 +381,7 @@ def sync_tasks_command( ) except HudRequestError as e: _show_upload_error(e, hud_console) - return + raise typer.Exit(1) from e created = int(result.get("tasks_created", 0)) updated = int(result.get("tasks_updated", 0)) diff --git a/hud/cli/tests/test_sync_projects.py b/hud/cli/tests/test_sync_projects.py index e85ed5cd2..39dd53800 100644 --- a/hud/cli/tests/test_sync_projects.py +++ b/hud/cli/tests/test_sync_projects.py @@ -6,9 +6,11 @@ from typing import TYPE_CHECKING, Any import pytest +from typer.testing import CliRunner import hud.cli.sync as sync_module from hud.eval import Task, Taskset +from hud.utils.exceptions import HudRequestError if TYPE_CHECKING: from pathlib import Path @@ -138,3 +140,37 @@ def upload(*args: Any, **kwargs: Any) -> dict[str, Any]: config = json.loads((tmp_path / ".hud" / "config.json").read_text()) assert config == {"tasksetId": "taskset-1"} + + +@pytest.mark.parametrize("status_code", [400, 403, 500]) +def test_rejected_upload_exits_with_failure( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, status_code: int +) -> None: + project_id = "22222222-2222-4222-8222-222222222222" + detail = "Taskset belongs to another Project" if status_code == 400 else "Upload rejected" + monkeypatch.chdir(tmp_path) + monkeypatch.setattr("hud.settings.settings.api_key", "test-key") + monkeypatch.setattr("hud.settings.settings.hud_api_url", "https://api.example") + (tmp_path / "tasks.json").write_text( + json.dumps([{"env": "example", "id": "solve", "slug": "one"}]) + ) + + def request(method: str, url: str, **kwargs: Any) -> dict[str, Any]: + if method == "GET" and url == f"https://api.example/v2/projects/{project_id}": + return {"id": project_id, "name": "browser-evals", "capabilities": {"create": True}} + assert method == "POST" and url == "https://api.example/v2/tasks/upload" + assert kwargs["json"]["project_id"] == project_id + assert len(kwargs["json"]["tasks"]) == 1 + raise HudRequestError(detail, status_code=status_code, response_json={"detail": detail}) + + monkeypatch.setattr("hud.utils.platform.make_request_sync", request) + + result = CliRunner().invoke( + sync_module.sync_app, + ["tasks", "demo", "tasks.json", "--project", project_id, "--force", "--yes"], + ) + + assert result.exit_code == 1 + assert detail in result.output + assert "Sync complete" not in result.output + assert not (tmp_path / ".hud" / "config.json").exists()