diff --git a/README.md b/README.md index f154e6089..d302802ee 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,17 @@ A **capability** is a connection the environment exposes; a **harness** attaches From the [platform UI](https://hud.ai) you can run batches, compare models on the same taskset, and inspect every trace. -→ [Run & deploy](https://docs.hud.ai/v6/reference/runtime) +A **project** holds the environments and tasksets a team creates and decides who can see them. Deploying without one uses your team's default project, so nothing here is required to get started. To put an environment somewhere else, pin the directory once and every later `hud deploy` and `hud sync tasks` follows it: + +```bash +hud project list # projects you can see +hud project use browser-evals # writes projectId to .hud/config.json +hud project # where does a deploy here land? +``` + +`hud deploy` and `hud sync tasks` also take `--project ` as a one-command override; it does not change the directory configuration. Use `hud set HUD_DEFAULT_PROJECT=` for a machine-wide fallback. Precedence is the flag, then the directory's `.hud/config.json`, then `HUD_DEFAULT_PROJECT`, then your team default. An environment or taskset that already exists stays where it is; naming a different project fails rather than moving it. + +→ [Projects](https://docs.hud.ai/v6/reference/projects) · [Run & deploy](https://docs.hud.ai/v6/reference/runtime) ## Train on rewards diff --git a/docs/docs.json b/docs/docs.json index 483742155..24febc806 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -64,7 +64,7 @@ "groups": [ { "group": "Start here", "pages": ["v6/start/index", "v6/start/quickstart", "v6/start/overview"] }, { "group": "Guides", "pages": ["v6/guides/creating-an-environment", "v6/guides/running-an-eval", "v6/guides/training-agents"] }, - { "group": "Reference", "pages": ["v6/reference/environment", "v6/reference/tasks", "v6/reference/capabilities", "v6/reference/agents", "v6/reference/runtime", "v6/reference/graders", "v6/reference/advice", "v6/reference/training", "v6/reference/types", "v6/reference/cli", "v6/reference/telemetry"] }, + { "group": "Reference", "pages": ["v6/reference/environment", "v6/reference/tasks", "v6/reference/projects", "v6/reference/capabilities", "v6/reference/agents", "v6/reference/runtime", "v6/reference/graders", "v6/reference/advice", "v6/reference/training", "v6/reference/types", "v6/reference/cli", "v6/reference/telemetry"] }, { "group": "Advanced", "pages": [ { "group": "Advanced", "expanded": false, "pages": ["v6/advanced/extending", "v6/advanced/robots"] } ] }, diff --git a/docs/v6/guides/creating-an-environment.mdx b/docs/v6/guides/creating-an-environment.mdx index d971cc6c5..18a073f3a 100644 --- a/docs/v6/guides/creating-an-environment.mdx +++ b/docs/v6/guides/creating-an-environment.mdx @@ -289,10 +289,23 @@ hud set HUD_API_KEY=your-key # get one at hud.ai hud deploy ``` +The environment is created in the directory's resolved [HUD Project](/v6/reference/projects), or in +the team default Project when none is configured. Select another Project persistently or on this +deploy: + +```bash +hud project use browser-evals +hud deploy + +# Or override this deploy without changing the directory Project +hud deploy --project browser-evals +``` + To run a taskset from the platform, publish it too. `hud sync` uploads a taskset and only what changed: ```bash hud sync tasks my-tasks # publish tasks.py as a named taskset +hud sync tasks my-tasks --project browser-evals hud sync env # sync environment metadata ``` diff --git a/docs/v6/reference/cli.mdx b/docs/v6/reference/cli.mdx index d9d4a2627..510a5b9b6 100644 --- a/docs/v6/reference/cli.mdx +++ b/docs/v6/reference/cli.mdx @@ -1,6 +1,6 @@ --- title: "CLI" -description: "Reference for the hud command-line interface: init, dev, run, eval, models, and other commands that span the HUD environment and agent lifecycle." +description: "Reference for the hud command-line interface: init, deploy, projects, sync, eval, and other commands across the HUD lifecycle." icon: "terminal" --- @@ -64,6 +64,7 @@ rebuilds that environment. ```bash hud deploy +hud deploy --project browser-evals ``` | Option | Description | @@ -71,6 +72,7 @@ hud deploy | `--all`, `-a` | Deploy all environments in the directory. | | `--env`, `-e` | Env var `KEY=VALUE` (repeatable). | | `--env-file` | Path to a `.env` file. | +| `--project` | One-command Project override by name or ID. Falls back to the directory Project, `HUD_DEFAULT_PROJECT`, then the team default. | ## Evaluate @@ -140,11 +142,63 @@ hud task grade fix_bug --source tasks.py --url tcp://127.0.0.1:8765 --answer ".. ## Platform +### `hud project` + +Show and choose the [HUD Project](/v6/reference/projects) where new environments and tasksets are +created. + ```bash -hud sync tasks my-taskset # publish tasks as a named taskset -hud sync env # sync environment metadata +hud project # resolved Project for the current directory +hud project list # visible Projects and access +hud project create browser-evals # create and use a Project (team admins) +hud project create browser-evals --no-use +hud project use browser-evals # save projectId to .hud/config.json +hud project use browser-evals -C ./my-env ``` +| Command or option | Description | +|-------------------|-------------| +| `hud project [--directory PATH]` | Show where new resources from a directory will be created. `-C` is an alias. | +| `hud project list` | List visible Projects, marking the team default and read-only entries. | +| `hud project create ` | Create and pin a Project. Accepts `--description`, `--directory`/`-C`, and `--no-use`. | +| `hud project use ` | Pin a directory to an existing writable Project. Accepts `--directory`/`-C`. | + +#### Project selection + +`hud deploy` and `hud sync tasks` resolve their destination in this order: + +| Priority | Source | Scope | +|----------|--------|-------| +| 1 | `--project ` | Current command | +| 2 | `.hud/config.json` → `projectId` | Current directory | +| 3 | `HUD_DEFAULT_PROJECT` | Machine-wide fallback | +| 4 | Team default Project | Zero-configuration fallback | + +`--project` does not change `.hud/config.json`. Use `hud project use ` when later deploys +and task syncs from the directory should use that Project. Set a fallback for unconfigured +directories with `hud set HUD_DEFAULT_PROJECT=`. + +`hud project list` marks the team default and read-only Projects. Pinning a directory, deploying, or +syncing to a Project requires create access. Only team admins can create Projects. + +An existing environment or taskset stays in its original Project. Selecting a different Project for +an existing resource fails rather than moving it; use its current Project or create a separately named +resource in the new one. + +### `hud sync` + +Publish task definitions or link a local directory to a deployed environment: + +```bash +hud sync tasks my-taskset # publish using resolved Project +hud sync tasks my-taskset --project browser-evals # override by Project name or ID +hud sync env # link environment metadata +``` + +`hud sync tasks --project` is a one-command override, like `hud deploy --project`. Existing resources +remain in their original Project. See +[Projects](/v6/reference/projects) for the conceptual overview. + External benchmark formats can be adapted into runnable `Taskset`s through the experimental [Harbor integration](/v6/experimental/harbor). diff --git a/docs/v6/reference/projects.mdx b/docs/v6/reference/projects.mdx new file mode 100644 index 000000000..7b87d1fae --- /dev/null +++ b/docs/v6/reference/projects.mdx @@ -0,0 +1,42 @@ +--- +title: "Projects" +description: "Understand how HUD Projects organize environments and tasksets." +icon: "folder-tree" +--- + +A HUD **Project** is the workspace boundary for a related set of environments and tasksets. Projects +also control who can see those resources and who can create new ones. + +Projects is currently a limited beta. If it is not enabled for your team, Project-specific CLI +commands stop with a message explaining how to request access; normal deploy and task sync workflows +continue to use the team default Project. + +## Default Project + +Every team has a default Project. If you deploy an environment or publish a taskset without choosing +another Project, HUD places it in the team default. This keeps the normal workflow zero-configuration. + +## Project selection + +A local environment directory can be associated with a Project. Deploys and task syncs from that +directory then use the same destination, keeping an environment and its tasksets together. + +For automation or exceptional cases, a command can select a different destination explicitly without +changing the directory association. A machine-wide preference can provide a fallback for directories +without an association. + +Project selection only controls where a **new** resource is created. It does not move resources +between Projects. + +## Resource ownership + +Each deployed environment or taskset belongs to one Project. If you need the same resource in another +Project, create it there under a separate name rather than changing the Project of the existing +resource. + +Projects may be writable or read-only for a given user. Only team admins can create Projects. + + + + + diff --git a/docs/v6/reference/tasks.mdx b/docs/v6/reference/tasks.mdx index 7a21ec070..43fb6a4c6 100644 --- a/docs/v6/reference/tasks.mdx +++ b/docs/v6/reference/tasks.mdx @@ -186,6 +186,19 @@ uploads a taskset and only what changed, a workflow covered in [creating an environment](/v6/guides/creating-an-environment#deploying-to-the-platform). In code, `diff(local, remote)` returns a `SyncPlan` describing the comparison: +To create the taskset outside the team default Project, select a directory Project first or pass an +explicit destination: + +```bash +hud project use browser-evals +hud sync tasks my-taskset + +# Or select it on the sync command +hud sync tasks my-taskset --project browser-evals +``` + +See [Projects](/v6/reference/projects) for placement precedence and existing-resource behavior. + ```python from hud.eval.sync import diff diff --git a/hud/cli/__init__.py b/hud/cli/__init__.py index 628bc2898..ba676c935 100644 --- a/hud/cli/__init__.py +++ b/hud/cli/__init__.py @@ -37,6 +37,7 @@ from .init import init_command # noqa: E402 from .jobs import jobs_app # noqa: E402 from .models import models_app # noqa: E402 +from .project import project_app # noqa: E402 from .qa import qa_app # noqa: E402 from .serve import serve_command # noqa: E402 from .sync import sync_app # noqa: E402 @@ -52,6 +53,7 @@ app.add_typer(jobs_app, name="jobs") app.add_typer(trace_app, name="trace") app.add_typer(qa_app, name="qa") +app.add_typer(project_app, name="project") @app.command(name="set") diff --git a/hud/cli/deploy.py b/hud/cli/deploy.py index dc9e26d6c..8981812a1 100644 --- a/hud/cli/deploy.py +++ b/hud/cli/deploy.py @@ -19,6 +19,7 @@ from hud.cli.utils.build_logs import poll_build_status, stream_build_logs from hud.cli.utils.config import parse_env_file, parse_key_value from hud.cli.utils.context import create_build_context_tarball, format_size +from hud.cli.utils.project import PROJECT_OPTION_HELP, Placement, resolve_writable_placement from hud.cli.utils.registry import get_registry_environment from hud.cli.utils.source import EnvironmentSource from hud.eval.runtime import ComposeProject, RuntimeConfig @@ -41,6 +42,7 @@ class _DeployPlan: name: str registry_id: str | None + placement: Placement runtime: str | None runtime_config: RuntimeConfig | None env_vars: dict[str, str] @@ -375,6 +377,7 @@ def _prepare_deploy_plan( env_file: str | None, no_env: bool, registry_id: str | None, + project: str | None, build_args: list[str] | None, build_secrets: list[str] | None, runtime: str | None, @@ -390,6 +393,7 @@ def _prepare_deploy_plan( platform, console, ) + placement = resolve_writable_placement(platform, env_source, flag=project, console=console) skip_dotenv = _skip_dotenv( env_source, env_dir, @@ -437,6 +441,7 @@ def _prepare_deploy_plan( return _DeployPlan( name=resolved_name, registry_id=registry_id, + placement=placement, runtime=normalized_runtime, runtime_config=loaded_runtime_config, env_vars=env_vars, @@ -453,6 +458,7 @@ def deploy_environment( no_cache: bool = False, verbose: bool = False, registry_id: str | None = None, + project: str | None = None, build_args: list[str] | None = None, build_secrets: list[str] | None = None, runtime: str | None = None, @@ -491,6 +497,7 @@ def deploy_environment( env_file=env_file, no_env=no_env, registry_id=registry_id, + project=project, build_args=build_args, build_secrets=build_secrets, runtime=runtime, @@ -555,6 +562,7 @@ async def _trigger_build( key: value for key, value in ( ("registry_id", plan.registry_id), + ("project_id", plan.placement.project_id), ("runtime_provider", plan.runtime), ( "runtime_config", @@ -715,6 +723,7 @@ def deploy_all( no_env: bool = False, no_cache: bool = False, verbose: bool = False, + project: str | None = None, build_args: list[str] | None = None, build_secrets: list[str] | None = None, runtime: str | None = None, @@ -755,6 +764,7 @@ def deploy_all( no_cache=no_cache, verbose=verbose, registry_id=None, + project=project, build_args=build_args, build_secrets=build_secrets, runtime=runtime, @@ -833,6 +843,11 @@ def deploy_command( help="Existing registry ID for rebuilds (advanced)", hidden=True, ), + project: str | None = typer.Option( + None, + "--project", + help=PROJECT_OPTION_HELP, + ), runtime: str | None = typer.Option( None, "--runtime", @@ -859,6 +874,7 @@ def deploy_command( no_env=no_env, no_cache=no_cache, verbose=verbose, + project=project, build_args=build_args, build_secrets=secrets, runtime=runtime, @@ -874,6 +890,7 @@ def deploy_command( no_cache=no_cache, verbose=verbose, registry_id=registry_id, + project=project, build_args=build_args, build_secrets=secrets, runtime=runtime, diff --git a/hud/cli/project.py b/hud/cli/project.py new file mode 100644 index 000000000..f4010cc33 --- /dev/null +++ b/hud/cli/project.py @@ -0,0 +1,198 @@ +"""``hud project`` — see and choose the Project new environments land in.""" + +from __future__ import annotations + +import httpx +import typer + +from hud.cli.utils.api import require_api_key +from hud.cli.utils.project import ( + Project, + ProjectNotFound, + ProjectNotWritable, + list_projects, + projects_not_enabled, + report_project_error, + require_projects_enabled, + resolve_placement, + resolve_project, +) +from hud.cli.utils.source import EnvironmentSource +from hud.utils.exceptions import HudRequestError +from hud.utils.hud_console import HUDConsole +from hud.utils.platform import PlatformClient + +project_app = typer.Typer( + name="project", + help="Show and choose the HUD Project new environments and tasksets land in", + add_completion=False, + rich_markup_mode="rich", +) + +_DIRECTORY_META_KEY = "hud_project_directory" + + +@project_app.command("list") +def list_command() -> None: + """List the Projects you can see. + + [not dim]Examples: + hud project list[/not dim] + """ + console = HUDConsole() + require_api_key("list projects") + + try: + projects = list_projects(PlatformClient.from_settings()) + except HudRequestError as e: + raise report_project_error(console, e) from e + + if not projects: + console.warning("No projects found") + console.hint("Create one with: hud project create ") + return + + console.info("Your projects:") + for project in sorted(projects, key=lambda p: (not p.is_default, p.name)): + tags: list[str] = [] + if project.is_default: + tags.append("default") + if not project.can_create: + tags.append("read-only") + suffix = f" [{', '.join(tags)}]" if tags else "" + console.info(f" {project.name} ({project.short_id}...){suffix}") + + +@project_app.command("create") +def create_command( + ctx: typer.Context, + name: str = typer.Argument(..., help="Name for the new project"), + description: str | None = typer.Option(None, "--description", help="What the project holds"), + directory: str | None = typer.Option( + None, + "--directory", + "-C", + help="Directory to pin it to (defaults to the group -C or current directory)", + ), + no_use: bool = typer.Option(False, "--no-use", help="Create without pinning this directory"), +) -> None: + """Create a Project and pin this directory to it. + + [not dim]Only team admins can create projects. + + Examples: + hud project create browser-evals + hud project create browser-evals --no-use[/not dim] + """ + console = HUDConsole() + require_api_key("create a project") + + platform = PlatformClient.from_settings() + payload: dict[str, str] = {"name": name} + if description: + payload["description"] = description + + try: + created = Project.from_record(platform.post("/projects", json=payload)) + except HudRequestError as e: + if projects_not_enabled(e): + raise report_project_error(console, e) from e + if e.status_code == httpx.codes.CONFLICT: + console.error(f"A project named '{name}' already exists") + console.hint(f"Pin this directory to it with: hud project use {name}") + raise typer.Exit(1) from e + if e.status_code == httpx.codes.FORBIDDEN: + console.error("Only team admins can create projects") + raise typer.Exit(1) from e + raise report_project_error(console, e) from e + + console.success(f"Created project: {created.name} ({created.short_id}...)") + if not no_use: + _pin(created, _command_directory(ctx, directory), console) + + +@project_app.command("use") +def use_command( + ctx: typer.Context, + ref: str = typer.Argument(..., help="Project name or ID"), + directory: str | None = typer.Option( + None, + "--directory", + "-C", + help="Directory to pin (defaults to the group -C or current directory)", + ), +) -> None: + """Pin a directory to a Project. + + [not dim]Writes projectId to .hud/config.json, so later deploys and task + syncs from this directory use the same project. Set a machine-wide fallback + with: hud set HUD_DEFAULT_PROJECT= + + Examples: + hud project use browser-evals + hud project use browser-evals -C ./envs/browser[/not dim] + """ + console = HUDConsole() + require_api_key("select a project") + + try: + project = resolve_project(PlatformClient.from_settings(), ref) + except (ProjectNotFound, HudRequestError) as e: + raise report_project_error(console, e) from e + if not project.can_create: + raise report_project_error(console, ProjectNotWritable(project)) + _pin(project, _command_directory(ctx, directory), console) + + +@project_app.callback(invoke_without_command=True) +def project_callback( + ctx: typer.Context, + directory: str = typer.Option(".", "--directory", "-C", help="Directory to report on"), +) -> None: + """Show the Project this directory places new environments and tasksets in. + + [not dim]Examples: + hud project # where does a deploy here land? + hud project list # projects you can see + hud project use browser-evals # pin this directory[/not dim] + """ + ctx.meta[_DIRECTORY_META_KEY] = directory + if ctx.invoked_subcommand is not None: + return + + console = HUDConsole() + require_api_key("resolve the current project") + platform = PlatformClient.from_settings() + try: + require_projects_enabled(platform) + placement = resolve_placement( + platform, + EnvironmentSource.open(directory), + flag=None, + ) + except (ProjectNotFound, HudRequestError) as e: + raise report_project_error(console, e) from e + + console.info(f"Project: {placement.label}") + if placement.project is None: + console.hint("Pin a different one with: hud project use ") + elif not placement.project.can_create: + console.warning("You do not have create access to this Project") + + +def _command_directory(ctx: typer.Context, directory: str | None) -> str: + """Let subcommand -C override the group-level project -C option.""" + if directory is not None: + return directory + inherited = ctx.meta.get(_DIRECTORY_META_KEY) + return inherited if isinstance(inherited, str) else "." + + +def _pin(project: Project, directory: str, console: HUDConsole) -> None: + changed = EnvironmentSource.open(directory).save_config({"projectId": project.id}) + console.success(f"Using project: {project.name} ({project.short_id}...)") + if changed: + console.dim_info("Config saved to:", ".hud/config.json") + + +__all__ = ["project_app"] diff --git a/hud/cli/sync.py b/hud/cli/sync.py index fab060487..929fe9adc 100644 --- a/hud/cli/sync.py +++ b/hud/cli/sync.py @@ -11,6 +11,11 @@ import typer from hud.cli.utils.api import require_api_key +from hud.cli.utils.project import ( + PROJECT_OPTION_HELP, + require_writable_placement, + resolve_placement_or_exit, +) from hud.cli.utils.registry import ( RegistryEnvironment, get_registry_environment, @@ -247,6 +252,11 @@ def sync_tasks_command( "--id", help="Taskset ID directly (skip name resolution)", ), + project: str | None = typer.Option( + None, + "--project", + help=PROJECT_OPTION_HELP, + ), task_filter: str | None = typer.Option( None, "--task", @@ -318,6 +328,12 @@ def sync_tasks_command( # Creating a new taskset is only allowed when targeting an explicit name # (not an --id or a stored id, which must already exist). allow_create = taskset is not None and taskset_id is None + placement = resolve_placement_or_exit( + platform, + EnvironmentSource.open(), + flag=project, + console=hud_console, + ) try: remote_taskset = _fetch_remote_taskset( @@ -348,6 +364,8 @@ def sync_tasks_command( hud_console.info("\n --dry-run: no changes made") return + require_writable_placement(placement, hud_console) + if not yes and not hud_console.confirm("Proceed?", default=False): hud_console.info("Aborted.") return @@ -355,10 +373,15 @@ def sync_tasks_command( # Upload tasks; the platform validates referenced environments. hud_console.progress_message("Uploading tasks...") try: - result = upload_taskset(platform, plan.taskset_name, plan.to_apply) + result = upload_taskset( + platform, + plan.taskset_name, + plan.to_apply, + project_id=placement.project_id, + ) except HudRequestError as e: _show_upload_error(e, hud_console) - return + raise typer.Exit(1) from e created = int(result.get("tasks_created", 0)) updated = int(result.get("tasks_updated", 0)) diff --git a/hud/cli/tests/test_deploy.py b/hud/cli/tests/test_deploy.py index e915092fe..e65d6d0a0 100644 --- a/hud/cli/tests/test_deploy.py +++ b/hud/cli/tests/test_deploy.py @@ -10,11 +10,15 @@ import typer from hud.cli.deploy import _resolve_environment_name +from hud.cli.utils.project import Placement, Project, ProjectSource from hud.cli.utils.registry import RegistryEnvironment from hud.cli.utils.source import EnvironmentSource from hud.utils.hud_console import HUDConsole from hud.utils.platform import PlatformClient +# Deploys that accept the team's default Project send no project_id. +_UNPLACED = Placement(project=None, source=ProjectSource.TEAM_DEFAULT) + @pytest.mark.parametrize(("value", "expected"), [("HUD", "hud"), ("modal", "modal")]) def test_normalize_runtime_uses_public_runtime_names(value: str, expected: str) -> None: @@ -232,6 +236,7 @@ def test_prepare_deploy_uses_context_recipe( env_file=None, no_env=True, registry_id=None, + project=None, build_args=None, build_secrets=None, runtime=None, @@ -331,6 +336,7 @@ def test_prepare_deploy_rejects_image_config_for_compose_context( env_file=None, no_env=True, registry_id=None, + project=None, build_args=None, build_secrets=None, runtime=None, @@ -450,6 +456,7 @@ async def test_upload_url_failure(self) -> None: plan=_DeployPlan( name="test-env", registry_id=None, + placement=_UNPLACED, runtime=None, runtime_config=None, env_vars={}, @@ -481,6 +488,7 @@ async def test_upload_url_network_error(self) -> None: plan=_DeployPlan( name="test-env", registry_id=None, + placement=_UNPLACED, runtime=None, runtime_config=None, env_vars={}, @@ -493,6 +501,90 @@ async def test_upload_url_network_error(self) -> None: assert result.success is False + @pytest.mark.asyncio + async def test_trigger_build_sends_resolved_project(self) -> None: + """A resolved placement reaches the platform as project_id.""" + from hud.cli.deploy import _DeployPlan, _trigger_build + from hud.utils.platform import PlatformClient + + class FakePlatform(PlatformClient): + payload: dict[str, object] | None = None + + async def apost( + self, + path: str, + *, + json: object | None = None, + ) -> dict[str, object]: + object.__setattr__(self, "payload", json) + return {"id": "build-1", "registry_id": "registry-1"} + + platform = FakePlatform("https://api.example", "key") + await _trigger_build( + platform, + build_id="build-1", + plan=_DeployPlan( + name="test-env", + registry_id=None, + placement=Placement( + project=Project( + id="project-1", + name="browser-evals", + is_default=False, + can_create=True, + ), + source=ProjectSource.FLAG, + ), + runtime=None, + runtime_config=None, + env_vars={}, + build_args={}, + build_secrets={}, + ), + no_cache=False, + ) + + assert platform.payload is not None + assert platform.payload["project_id"] == "project-1" + + @pytest.mark.asyncio + async def test_trigger_build_omits_project_for_the_team_default(self) -> None: + """The zero-config deploy stays byte-identical to before projects existed.""" + from hud.cli.deploy import _DeployPlan, _trigger_build + from hud.utils.platform import PlatformClient + + class FakePlatform(PlatformClient): + payload: dict[str, object] | None = None + + async def apost( + self, + path: str, + *, + json: object | None = None, + ) -> dict[str, object]: + object.__setattr__(self, "payload", json) + return {"id": "build-1", "registry_id": "registry-1"} + + platform = FakePlatform("https://api.example", "key") + await _trigger_build( + platform, + build_id="build-1", + plan=_DeployPlan( + name="test-env", + registry_id=None, + placement=_UNPLACED, + runtime=None, + runtime_config=None, + env_vars={}, + build_args={}, + build_secrets={}, + ), + no_cache=False, + ) + + assert platform.payload is not None + assert "project_id" not in platform.payload + class TestSaveDeployLink: """Tests for _save_deploy_link function.""" @@ -513,6 +605,7 @@ def test_saves_deploy_link(self, tmp_path: Path) -> None: saved = json.load(f) assert saved["registryId"] == "test-registry-id-12345" + assert "projectId" not in saved def test_creates_hud_directory(self, tmp_path: Path) -> None: """Test that .hud directory is created if missing.""" diff --git a/hud/cli/tests/test_project_command.py b/hud/cli/tests/test_project_command.py new file mode 100644 index 000000000..fa9bb624f --- /dev/null +++ b/hud/cli/tests/test_project_command.py @@ -0,0 +1,133 @@ +"""CLI parsing for Project commands.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING +from unittest.mock import MagicMock + +import pytest +from typer.testing import CliRunner + +from hud.cli import project +from hud.cli.utils.project import Project +from hud.utils.exceptions import HudRequestError + +if TYPE_CHECKING: + from pathlib import Path + + +@pytest.fixture +def project_record() -> Project: + return Project( + id="22222222-2222-4222-8222-222222222222", + name="browser-evals", + is_default=False, + can_create=True, + ) + + +def _projects_disabled() -> HudRequestError: + return HudRequestError( + "Request failed: Projects are not enabled", + status_code=403, + response_json={"error": "forbidden", "detail": "Projects are not enabled"}, + ) + + +def test_bare_project_reports_disabled_feature(monkeypatch: pytest.MonkeyPatch) -> None: + platform = MagicMock() + platform.get.side_effect = _projects_disabled() + monkeypatch.setattr(project, "require_api_key", lambda _: None) + monkeypatch.setattr(project.PlatformClient, "from_settings", lambda: platform) + + result = CliRunner().invoke(project.project_app) + + assert result.exit_code == 1 + assert "Projects are not enabled for your team" in result.output + assert "Failed to reach" not in result.output + platform.get.assert_called_once_with("/projects", params={"limit": 1}) + + +def test_create_distinguishes_disabled_feature_from_admin_access( + monkeypatch: pytest.MonkeyPatch, +) -> None: + platform = MagicMock() + platform.post.side_effect = _projects_disabled() + monkeypatch.setattr(project, "require_api_key", lambda _: None) + monkeypatch.setattr(project.PlatformClient, "from_settings", lambda: platform) + + result = CliRunner().invoke(project.project_app, ["create", "browser-evals"]) + + assert result.exit_code == 1 + assert "Projects are not enabled for your team" in result.output + assert "Only team admins" not in result.output + + +def test_group_directory_is_inherited_by_use( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + project_record: Project, +) -> None: + pinned: list[str] = [] + monkeypatch.setattr(project, "require_api_key", lambda _: None) + monkeypatch.setattr(project, "resolve_project", lambda _platform, _ref: project_record) + monkeypatch.setattr( + project, "_pin", lambda _project, directory, _console: pinned.append(directory) + ) + + result = CliRunner().invoke( + project.project_app, + ["-C", str(tmp_path), "use", "browser-evals"], + ) + + assert result.exit_code == 0 + assert pinned == [str(tmp_path)] + + +def test_subcommand_directory_overrides_group_directory( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + project_record: Project, +) -> None: + pinned: list[str] = [] + override = tmp_path / "override" + monkeypatch.setattr(project, "require_api_key", lambda _: None) + monkeypatch.setattr(project, "resolve_project", lambda _platform, _ref: project_record) + monkeypatch.setattr( + project, "_pin", lambda _project, directory, _console: pinned.append(directory) + ) + + result = CliRunner().invoke( + project.project_app, + ["-C", str(tmp_path), "use", "browser-evals", "-C", str(override)], + ) + + assert result.exit_code == 0 + assert pinned == [str(override)] + + +def test_group_directory_is_inherited_by_create( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + project_record: Project, +) -> None: + pinned: list[str] = [] + platform = MagicMock() + platform.post.return_value = { + "id": project_record.id, + "name": project_record.name, + "capabilities": {"create": True}, + } + monkeypatch.setattr(project, "require_api_key", lambda _: None) + monkeypatch.setattr(project.PlatformClient, "from_settings", lambda: platform) + monkeypatch.setattr( + project, "_pin", lambda _project, directory, _console: pinned.append(directory) + ) + + result = CliRunner().invoke( + project.project_app, + ["-C", str(tmp_path), "create", "browser-evals"], + ) + + assert result.exit_code == 0 + assert pinned == [str(tmp_path)] diff --git a/hud/cli/tests/test_sync_projects.py b/hud/cli/tests/test_sync_projects.py new file mode 100644 index 000000000..39dd53800 --- /dev/null +++ b/hud/cli/tests/test_sync_projects.py @@ -0,0 +1,176 @@ +"""Project placement behavior for ``hud sync tasks``.""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any + +import pytest +from typer.testing import CliRunner + +import hud.cli.sync as sync_module +from hud.eval import Task, Taskset +from hud.utils.exceptions import HudRequestError + +if TYPE_CHECKING: + from pathlib import Path + + +class _ReadOnlyPlatform: + def get(self, url: str, *, params: dict[str, Any] | None = None) -> dict[str, Any]: + assert url == "/projects" + return { + "items": [ + { + "id": "33333333-3333-4333-8333-333333333333", + "name": "locked-down", + "capabilities": {"create": False}, + } + ] + } + + +class _WritablePlatform: + def get(self, url: str, *, params: dict[str, Any] | None = None) -> dict[str, Any]: + assert url == "/projects" + return { + "items": [ + { + "id": "22222222-2222-4222-8222-222222222222", + "name": "browser-evals", + "capabilities": {"create": True}, + } + ] + } + + +def _run_sync( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + remote: Taskset, + *, + dry_run: bool, +) -> None: + task = Task(env="example", id="solve", slug="one") + local = Taskset("demo", [task]) + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(sync_module, "require_api_key", lambda _: None) + monkeypatch.setattr( + sync_module.PlatformClient, + "from_settings", + lambda: _ReadOnlyPlatform(), + ) + monkeypatch.setattr(sync_module, "_load_local_taskset", lambda *args, **kwargs: local) + monkeypatch.setattr(sync_module, "_fetch_remote_taskset", lambda *args, **kwargs: remote) + monkeypatch.setattr( + sync_module, + "upload_taskset", + lambda *args, **kwargs: pytest.fail("read-only no-op must not upload"), + ) + + sync_module.sync_tasks_command( + taskset="demo", + source=".", + taskset_id=None, + project="locked-down", + task_filter=None, + exclude=None, + yes=True, + dry_run=dry_run, + force=False, + export=None, + ) + + +def test_read_only_project_allows_up_to_date_sync( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + task = Task(env="example", id="solve", slug="one") + _run_sync(monkeypatch, tmp_path, Taskset("demo", [task]), dry_run=False) + + +def test_read_only_project_allows_dry_run( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _run_sync(monkeypatch, tmp_path, Taskset("demo", []), dry_run=True) + + +def test_project_override_does_not_pin_directory( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + task = Task(env="example", id="solve", slug="one") + local = Taskset("demo", [task]) + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(sync_module, "require_api_key", lambda _: None) + monkeypatch.setattr( + sync_module.PlatformClient, + "from_settings", + lambda: _WritablePlatform(), + ) + monkeypatch.setattr(sync_module, "_load_local_taskset", lambda *args, **kwargs: local) + monkeypatch.setattr( + sync_module, + "_fetch_remote_taskset", + lambda *args, **kwargs: Taskset("demo", []), + ) + + def upload(*args: Any, **kwargs: Any) -> dict[str, Any]: + assert kwargs["project_id"] == "22222222-2222-4222-8222-222222222222" + return {"taskset_id": "taskset-1", "tasks_created": 1} + + monkeypatch.setattr(sync_module, "upload_taskset", upload) + + sync_module.sync_tasks_command( + taskset="demo", + source=".", + taskset_id=None, + project="browser-evals", + task_filter=None, + exclude=None, + yes=True, + dry_run=False, + force=False, + export=None, + ) + + config = json.loads((tmp_path / ".hud" / "config.json").read_text()) + assert config == {"tasksetId": "taskset-1"} + + +@pytest.mark.parametrize("status_code", [400, 403, 500]) +def test_rejected_upload_exits_with_failure( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, status_code: int +) -> None: + project_id = "22222222-2222-4222-8222-222222222222" + detail = "Taskset belongs to another Project" if status_code == 400 else "Upload rejected" + monkeypatch.chdir(tmp_path) + monkeypatch.setattr("hud.settings.settings.api_key", "test-key") + monkeypatch.setattr("hud.settings.settings.hud_api_url", "https://api.example") + (tmp_path / "tasks.json").write_text( + json.dumps([{"env": "example", "id": "solve", "slug": "one"}]) + ) + + def request(method: str, url: str, **kwargs: Any) -> dict[str, Any]: + if method == "GET" and url == f"https://api.example/v2/projects/{project_id}": + return {"id": project_id, "name": "browser-evals", "capabilities": {"create": True}} + assert method == "POST" and url == "https://api.example/v2/tasks/upload" + assert kwargs["json"]["project_id"] == project_id + assert len(kwargs["json"]["tasks"]) == 1 + raise HudRequestError(detail, status_code=status_code, response_json={"detail": detail}) + + monkeypatch.setattr("hud.utils.platform.make_request_sync", request) + + result = CliRunner().invoke( + sync_module.sync_app, + ["tasks", "demo", "tasks.json", "--project", project_id, "--force", "--yes"], + ) + + assert result.exit_code == 1 + assert detail in result.output + assert "Sync complete" not in result.output + assert not (tmp_path / ".hud" / "config.json").exists() diff --git a/hud/cli/utils/project.py b/hud/cli/utils/project.py new file mode 100644 index 000000000..c12a71e9c --- /dev/null +++ b/hud/cli/utils/project.py @@ -0,0 +1,270 @@ +"""Project lookup and placement resolution for the CLI.""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass +from enum import Enum +from typing import TYPE_CHECKING, Any + +import typer + +from hud.utils.exceptions import HudRequestError +from hud.utils.naming import normalize_environment_name + +if TYPE_CHECKING: + from collections.abc import Iterator + + from hud.cli.utils.source import EnvironmentSource + from hud.utils.hud_console import HUDConsole + from hud.utils.platform import PlatformClient + + +class ProjectSource(Enum): + """Where a resolved Project came from, most specific first.""" + + FLAG = "--project" + CONFIG = ".hud/config.json" + GLOBAL_DEFAULT = "HUD_DEFAULT_PROJECT" + TEAM_DEFAULT = "team default" + + +PROJECT_OPTION_HELP = ( + "Project for this command (name or ID). Defaults to the directory's saved " + "project, HUD_DEFAULT_PROJECT, then your team default. Does not change " + "directory configuration." +) + +_PROJECTS_DISABLED_DETAIL = "projects are not enabled" +_PROJECTS_DISABLED_ERROR = "projects_not_enabled" + + +@dataclass(frozen=True) +class Project: + id: str + name: str + is_default: bool + can_create: bool + + @classmethod + def from_record(cls, data: dict[str, Any]) -> Project: + capabilities = data.get("capabilities") + return cls( + id=str(data["id"]), + name=str(data.get("name") or "unnamed"), + is_default=bool(data.get("is_default")), + can_create=bool(capabilities.get("create")) + if isinstance(capabilities, dict) + else False, + ) + + @property + def short_id(self) -> str: + return self.id[:8] + + +@dataclass(frozen=True) +class Placement: + """The Project selected for the current directory.""" + + project: Project | None + source: ProjectSource + + @property + def project_id(self) -> str | None: + """The id to send to the platform, or None to accept the team default.""" + return self.project.id if self.project else None + + @property + def label(self) -> str: + if self.project is None: + return "team default Project" + return f"{self.project.name} (via {self.source.value})" + + +class ProjectNotFound(LookupError): + """No visible Project matches the given reference.""" + + def __init__(self, ref: str, available: list[Project]) -> None: + self.ref = ref + self.available = available + super().__init__(f"No project found matching '{ref}'") + + +class ProjectNotWritable(PermissionError): + """The caller may see the Project but may not create resources in it.""" + + def __init__(self, project: Project) -> None: + self.project = project + super().__init__( + f"You do not have permission to create environments or tasksets in " + f"project '{project.name}'" + ) + + +def list_projects(platform: PlatformClient) -> list[Project]: + """Every Project visible to the caller.""" + return list(_iter_projects(platform)) + + +def _iter_projects(platform: PlatformClient, *, search: str | None = None) -> Iterator[Project]: + params: dict[str, str | int] = {"limit": 50, "offset": 0} + if search is not None: + params["search"] = search + offset = 0 + while True: + params["offset"] = offset + data = platform.get("/projects", params=params) + projects = _projects_from_page(data) + yield from projects + if not projects: + return + offset += len(data["items"]) + if offset >= data["total"]: + return + + +def require_projects_enabled(platform: PlatformClient) -> None: + """Check access to the feature-gated Projects API.""" + platform.get("/projects", params={"limit": 1}) + + +def _projects_from_page(data: Any) -> list[Project]: + """Parse the platform's paginated Project response.""" + records = data.get("items") if isinstance(data, dict) else None + if not isinstance(records, list): + return [] + return [Project.from_record(item) for item in records if isinstance(item, dict)] + + +def resolve_project(platform: PlatformClient, ref: str) -> Project: + """Map a Project name or id to the Project itself. + + Names are normalized the same way the platform normalizes them on create, + so `My Project` and `my-project` resolve to the same row. + """ + try: + project_id = str(uuid.UUID(ref)) + except ValueError: + name = normalize_environment_name(ref, default="") + projects = _iter_projects(platform, search=name) + match = next((p for p in projects if p.name == name), None) + else: + try: + return Project.from_record(platform.get(f"/projects/{project_id}")) + except HudRequestError as e: + if e.status_code != 404: + raise + match = None + + if match is None: + raise ProjectNotFound(ref, list_projects(platform)) + return match + + +def resolve_placement( + platform: PlatformClient, + env_source: EnvironmentSource, + *, + flag: str | None, +) -> Placement: + """Resolve the configured Project.""" + from hud.settings import settings + + for ref, source in ( + (flag, ProjectSource.FLAG), + (env_source.project_id, ProjectSource.CONFIG), + (settings.default_project, ProjectSource.GLOBAL_DEFAULT), + ): + if ref: + project = resolve_project(platform, ref) + return Placement(project=project, source=source) + + return Placement(project=None, source=ProjectSource.TEAM_DEFAULT) + + +def report_project_error(console: HUDConsole, error: Exception) -> typer.Exit: + """Explain why a Project could not be used, and return the exit to raise.""" + if isinstance(error, HudRequestError) and projects_not_enabled(error): + console.error("Projects are not enabled for your team") + console.hint("Contact HUD to enable the Projects beta for your team") + elif isinstance(error, ProjectNotFound): + console.error(str(error)) + if error.available: + console.info("Projects you can see:") + for candidate in error.available: + console.info(f" {candidate.name} ({candidate.short_id}...)") + else: + console.hint("Create one with: hud project create ") + elif isinstance(error, ProjectNotWritable): + console.error(str(error)) + console.hint("Ask a project manager for 'create' scope, or pick another project") + else: + console.error(f"Failed to reach the HUD platform: {error}") + return typer.Exit(1) + + +def projects_not_enabled(error: HudRequestError) -> bool: + """Whether the Projects API rejected a caller at its feature gate.""" + if error.status_code != 403 or not isinstance(error.response_json, dict): + return False + if error.response_json.get("error") == _PROJECTS_DISABLED_ERROR: + return True + detail = error.response_json.get("detail") + return isinstance(detail, str) and detail.casefold() == _PROJECTS_DISABLED_DETAIL + + +def resolve_writable_placement( + platform: PlatformClient, + env_source: EnvironmentSource, + *, + flag: str | None, + console: HUDConsole, +) -> Placement: + """Resolve and announce a Project that accepts new resources.""" + placement = resolve_placement_or_exit(platform, env_source, flag=flag, console=console) + require_writable_placement(placement, console) + return placement + + +def resolve_placement_or_exit( + platform: PlatformClient, + env_source: EnvironmentSource, + *, + flag: str | None, + console: HUDConsole, +) -> Placement: + """Resolve and announce a Project without requiring create access.""" + try: + placement = resolve_placement(platform, env_source, flag=flag) + except (ProjectNotFound, HudRequestError) as e: + raise report_project_error(console, e) from e + + console.info(f"Project: {placement.label}") + return placement + + +def require_writable_placement(placement: Placement, console: HUDConsole) -> None: + """Exit when an operation would write to a read-only Project.""" + if placement.project is not None and not placement.project.can_create: + error = ProjectNotWritable(placement.project) + raise report_project_error(console, error) from error + + +__all__ = [ + "PROJECT_OPTION_HELP", + "Placement", + "Project", + "ProjectNotFound", + "ProjectNotWritable", + "ProjectSource", + "list_projects", + "projects_not_enabled", + "report_project_error", + "require_projects_enabled", + "require_writable_placement", + "resolve_placement", + "resolve_placement_or_exit", + "resolve_project", + "resolve_writable_placement", +] diff --git a/hud/cli/utils/source.py b/hud/cli/utils/source.py index 33fcbaac3..a7638828a 100644 --- a/hud/cli/utils/source.py +++ b/hud/cli/utils/source.py @@ -210,6 +210,12 @@ def taskset_id(self) -> str | None: value = self.load_config().get("tasksetId") return value if isinstance(value, str) else None + @property + def project_id(self) -> str | None: + """The Project this directory's environment and taskset belong to.""" + value = self.load_config().get("projectId") + return value if isinstance(value, str) else None + def iter_source_files(self) -> Iterator[Path]: for name in self.SOURCE_INCLUDE_FILES: path = self.root / name diff --git a/hud/cli/utils/tests/test_project.py b/hud/cli/utils/tests/test_project.py new file mode 100644 index 000000000..54f16cf23 --- /dev/null +++ b/hud/cli/utils/tests/test_project.py @@ -0,0 +1,276 @@ +"""Project lookup and placement precedence for CLI create-and-link flows.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any +from urllib.parse import parse_qs, urlsplit + +import pytest +import typer + +from hud.cli.utils.project import ( + Project, + ProjectNotFound, + ProjectSource, + list_projects, + projects_not_enabled, + resolve_placement, + resolve_project, + resolve_writable_placement, +) +from hud.cli.utils.source import EnvironmentSource +from hud.utils.exceptions import HudRequestError +from hud.utils.hud_console import HUDConsole +from hud.utils.platform import PlatformClient + +if TYPE_CHECKING: + from pathlib import Path + +_DEFAULT_ID = "11111111-1111-4111-8111-111111111111" +_BROWSER_ID = "22222222-2222-4222-8222-222222222222" +_READONLY_ID = "33333333-3333-4333-8333-333333333333" + + +def _record( + project_id: str, name: str, *, is_default: bool = False, create: bool = True +) -> dict[str, Any]: + return { + "id": project_id, + "name": name, + "is_default": is_default, + "capabilities": {"view": True, "create": create, "manage": False}, + } + + +@pytest.fixture +def calls() -> list[str]: + """URLs the fake platform transport was asked for.""" + return [] + + +@pytest.fixture +def records() -> list[dict[str, Any]]: + return [ + _record(_DEFAULT_ID, "default", is_default=True), + _record(_BROWSER_ID, "browser-evals"), + _record(_READONLY_ID, "locked-down", create=False), + ] + + +@pytest.fixture +def platform( + monkeypatch: pytest.MonkeyPatch, calls: list[str], records: list[dict[str, Any]] +) -> PlatformClient: + """A client backed by a paginated, searchable Projects API.""" + + def fake_request(method: str, url: str, **kwargs: Any) -> dict[str, Any]: + calls.append(url) + parsed = urlsplit(url) + params = parse_qs(parsed.query) + project_id = parsed.path.rsplit("/", 1)[-1] + if project_id in {_DEFAULT_ID, _BROWSER_ID, _READONLY_ID}: + return next(record for record in records if record["id"] == project_id) + search = params.get("search", [""])[0] + matches = [ + record + for record in records + if search in record["name"] or search in (record.get("description") or "") + ] + limit = int(params.get("limit", ["50"])[0]) + offset = int(params.get("offset", ["0"])[0]) + return { + "items": matches[offset : offset + limit], + "total": len(matches), + "limit": limit, + "offset": offset, + } + + monkeypatch.setattr("hud.utils.platform.make_request_sync", fake_request) + return PlatformClient("https://api.example", "key") + + +def _no_global_default(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("hud.settings.settings.default_project", None) + + +def test_list_reads_paginated_items(platform: PlatformClient) -> None: + assert [project.name for project in list_projects(platform)] == [ + "default", + "browser-evals", + "locked-down", + ] + + +@pytest.mark.parametrize("count", [0, 50, 51, 101]) +def test_list_returns_every_page( + platform: PlatformClient, records: list[dict[str, Any]], calls: list[str], count: int +) -> None: + records[:] = [_record(str(i), f"project-{i}") for i in range(count)] + + assert [project.id for project in list_projects(platform)] == [r["id"] for r in records] + assert len(calls) == max(1, (count + 49) // 50) + + +@pytest.mark.parametrize("match_index", [0, 50, 100]) +def test_resolve_searches_until_the_exact_name_is_found( + platform: PlatformClient, + records: list[dict[str, Any]], + calls: list[str], + match_index: int, +) -> None: + records[:] = [_record(str(i), f"browser-evals-{i}") for i in range(101)] + records[match_index] = _record(_BROWSER_ID, "browser-evals") + + assert resolve_project(platform, "Browser Evals").id == _BROWSER_ID + assert len(calls) == match_index // 50 + 1 + assert all(parse_qs(urlsplit(url).query)["search"] == ["browser-evals"] for url in calls) + + +def test_resolve_exhausts_search_and_lists_all_alternatives_when_no_exact_name_exists( + platform: PlatformClient, records: list[dict[str, Any]], calls: list[str] +) -> None: + records[:] = [ + {**_record(str(i), f"project-{i}"), "description": "browser-evals"} for i in range(51) + ] + + with pytest.raises(ProjectNotFound) as excinfo: + resolve_project(platform, "browser-evals") + + assert [p.id for p in excinfo.value.available] == [r["id"] for r in records] + queries = [parse_qs(urlsplit(url).query) for url in calls] + assert [q.get("search") for q in queries] == [["browser-evals"], ["browser-evals"], None, None] + assert [q["offset"] for q in queries] == [["0"], ["50"], ["0"], ["50"]] + + +def test_projects_not_enabled_matches_only_the_feature_gate() -> None: + assert projects_not_enabled( + HudRequestError( + "forbidden", + status_code=403, + response_json={"error": "projects_not_enabled", "detail": "Projects disabled"}, + ) + ) + assert projects_not_enabled( + HudRequestError( + "forbidden", + status_code=403, + response_json={"error": "forbidden", "detail": "Projects are not enabled"}, + ) + ) + assert not projects_not_enabled( + HudRequestError( + "forbidden", + status_code=403, + response_json={"error": "forbidden", "detail": "Missing create scope"}, + ) + ) + + +def test_resolve_matches_a_normalized_name(platform: PlatformClient) -> None: + """A human-typed name resolves through the same normalization the platform applies.""" + assert resolve_project(platform, "Browser Evals").id == _BROWSER_ID + assert resolve_project(platform, "browser-evals").id == _BROWSER_ID + + +def test_resolve_matches_an_id(platform: PlatformClient) -> None: + assert resolve_project(platform, _BROWSER_ID).name == "browser-evals" + assert resolve_project(platform, _BROWSER_ID.upper()).name == "browser-evals" + + +def test_resolve_reports_the_visible_alternatives(platform: PlatformClient) -> None: + with pytest.raises(ProjectNotFound) as excinfo: + resolve_project(platform, "nope") + + assert [p.name for p in excinfo.value.available] == [ + "default", + "browser-evals", + "locked-down", + ] + + +def test_flag_outranks_directory_config( + platform: PlatformClient, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr("hud.settings.settings.default_project", "locked-down") + source = EnvironmentSource.open(tmp_path) + source.save_config({"projectId": _DEFAULT_ID}) + + placement = resolve_placement(platform, source, flag="browser-evals") + + assert placement.project is not None + assert placement.project.id == _BROWSER_ID + assert placement.source is ProjectSource.FLAG + + +def test_directory_config_applies_without_a_flag( + platform: PlatformClient, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Placement is a property of the environment, not of who deploys it.""" + monkeypatch.setattr("hud.settings.settings.default_project", "default") + source = EnvironmentSource.open(tmp_path) + source.save_config({"projectId": _BROWSER_ID}) + + placement = resolve_placement(platform, source, flag=None) + + assert placement.project is not None + assert placement.project.id == _BROWSER_ID + assert placement.source is ProjectSource.CONFIG + + +def test_global_default_applies_to_an_unpinned_directory( + platform: PlatformClient, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr("hud.settings.settings.default_project", "browser-evals") + + placement = resolve_placement(platform, EnvironmentSource.open(tmp_path), flag=None) + + assert placement.project is not None + assert placement.project.id == _BROWSER_ID + assert placement.source is ProjectSource.GLOBAL_DEFAULT + + +def test_unconfigured_placement_sends_no_project_and_makes_no_call( + platform: PlatformClient, + calls: list[str], + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The zero-config path stays free: no project on the wire, no lookup.""" + _no_global_default(monkeypatch) + placement = resolve_placement(platform, EnvironmentSource.open(tmp_path), flag=None) + + assert placement.project_id is None + assert placement.source is ProjectSource.TEAM_DEFAULT + assert placement.label == "team default Project" + assert calls == [] + + +def test_placement_resolves_a_project_the_caller_cannot_create_in( + platform: PlatformClient, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _no_global_default(monkeypatch) + source = EnvironmentSource.open(tmp_path) + placement = resolve_placement(platform, source, flag="locked-down") + assert placement.project is not None + assert placement.project.id == _READONLY_ID + + with pytest.raises(typer.Exit): + resolve_writable_placement( + platform, + source, + flag="locked-down", + console=HUDConsole(), + ) + + +def test_from_record_defaults_capabilities_to_read_only() -> None: + """A response without capabilities is not assumed writable.""" + assert Project.from_record({"id": "x", "name": "y"}).can_create is False diff --git a/hud/eval/sync.py b/hud/eval/sync.py index 67a588334..c90e7438e 100644 --- a/hud/eval/sync.py +++ b/hud/eval/sync.py @@ -135,12 +135,16 @@ def upload_taskset( platform: PlatformClient, name: str, tasks: list[Task], + *, + project_id: str | None = None, ) -> dict[str, Any]: """Upload tasks to a platform taskset, creating it if needed.""" payload: dict[str, Any] = { "taskset_name": name, "tasks": [task_upload_payload(task) for task in tasks], } + if project_id: + payload["project_id"] = project_id data = platform.post("/tasks/upload", json=payload) return data if isinstance(data, dict) else {} diff --git a/hud/eval/tests/test_sync.py b/hud/eval/tests/test_sync.py index 0b43e4a9c..267a80b5f 100644 --- a/hud/eval/tests/test_sync.py +++ b/hud/eval/tests/test_sync.py @@ -135,6 +135,41 @@ def fake_request( } +def test_upload_taskset_places_a_new_taskset(monkeypatch: pytest.MonkeyPatch) -> None: + """`project_id` reaches the platform so a created taskset lands in that Project.""" + posted: dict[str, Any] = {} + + def fake_request(method: str, url: str, json: object = None, **kwargs: Any) -> dict[str, Any]: + posted.update(json=json) + return {} + + monkeypatch.setattr("hud.utils.platform.make_request_sync", fake_request) + + upload_taskset( + PlatformClient("https://api.example", "token"), + "demo", + [], + project_id="project-1", + ) + + assert posted["json"]["project_id"] == "project-1" + + +def test_upload_taskset_omits_project_when_unset(monkeypatch: pytest.MonkeyPatch) -> None: + """Without a project the payload is unchanged, so the default Project applies.""" + posted: dict[str, Any] = {} + + def fake_request(method: str, url: str, json: object = None, **kwargs: Any) -> dict[str, Any]: + posted.update(json=json) + return {} + + monkeypatch.setattr("hud.utils.platform.make_request_sync", fake_request) + + upload_taskset(PlatformClient("https://api.example", "token"), "demo", []) + + assert "project_id" not in posted["json"] + + def test_task_upload_payload_sends_env_and_bare_task_id() -> None: payload = task_upload_payload(Task(env="e", id="solve", args={"n": 1})) diff --git a/hud/settings.py b/hud/settings.py index 27d896b32..1b7426fc2 100644 --- a/hud/settings.py +++ b/hud/settings.py @@ -92,6 +92,12 @@ def settings_customise_sources( validation_alias="HUD_API_KEY", ) + default_project: str | None = Field( + default=None, + description="Default HUD Project name or ID for directories without a saved Project", + validation_alias="HUD_DEFAULT_PROJECT", + ) + anthropic_api_key: str | None = Field( default=None, description="API key for Anthropic models", diff --git a/hud/tests/test_settings.py b/hud/tests/test_settings.py index 25db843d0..08828791e 100644 --- a/hud/tests/test_settings.py +++ b/hud/tests/test_settings.py @@ -36,6 +36,12 @@ def test_file_tracking_can_be_disabled_by_env(monkeypatch): assert Settings().file_tracking_enabled is False +def test_default_project_accepts_a_name_or_id(monkeypatch): + monkeypatch.setenv("HUD_DEFAULT_PROJECT", "browser-evals") + + assert Settings().default_project == "browser-evals" + + def test_cli_analytics_is_independent_of_trace_telemetry(monkeypatch): monkeypatch.setenv("HUD_TELEMETRY_ENABLED", "true") monkeypatch.setenv("HUD_CLI_ANALYTICS_ENABLED", "false")