diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d0518f0 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,31 @@ +name: ci + +on: + pull_request: + push: + branches: [main] + +jobs: + # Offline by design: no network, no credentials, sub-second. Live round trips + # are a manual pre-release step, not a per-PR gate. + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + - run: uv venv --python 3.12 + - run: uv pip install -e '.[dev]' + - run: uv run ruff check . + - run: uv run ruff format --check . + - run: uv run pytest -q + # `uv run` resolves from uv.lock, whose click sits in the middle of the + # range the pin allows. Both ends are what an install in the wild gets, + # and discovery is a published contract read out of Click's own objects, + # so both ends are run and their answers compared. + - run: uv pip install 'click~=8.1.0' + - run: .venv/bin/python -m pytest -q + - run: .venv/bin/python -m unstract_cli -o json --discover full > floor.json + - run: uv pip install -U click + - run: .venv/bin/python -m pytest -q + - run: .venv/bin/python -m unstract_cli -o json --discover full > latest.json + - run: diff floor.json latest.json diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..1dc86fd --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,172 @@ +name: Release Tag and Publish Package + +on: + workflow_dispatch: + inputs: + version_bump: + description: "Version bump type. `none` publishes the version already in the repo, which is what the first release needs." + required: true + default: "patch" + type: choice + options: + - patch + - minor + - major + - none + pre_release: + description: "Publish a release candidate (`rcN`) instead of the version itself. Dispatch again with this off to promote the same version to stable." + required: false + default: false + type: boolean + release_notes: + description: "Release notes (optional)" + required: false + type: string + +jobs: + release-and-publish: + runs-on: ubuntu-latest + permissions: + contents: write + # Publishing is by PyPI Trusted Publisher, so there is no API token. + id-token: write + steps: + - name: Generate GitHub App Token + id: generate-token + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ vars.PUSH_TO_MAIN_APP_ID }} + private-key: ${{ secrets.PUSH_TO_MAIN_APP_PRIVATE_KEY }} + owner: Zipstack + repositories: | + unstract-cli + + - uses: actions/checkout@v4 + with: + token: ${{ steps.generate-token.outputs.token }} + fetch-depth: 0 + + - name: Configure Git + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - uses: astral-sh/setup-uv@v5 + + # The same install as ci.yml, so what the release run lints and tests is + # what the PR gate lints and tests. + - run: uv venv --python 3.12 + - run: uv pip install -e '.[dev]' + + # Staged locally only: nothing is committed, tagged or released until the + # checks, the build and the publish have all passed, so a failure leaves + # main untouched. + - name: Compute new version + id: version + run: | + VERSION_FILE=src/unstract_cli/__init__.py + CURRENT_VERSION=$(sed -nE 's/^__version__ = "(.*)"/\1/p' "$VERSION_FILE") + echo "Current version: $CURRENT_VERSION" + + IFS='.' read -r MAJOR MINOR PATCH <<< "$CURRENT_VERSION" + case "${{ github.event.inputs.version_bump }}" in + major) MAJOR=$((MAJOR + 1)); MINOR=0; PATCH=0 ;; + minor) MINOR=$((MINOR + 1)); PATCH=0 ;; + patch) PATCH=$((PATCH + 1)) ;; + esac + NEXT_VERSION="$MAJOR.$MINOR.$PATCH" + + # A pre-release is a candidate for NEXT_VERSION, not a version of its + # own, so it never moves the committed one: the file keeps naming the + # last stable release, and repeat dispatches count up from the rc tags + # already published for that target. + if [ "${{ github.event.inputs.pre_release }}" = "true" ]; then + HIGHEST_RC=$(git tag -l "v${NEXT_VERSION}rc*" \ + | sed -nE "s/^v${NEXT_VERSION}rc([0-9]+)$/\1/p" | sort -n | tail -1) + NEW_VERSION="${NEXT_VERSION}rc$(( ${HIGHEST_RC:-0} + 1 ))" + else + NEW_VERSION="$NEXT_VERSION" + fi + + echo "New version: $NEW_VERSION" + echo "version=$NEW_VERSION" >> "$GITHUB_OUTPUT" + + sed -i "s/^__version__ = \".*\"/__version__ = \"$NEW_VERSION\"/" "$VERSION_FILE" + + if git rev-parse -q --verify "refs/tags/v$NEW_VERSION" >/dev/null; then + echo "Tag v$NEW_VERSION already exists. Exiting..." + exit 1 + fi + + - name: Verify version update + run: | + BUILT_VERSION=$(uv run python -c "import unstract_cli; print(unstract_cli.__version__)") + echo "Package version: $BUILT_VERSION" + echo "Target version: ${{ steps.version.outputs.version }}" + if [ "$BUILT_VERSION" != "${{ steps.version.outputs.version }}" ]; then + echo "Version mismatch! Exiting..." + exit 1 + fi + + - name: Run linting + run: | + uv run ruff check . + uv run ruff format --check . + + - name: Run tests + run: uv run pytest -q + + - name: Build package + run: uv build + + # Publishing is the only step that cannot be undone, so the git metadata + # is written after it: a failure before this point leaves nothing to + # unpublish, and one after it is retried by hand against a live artifact. + - name: Publish to PyPI + run: uv publish + + - name: Commit version bump and create release + run: | + NEW_VERSION="${{ steps.version.outputs.version }}" + + # A pre-release leaves the committed version alone, and `none` + # publishes the version already in the file, so both reach here with + # nothing to commit. + if [ "${{ github.event.inputs.pre_release }}" = "true" ]; then + git checkout -- src/unstract_cli/__init__.py + elif ! git diff --quiet; then + git add src/unstract_cli/__init__.py + git commit -m "chore: bump version to $NEW_VERSION [skip ci]" + git push origin main + fi + + git tag "v$NEW_VERSION" + git push origin "v$NEW_VERSION" + + RELEASE_NOTES="${{ github.event.inputs.release_notes }}" + if [ -z "$RELEASE_NOTES" ]; then + gh release create "v$NEW_VERSION" \ + --title "Release v$NEW_VERSION" \ + --generate-notes \ + ${{ github.event.inputs.pre_release == 'true' && '--prerelease' || '' }} + else + gh release create "v$NEW_VERSION" \ + --title "Release v$NEW_VERSION" \ + --notes "$RELEASE_NOTES" \ + --generate-notes \ + ${{ github.event.inputs.pre_release == 'true' && '--prerelease' || '' }} + fi + + echo "Created release v$NEW_VERSION" + env: + GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} + + - name: Success message + run: | + echo "Published ${{ steps.version.outputs.version }} to PyPI with uv publish using Trusted Publishers" + echo "Release: https://github.com/${{ github.repository }}/releases/tag/v${{ steps.version.outputs.version }}" + echo "PyPI: https://pypi.org/project/unstract-cli/${{ steps.version.outputs.version }}/" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..130baad --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.venv/ +__pycache__/ +*.egg-info/ +.pytest_cache/ +.ruff_cache/ +dist/ +build/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..3c59880 --- /dev/null +++ b/README.md @@ -0,0 +1,141 @@ +# unstract-cli + +`unstract` — one CLI for the Unstract suite: extract a document with +LLMWhisperer, run it through a Document Studio API deployment, get structured +JSON back. It also clones one organization's resources into another. + +```bash +curl -LsSf https://raw.githubusercontent.com/Zipstack/unstract-cli/main/install.sh | sh +unstract config init +unstract config doctor +``` + +The installer fetches `uv` if it is missing and installs the CLI with it; `uv` +brings its own Python, so nothing on the machine has to match. Already have +`uv`? `uv tool install git+https://github.com/Zipstack/unstract-cli` is the same +thing. Set `UNSTRACT_CLI_SOURCE` to install a branch or a local checkout +instead. + +Or run it without installing: `uvx --from git+https://github.com/Zipstack/unstract-cli unstract --discover groups`. + +## Output + +`unstract` prints a table by default — in a terminal and in a pipe alike, so +what you see while trying something is what a script sees running it. + +**Parsing anything? Pass `-o json`.** stdout then carries exactly one envelope, +on success and on failure alike: + +```json +{"ok": true, "data": {...}, "error": null, "meta": {"contract_version": 1}} +``` + +`-o json` output depends on nothing but the command and its arguments — not the +terminal, not the config, not the environment. `-o raw` prints one field +unwrapped, for piping a document's text somewhere else. Diagnostics, warnings +and progress always go to stderr. + +Consuming the JSON: ignore fields you do not recognise, and refuse a +`meta.contract_version` above the one you were written against. `unstract +--discover full` publishes the whole contract alongside every command and flag. + +If a coding agent is driving (detected from the environment it sets), the +*default* becomes json. `--agent yes|no` forces that either way, and an explicit +`-o` always wins over both. + +Failures exit non-zero with a stable code. The codes are this CLI's own +convention, not a service's — they are the `ExitCode` enum in +`core/errors.py`, and `--discover full` publishes the table so a caller does not +have to copy it: + +| Code | Meaning | +|------|---------| +| 0 | success | +| 1 | generic failure | +| 2 | usage error | +| 3 | authentication failed | +| 4 | not found | +| 5 | validation failed | +| 6 | rate limited | +| 7 | timed out (the job handle is in the error payload — resume, do not resubmit) | +| 8 | server error | +| 9 | result already consumed (one-shot read; use `--save` next time) | +| 10 | the result was read but could not be saved — it is in `error.details` | +| 130 | interrupted (128 + SIGINT) — the user stopped it, not a failure | + +## Configuration + +`~/.unstract/config.toml`, or a project-local `.unstract.toml` found by upward +search, or `$UNSTRACT_CONFIG`, or `--config`. Every setting resolves +**flag > env > profile > built-in default**, and the CLI is fully usable with no +config file at all. The flag tier is the connection options on each product +group — `unstract docstudio --base-url … --org-id … deployment run …`, and +`--base-url`/`--api-key` on `whisper` — which override the profile for that one +invocation without writing anything. + +```toml +default_profile = "cloud-us" + +[profiles.cloud-us.llmwhisperer] +base_url = "https://llmwhisperer-api.us-central.unstract.com/api/v2" +api_key = "env:LLMWHISPERER_API_KEY" + +[profiles.cloud-us.docstudio] +base_url = "https://us-central.unstract.com" +org_id = "org_ABC123" +api_key = "env:UNSTRACT_DEPLOYMENT_KEY" + +[profiles.cloud-us.deployments.invoices] +api_name = "invoice-parser" +``` + +One `api_key` on the `docstudio` block covers every alias under it: a key minted +under **Settings → API Key Manager** authenticates every API deployment in the +organisation, so an alias normally carries only its `api_name`. Give an alias its +own `api_key` when its deployment has a separate key of its own. + +Get an LLMWhisperer key from the LLMWhisperer console; a deployment key is shown +on the API deployment's own page in the Unstract UI, and an organisation-wide one +under Settings → API Key Manager. `config init` also writes an +`onprem-example` profile as a shape to copy for a self-hosted install — its host +is a placeholder, and only the *active* profile is ever resolved. + +A credential can be written into the file literally, but `env:VAR_NAME` +indirection is what `config init` writes and what the examples use: the file +then records where a secret lives rather than the secret itself, and stays safe +to copy or commit. Either way the file is created `0600`, and `config doctor` +warns when its mode is wider than that. + +`unstract config doctor` reports where each setting resolved from — including +whether an `env:` reference is actually set in the current process — without +echoing any value. It exits non-zero when one of its own checks failed, so a +setup script can branch on it. + +A project-local `.unstract.toml` **found by upward search** may not supply +`api_key` or `base_url`. Those are ignored, with a warning; everything else in it +— profile selection, `org_id`, deployment aliases — applies as usual. A checkout +you did not write is not trusted to name the host your key is sent to. Name the +file explicitly (`--config` or `$UNSTRACT_CONFIG`) and it is honoured in full. + +What that protects is the key and the host, not the routing: `org_id`, +`api_name` and profile selection stay repo-controllable by design, so a +project file can still decide *which* deployment a command runs against on a +host you trust. Read one before you run inside a checkout you did not write. + +`clone` is the exception, and it is an operator command: a human moving one +organisation's resources into another, holding two admin Platform keys. It is +not part of the document-processing path the rest of this CLI wraps, so an agent +serving a user request should not reach for it unasked. It talks to two +deployments at once, which no single profile describes, so it takes both +endpoints as flags and both keys from `UNSTRACT_SRC_PLATFORM_KEY` / +`UNSTRACT_TGT_PLATFORM_KEY`. It exits 0 when nothing failed, which is not the +same as everything having moved: oversize and unsupported documents are skipped +by design, and `data.skipped` counts them. + +## Development + +```bash +uv venv && uv pip install -e '.[dev]' +uv run pytest # offline; no network, no credentials +uv run ruff check . +``` diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..134cf20 --- /dev/null +++ b/install.sh @@ -0,0 +1,41 @@ +#!/bin/sh +# Installs the `unstract` CLI. Override the source to install a branch or a +# local checkout: +# UNSTRACT_CLI_SOURCE=/path/to/checkout sh install.sh +set -eu + +# Flips to the bare PyPI name once the CLI is published there. +SOURCE="${UNSTRACT_CLI_SOURCE:-git+https://github.com/Zipstack/unstract-cli@main}" + +if ! command -v uv >/dev/null 2>&1; then + echo "Installing uv..." >&2 + curl -LsSf https://astral.sh/uv/install.sh | sh + # The installer only edits shell rc files, which this shell has already read. + PATH="${XDG_BIN_HOME:-${HOME}/.local/bin}:${HOME}/.cargo/bin:${PATH}" + export PATH +fi + +if ! command -v uv >/dev/null 2>&1; then + echo "uv is installed but not on PATH; open a new shell and re-run." >&2 + exit 1 +fi + +# uv fetches its own interpreter, so the CLI's Python floor is not the user's problem. +uv tool install --force "$SOURCE" + +if command -v unstract >/dev/null 2>&1; then + echo + unstract --version 2>/dev/null || true + echo "Run 'unstract config init' to get started." >&2 + exit 0 +fi + +cat >&2 <=8.1,<9", + # Writing the config file only; reading it uses the stdlib `tomllib`. + "tomli-w>=1.0", + # Pinned exactly: the CLI derives its flags and help text from these + # clients, so one that moves changes the CLI's surface. Each release re-pins + # deliberately, against the specs vendored in `src/unstract_cli/specs`. + "unstract-client==1.6.0", + "llmwhisperer-client==2.9.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.0", + "ruff>=0.6", +] + +[project.scripts] +unstract = "unstract_cli.__main__:main" +# An alias for anyone who has the name `unstract` taken by something else. +unstract-cli = "unstract_cli.__main__:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.version] +path = "src/unstract_cli/__init__.py" + +[tool.hatch.build.targets.wheel] +packages = ["src/unstract_cli"] + +[tool.ruff] +line-length = 90 +target-version = "py312" +src = ["src", "tests"] + +[tool.ruff.lint] +select = ["E", "F", "W", "I", "N", "UP", "B", "C4", "SIM"] +ignore = ["E501"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/src/unstract_cli/__init__.py b/src/unstract_cli/__init__.py new file mode 100644 index 0000000..c85c094 --- /dev/null +++ b/src/unstract_cli/__init__.py @@ -0,0 +1,3 @@ +"""Unstract CLI.""" + +__version__ = "0.1.0" diff --git a/src/unstract_cli/__main__.py b/src/unstract_cli/__main__.py new file mode 100644 index 0000000..6150bf9 --- /dev/null +++ b/src/unstract_cli/__main__.py @@ -0,0 +1,87 @@ +"""Entry point: turns every failure into an envelope plus a stable exit code. + +Click's own error handling is bypassed on purpose. By default it prints prose to +stderr and exits 1 or 2 with nothing on stdout, which leaves a caller parsing +stdout with an empty stream and no way to tell a usage error from a server +failure. +""" + +from __future__ import annotations + +import sys + +import click + +from unstract_cli.app import cli +from unstract_cli.config import ConfigError +from unstract_cli.core.errors import CLIError, ExitCode +from unstract_cli.core.output import AgentMode, OutputFormat, emit_error, resolve_format + + +def _option_from_argv(argv: list[str], *spellings: str) -> str | None: + """Best-effort read of one option before Click has parsed anything. + + A failure during parsing still has to be rendered, and the parsed context + does not exist yet at that point. + """ + for i, arg in enumerate(argv): + for spelling in spellings: + if arg.startswith(f"{spelling}="): + return arg.split("=", 1)[1] + if arg == spelling and i + 1 < len(argv): + return argv[i + 1] + return None + + +def _format_from_argv(argv: list[str]) -> OutputFormat: + """Resolve the format the same way the parsed run would.""" + try: + return resolve_format( + _option_from_argv(argv, "--output", "-o"), + _option_from_argv(argv, "--agent") or AgentMode.AUTO, + ) + except ValueError: + # An unusable value here is Click's error to report, not ours to guess + # around; render the failure in the default and let it through. + return resolve_format(None) + + +def main(argv: list[str] | None = None) -> int: + args = list(sys.argv[1:] if argv is None else argv) + fmt = _format_from_argv(args) + try: + cli.main(args=args, standalone_mode=False) + except CLIError as exc: + return int(emit_error(exc, fmt)) + except ConfigError as exc: + return int(emit_error(CLIError(str(exc), ExitCode.USAGE), fmt)) + except click.UsageError as exc: + return int( + emit_error( + CLIError(exc.format_message(), ExitCode.USAGE, hint="Run with --help."), + fmt, + ) + ) + except OSError as exc: + # Not a crash worth a traceback: a full disk or an unwritable path is + # the caller's to fix, and they still need a parseable envelope. + return int( + emit_error( + CLIError(str(exc), ExitCode.GENERIC, hint="Check the path and disk."), + fmt, + ) + ) + except (click.Abort, KeyboardInterrupt): + # Nothing here prompts, so Click's Abort can only mean an interrupt. + return int( + emit_error( + CLIError("Interrupted.", ExitCode.INTERRUPTED, retryable=True), fmt + ) + ) + except click.exceptions.Exit as exc: # --help and --version exit through here + return int(exc.exit_code) + return int(ExitCode.SUCCESS) + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/src/unstract_cli/app.py b/src/unstract_cli/app.py new file mode 100644 index 0000000..2eb6306 --- /dev/null +++ b/src/unstract_cli/app.py @@ -0,0 +1,262 @@ +"""The root Click application: global options and the command groups. + +Global options are declared once here and reach every command through the Click +context, so no command re-implements profile selection or output formatting. +""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any + +import click + +from unstract_cli.commands.config_cmd import config_group +from unstract_cli.config import ( + DOCSTUDIO, + LLMWHISPERER, + ConfigError, + ResolvedConfig, + load_config, + set_config_path, +) +from unstract_cli.core.discover import TIERS, discover +from unstract_cli.core.errors import CLIError, ExitCode +from unstract_cli.core.output import ( + AgentMode, + OutputFormat, + diagnostic, + emit_result, + resolve_format, +) + + +@dataclass +class Context: + """Everything a command needs from the global options.""" + + output: OutputFormat = OutputFormat.TABLE + quiet: bool = False + verbosity: int = 0 + profile: str | None = None + #: Socket timeout for the deployment client, which has none of its own. + transport_timeout: float | None = None + #: Command-line overrides, keyed `product.setting` -- the top tier of + #: flag > env > profile > default. + overrides: dict[str, Any] = field(default_factory=dict) + _config: ResolvedConfig | None = field(default=None, repr=False) + + @property + def config(self) -> ResolvedConfig: + """Load the config lazily, so commands that need none never read a file.""" + if self._config is None: + try: + cfg = load_config() + except ConfigError as exc: + raise CLIError(str(exc), ExitCode.USAGE) from exc + for warning in cfg.warnings: + diagnostic(warning, quiet=self.quiet, verbosity=self.verbosity) + self._config = ResolvedConfig( + file=cfg, profile_name=self.profile, overrides=self.overrides + ) + return self._config + + def override(self, product: str, values: dict[str, Any]) -> None: + """Record the connection flags given for one product. + + Called from the product group, before any command runs, so the flag tier + is populated by the time a command resolves anything. + """ + for key, value in values.items(): + if value is None: + continue + if key == "api_key": + diagnostic( + "warning: a key passed on the command line lands in shell " + "history and in the process list. Prefer the environment " + "variable or `env:` indirection in a profile.", + quiet=self.quiet, + verbosity=self.verbosity, + ) + self.overrides[f"{product}.{key}"] = value + + def secrets(self) -> list[str]: + """Resolved credentials, for scrubbing anything on its way to a stream.""" + out: list[str] = [] + for product in (LLMWHISPERER, DOCSTUDIO): + try: + if value := self.config.get(product, "api_key"): + out.append(str(value)) + except ConfigError: + continue + return out + + +pass_context = click.make_pass_decorator(Context, ensure=True) + + +# `invoke_without_command` so `--discover` is answerable on its own: it is +# how a caller learns which commands exist, so it cannot require one. +@click.group( + invoke_without_command=True, + context_settings={"help_option_names": ["-h", "--help"]}, +) +@click.option( + "--config", + "config_file", + default=None, + type=click.Path(dir_okay=False), + help="Config file to use, overriding discovery.", +) +@click.option("--profile", "-p", default=None, help="Configuration profile to use.") +@click.option( + "--output", + "-o", + default=None, + type=click.Choice([f.value for f in OutputFormat]), + help="Output format. Defaults to table; pass json to parse the output.", +) +@click.option( + "--agent", + type=click.Choice([m.value for m in AgentMode]), + default=AgentMode.AUTO.value, + help="Whether a coding agent is driving this: sets the default format to " + "json. Only the default -- an explicit --output always wins.", +) +@click.option( + "--quiet", + "-q", + is_flag=True, + default=False, + help="Suppress diagnostics on stderr. stdout is unaffected.", +) +@click.option("--verbose", "-v", count=True, help="Increase diagnostic detail.") +@click.option( + "--discover", + "discover_tier", + type=click.Choice(TIERS), + default=None, + help="Describe this CLI as JSON instead of running a command, useful for agents.", +) +@click.version_option(package_name="unstract-cli") +@click.pass_context +def cli( + ctx: click.Context, + config_file: str | None, + profile: str | None, + output: str | None, + agent: str, + quiet: bool, + verbose: int, + discover_tier: str | None, +) -> None: + """The official CLI for Unstract. + + LLMWhisperer extracts text and layout from documents; Document Studio runs + them through API deployments that return structured JSON. + + Scripting or driving this from an agent: `-o json` prints one + `{ok, data, error, meta}` envelope on stdout and nothing else, failures + exit non-zero with a stable code, and `--discover groups|summary|full` + describes the commands, their flags and the output contract as JSON without + running anything. + """ + set_config_path(config_file) + ctx.obj = Context( + output=resolve_format(output, agent), + quiet=quiet, + verbosity=verbose, + profile=profile, + ) + if discover_tier: + # Discovery is how a caller learns what to run, so it has to answer + # before any configuration exists -- and always as JSON, because the + # only consumer of a machine-readable description is a machine. + emit_result(discover(cli, discover_tier), OutputFormat.JSON) + ctx.exit(int(ExitCode.SUCCESS)) + if ctx.invoked_subcommand is None: + click.echo(ctx.get_help()) + ctx.exit(int(ExitCode.SUCCESS)) + + +def _connection_options(*, org_id: bool = False) -> Callable[[Any], Any]: + """The per-product connection settings, as flags. + + They sit on the product group rather than on each command: they say where to + connect, which is the same question for every command underneath. + """ + options = [ + click.option("--base-url", default=None, help="Service URL to use."), + click.option("--api-key", default=None, help="API key to use."), + ] + if org_id: + options.append( + click.option("--org-id", default=None, help="Organisation to run against.") + ) + + def decorate(func: Any) -> Any: + for option in reversed(options): + func = option(func) + return func + + return decorate + + +@cli.group("whisper") +@_connection_options() +@pass_context +def whisper_group(ctx: Context, **overrides: str | None) -> None: + """Extract text and layout from documents with LLMWhisperer.""" + ctx.override(LLMWHISPERER, overrides) + + +@cli.group("docstudio") +@_connection_options(org_id=True) +@click.option( + "--transport-timeout", + type=float, + default=None, + help="Seconds before a stalled connection is given up on. Unset means it " + "is not, which is what the client has always done.", +) +@pass_context +def docstudio_group( + ctx: Context, transport_timeout: float | None, **overrides: str | None +) -> None: + """Run Document Studio API deployments.""" + ctx.transport_timeout = transport_timeout + ctx.override(DOCSTUDIO, overrides) + + +@docstudio_group.group("deployment") +def deployment_group() -> None: + """Work with a deployed API.""" + + +cli.add_command(config_group) + +# Imported for their side effect of registering commands, and imported last +# because those modules hang their commands off the groups declared just above. +from unstract_cli.commands import clone_cmd, docstudio_cmd, whisper_cmd # noqa: E402,F401 + + +def command_tree() -> dict[str, Any]: + """The registered command tree, read back from Click itself. + + Describing commands anywhere but from the parser lets the description drift + from what the parser accepts, so discovery and help always read this. + """ + + def walk(command: click.Command) -> dict[str, Any]: + entry: dict[str, Any] = {"help": (command.help or "").strip().split("\n")[0]} + if isinstance(command, click.Group): + entry["commands"] = { + name: walk(sub) for name, sub in sorted(command.commands.items()) + } + return entry + + return walk(cli)["commands"] + + +__all__ = ["Context", "cli", "command_tree", "pass_context"] diff --git a/src/unstract_cli/commands/__init__.py b/src/unstract_cli/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/unstract_cli/commands/clone_cmd.py b/src/unstract_cli/commands/clone_cmd.py new file mode 100644 index 0000000..a2cfcbc --- /dev/null +++ b/src/unstract_cli/commands/clone_cmd.py @@ -0,0 +1,210 @@ +"""`unstract clone` -- copying one organization's resources into another. + +Two endpoints, each with its own key, so this command takes them as flags rather +than from a profile: a profile describes one connection. +""" + +from __future__ import annotations + +import logging +from typing import Any + +import click + +# The size grammar and the list syntax come from the client rather than a copy +# here, so both spellings of this command accept the same strings. +from unstract.clone.cli import _parse_size, _split_csv +from unstract.clone.context import ( + DEFAULT_CONCURRENCY, + CloneOptions, + OrgEndpoint, +) +from unstract.clone.exceptions import CloneError +from unstract.clone.orchestrator import clone as run_clone +from unstract.clone.report import CloneReport + +from unstract_cli.app import Context, cli, pass_context +from unstract_cli.commands.common import finish +from unstract_cli.core.errors import CLIError, ExitCode, remember_secret +from unstract_cli.core.output import OutputFormat, emit_text + + +@cli.command("clone") +@click.option("--source-url", required=True, help="Base URL of the source deployment.") +@click.option( + "--source-org", required=True, help="Source organization_id (slug in the URL path)." +) +@click.option( + "--source-key", + envvar="UNSTRACT_SRC_PLATFORM_KEY", + required=True, + help="Source admin's Platform API key (or env UNSTRACT_SRC_PLATFORM_KEY).", +) +@click.option("--target-url", required=True, help="Base URL of the target deployment.") +@click.option( + "--target-org", required=True, help="Target organization_id (slug in the URL path)." +) +@click.option( + "--target-key", + envvar="UNSTRACT_TGT_PLATFORM_KEY", + required=True, + help="Target admin's Platform API key (or env UNSTRACT_TGT_PLATFORM_KEY).", +) +@click.option( + "--dry-run", is_flag=True, help="Plan only -- do not write anything to the target." +) +@click.option( + "--include", default=None, help="Comma-separated phases to run (default: all)." +) +@click.option("--exclude", default=None, help="Comma-separated phases to skip.") +@click.option( + "--on-name-conflict", + type=click.Choice(["adopt", "abort"]), + default="adopt", + show_default=True, + help="What to do when a like-named entity exists on the target.", +) +@click.option( + "--api-prefix", + default="api/v1", + show_default=True, + help="Backend URL prefix, matching the deployment's own.", +) +@click.option( + "--file-strategy", + type=click.Choice(["platform_api", "skip"]), + default="platform_api", + show_default=True, + help="How to move Prompt Studio documents. 'skip' copies metadata only.", +) +@click.option("--skip-files", is_flag=True, help="Alias for --file-strategy=skip.") +@click.option( + "--max-file-size", + default="25MB", + show_default=True, + help="Per-file cap for the files phase. Oversize files are reported, not fatal.", +) +@click.option( + "--concurrency", + type=click.IntRange(min=1, max=32), + default=DEFAULT_CONCURRENCY, + show_default=True, + help="Per-phase worker count. 1 is strictly sequential.", +) +@click.option( + "--clone-group-members", + is_flag=True, + help="Also add group members on the target, matched by email.", +) +@pass_context +def clone( + ctx: Context, + source_url: str, + source_org: str, + source_key: str, + target_url: str, + target_org: str, + target_key: str, + **params: Any, +) -> None: + """Copy an organization's resources into another organization. + + Adapters, connectors, workflows, pipelines, API deployments, Prompt Studio + projects and their files, user groups and sharing state. Run --dry-run first: + it reports what would be written without writing it. + """ + for key in (source_key, target_key): + remember_secret(key) + _configure_logging(ctx) + + options = CloneOptions( + dry_run=params["dry_run"], + include=_split_csv(params["include"]), + exclude=_split_csv(params["exclude"]) or (), + on_name_conflict=params["on_name_conflict"], + verbose=ctx.verbosity > 0, + file_strategy="skip" if params["skip_files"] else params["file_strategy"], + max_file_size=_parse_size(params["max_file_size"]), + concurrency=params["concurrency"], + clone_group_members=params["clone_group_members"], + ) + + def endpoint(url: str, org: str, key: str) -> OrgEndpoint: + return OrgEndpoint( + base_url=url, + organization_id=org, + platform_key=key, + api_path_prefix=params["api_prefix"], + ) + + try: + report = run_clone( + endpoint(source_url, source_org, source_key), + endpoint(target_url, target_org, target_key), + options, + ) + except CloneError as exc: + raise CLIError( + str(exc), + ExitCode.USAGE, + hint="The clone could not start. Check the URLs, orgs and keys.", + ) from exc + + _finish(ctx, report) + + +def _configure_logging(ctx: Context) -> None: + """Send the orchestrator's progress to stderr, at the run's own verbosity.""" + logging.basicConfig( + level=logging.WARNING + if ctx.quiet + else (logging.DEBUG if ctx.verbosity else logging.INFO), + format="%(asctime)s %(levelname)-7s %(name)s: %(message)s", + datefmt="%H:%M:%S", + ) + + +def _skipped(report: CloneReport) -> dict[str, Any]: + """What the run did not copy, summarised at the top of the payload. + + Skipping an oversize or unsupported file is reported rather than fatal, so + the run still exits 0; a consumer reading only the exit code would otherwise + have to walk the whole report to discover documents that never arrived. + """ + by_phase = {phase.name: phase.skipped for phase in report.phases if phase.skipped} + return { + "total": sum(by_phase.values()), + "by_phase": by_phase, + "oversize_files": len(report.oversize_files), + "unsupported_files": len(report.unsupported_files), + } + + +def _finish(ctx: Context, report: CloneReport) -> None: + """Emit the report, then fail if the clone did not fully succeed.""" + failure = None + if report.aborted: + failure = f"Clone aborted: {report.abort_reason}" + elif failed := [phase.name for phase in report.phases if phase.failed]: + failure = f"Clone completed with failures in: {', '.join(sorted(failed))}" + + payload = {**report.as_dict(), "skipped": _skipped(report)} + # A person running this reads the report itself; every other format gets the + # single envelope, which carries the same content as data. + rendered = ctx.output is OutputFormat.TABLE + if rendered: + emit_text(report.render(), secrets=ctx.secrets()) + elif not failure: + finish(ctx, payload) + + if failure: + raise CLIError( + failure, + ExitCode.GENERIC, + details=None if rendered else payload, + hint="The report lists what was copied and what was not. Re-running " + "adopts what already exists on the target rather than duplicating it.", + ) + + +__all__ = ["clone"] diff --git a/src/unstract_cli/commands/common.py b/src/unstract_cli/commands/common.py new file mode 100644 index 0000000..6347d60 --- /dev/null +++ b/src/unstract_cli/commands/common.py @@ -0,0 +1,106 @@ +"""Pieces every product command shares: the wait flags and result emission.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +import click + +from unstract_cli.app import Context +from unstract_cli.core.output import emit_result + +#: Poll interval and the ceiling on the whole wait. Both are flags. +DEFAULT_INTERVAL = 3.0 +DEFAULT_TIMEOUT = 300.0 + +F = Callable[..., Any] + + +def wait_options(*, default: bool = True) -> Callable[[F], F]: + """`--wait` and its two knobs. + + ``--wait`` is a gate, not a duration: how long to wait is ``--timeout`` and + how often to check is ``--interval``, so neither has two spellings. + """ + + def decorate(func: F) -> F: + for option in reversed( + [ + click.option( + "--wait/--no-wait", + default=default, + help="Poll until the job reaches a terminal state.", + ), + click.option( + "--interval", + type=float, + default=DEFAULT_INTERVAL, + show_default=True, + help="Seconds between polls.", + ), + click.option( + "--timeout", + "wait_timeout", + type=float, + default=DEFAULT_TIMEOUT, + show_default=True, + help="Seconds to wait before giving up. The job keeps running.", + ), + click.option( + "--save", + type=click.Path(dir_okay=False), + default=None, + help="Write the result here before printing it.", + ), + ] + ): + func = option(func) + return func + + return decorate + + +def raw_fields(*fields: str) -> Callable[[click.Command], click.Command]: + """Declare what `--output raw` prints for this command, best answer first. + + Several, because one command has several answers: a queued run replies with + a handle and no result, and a status read replies with a state until there + is a result. Raw prints the first of these the answer actually carries. + + Recorded on the command so `--discover full` can report the whole list: a + caller asking for raw output has to know what it is going to get, and one + field named there would be wrong for every other shape the command returns. + """ + + def decorate(command: click.Command) -> click.Command: + command.raw_fields = fields + return command + + return decorate + + +def finish( + ctx: Context, + data: Any, + *, + raw_fields: tuple[str, ...] = (), + meta: dict[str, Any] | None = None, +) -> None: + """Emit one result envelope, scrubbing any resolved credential from it.""" + emit_result( + data, + ctx.output, + meta=meta, + raw_fields=raw_fields, + secrets=ctx.secrets(), + ) + + +__all__ = [ + "DEFAULT_INTERVAL", + "DEFAULT_TIMEOUT", + "finish", + "raw_fields", + "wait_options", +] diff --git a/src/unstract_cli/commands/config_cmd.py b/src/unstract_cli/commands/config_cmd.py new file mode 100644 index 0000000..aa02390 --- /dev/null +++ b/src/unstract_cli/commands/config_cmd.py @@ -0,0 +1,374 @@ +"""The `config` command group -- local only, no network calls. + +These commands map to no API operation: they operate purely on the local config +layer, and they are how a user or an agent bootstraps every other command. + +Nothing here prompts: `init` refuses to clobber an existing file unless +`--force` is passed, rather than asking, so the CLI behaves the same whether or +not a human is watching. +""" + +from __future__ import annotations + +from typing import Any + +import click + +from unstract_cli.config import ( + DOCSTUDIO, + KEY_SOURCES, + LLMWHISPERER, + PRODUCTS, + ConfigError, + ConfigFile, + ResolvedConfig, + config_path, + load_config, + save_config, + settings_for, + starter_profiles, +) +from unstract_cli.core.clients import llmwhisperer, translated +from unstract_cli.core.errors import CLIError, ExitCode +from unstract_cli.core.output import ( + OutputFormat, + diagnostic, + emit_result, + resolve_format, +) + +#: Keys whose value is never echoed back, even on explicit request: this output +#: is as likely to land in a log or a transcript as on a screen. +_SECRET_KEY_HINTS = ("key", "token", "secret") + + +def _is_secret(key: str) -> bool: + return any(hint in key.lower() for hint in _SECRET_KEY_HINTS) + + +def _fmt(obj: Any) -> OutputFormat: + """Output format from the root context, defaulting when invoked standalone.""" + return getattr(obj, "output", None) or resolve_format(None) + + +def _check_product(product: str) -> str: + if product not in PRODUCTS: + raise CLIError( + f"Unknown config target {product!r}.", + ExitCode.USAGE, + hint="Valid targets: " + ", ".join(PRODUCTS) + ".", + ) + return product + + +@click.group(name="config", help="Manage CLI configuration profiles (local only).") +def config_group() -> None: + """Local configuration management. These commands make no network calls.""" + + +@config_group.command("init", help="Create a starter config file with profile stubs.") +@click.option( + "--force", is_flag=True, default=False, help="Overwrite an existing config file." +) +@click.pass_obj +def config_init(obj: Any, force: bool) -> None: + path = config_path() + if path.exists() and not force: + # Never prompt: state the situation and the exact flag that resolves it. + raise CLIError( + f"Config already exists at {path}.", + ExitCode.USAGE, + hint="Pass --force to overwrite it, or edit the file directly.", + ) + + replaced = path.exists() + new = ConfigFile( + default_profile="cloud-us", profiles=starter_profiles(), path=path, exists=True + ) + written = save_config(new, path) + emit_result( + { + "created": str(written), + "default_profile": "cloud-us", + "profiles": sorted(new.profiles), + "replaced_existing": replaced, + "note": ( + "Credentials use env: indirection, so this file holds no secrets. " + "Set the referenced environment variables to authenticate. " + KEY_SOURCES + ), + }, + _fmt(obj), + ) + + +@config_group.command("list", help="List profiles defined in the config file.") +@click.pass_obj +def config_list(obj: Any) -> None: + cfg = _loaded(obj) + emit_result( + { + "path": str(cfg.path), + "exists": cfg.exists, + "default_profile": cfg.default_profile, + "profiles": { + name: { + block: sorted(settings) if isinstance(settings, dict) else settings + for block, settings in blocks.items() + } + for name, blocks in cfg.profiles.items() + }, + }, + _fmt(obj), + ) + + +@config_group.command("get") +@click.argument("product") +@click.argument("key") +@click.pass_obj +def config_get(obj: Any, product: str, key: str) -> None: + """Show a resolved setting, following flag > env > profile > default. + + PRODUCT and KEY are positional -- not flags. Credentials are reported as + configured or not, never echoed. + + \b + Examples: + unstract config get docstudio org_id + unstract --profile cloud-eu config get llmwhisperer base_url + """ + _check_product(product) + try: + value = _resolved(obj).get(product, key) + except ConfigError as exc: + raise CLIError(str(exc), ExitCode.USAGE) from exc + + emit_result( + { + "product": product, + "key": key, + "value": ("***SET***" if value else None) if _is_secret(key) else value, + "configured": value is not None, + }, + _fmt(obj), + ) + + +@config_group.command("set") +@click.argument("product") +@click.argument("key") +@click.argument("value") +@click.option("--profile", "-p", "profile", default=None, help="Profile to write to.") +@click.pass_obj +def config_set(obj: Any, product: str, key: str, value: str, profile: str | None) -> None: + """Set a value in the config file. + + PRODUCT, KEY and VALUE are positional -- not flags. Writes to the active + profile unless --profile names another. + + \b + Examples: + unstract config set docstudio org_id org_ABC123 + unstract config set llmwhisperer api_key 'env:LLMWHISPERER_API_KEY' + + \b + Prefer `env:VAR_NAME` for credentials: the file then records where the secret + lives rather than the secret itself, and a literal value also lands in your + shell history. + """ + _check_product(product) + cfg = _loaded(obj) + name = profile or getattr(obj, "profile", None) or cfg.default_profile or "cloud-us" + + cfg.profiles.setdefault(name, {}).setdefault(product, {})[key] = value + if not cfg.default_profile: + cfg.default_profile = name + written = save_config(cfg) + + warning = None + if _is_secret(key) and not value.startswith("env:"): + warning = ( + "Value stored literally. Prefer `env:VAR_NAME` so the config file holds " + "a reference rather than the secret itself." + ) + + emit_result( + { + "profile": name, + "product": product, + "key": key, + "path": str(written), + "warning": warning, + }, + _fmt(obj), + ) + + +def _probe(resolved: ResolvedConfig) -> dict[str, Any]: + """Check each product's credentials against the service, where that is possible. + + LLMWhisperer has a read-only usage endpoint, so its key can be verified for + real. A deployment has no side-effect-free endpoint -- the only thing to call + is an execution -- so its entry reports that the settings resolve and says + plainly that nothing was verified. Claiming otherwise would be worse than + not checking. + """ + out: dict[str, Any] = {} + try: + with translated(endpoint="get-usage-info"): + llmwhisperer(resolved).get_usage_info() + except CLIError as exc: + out[LLMWHISPERER] = { + "checked": True, + "ok": False, + "detail": exc.message, + "exit_code": int(exc.exit_code), + } + except ConfigError as exc: + out[LLMWHISPERER] = {"checked": False, "ok": False, "detail": str(exc)} + else: + out[LLMWHISPERER] = { + "checked": True, + "ok": True, + "detail": "The key was accepted by the usage endpoint.", + } + + resolves = all( + resolved.get(DOCSTUDIO, key) for key in ("org_id", "api_key", "base_url") + ) + out[DOCSTUDIO] = { + "checked": False, + # Null, not True: nothing was called, so there is no verdict to report. + # A `true` beside `checked: false` reads as a live check that passed. + "ok": None, + "resolved": resolves, + "detail": ( + "Credentials resolve (org and key present) but were NOT verified -- " + "the deployment API has no side-effect-free endpoint to call, so a " + "wrong key is only discovered by running a deployment." + if resolves + else "Organisation or key is missing; nothing was called." + ), + } + return out + + +@config_group.command("doctor", help="Diagnose how each setting resolves.") +@click.option( + "--probe/--no-probe", + default=False, + help="Also check the resolved credentials against the service.", +) +@click.pass_obj +def config_doctor(obj: Any, probe: bool) -> None: + """Report where each setting resolves from, without echoing any secret. + + Answers the question that costs the most time: the CLI reports a key as "not + configured", but you set it -- where is it looking? For `env:` references it + says whether the variable is present in THIS process, a shell `export` in a + login profile the CLI never inherited being the classic trap. + + Resolution is answered offline. --probe adds the second question -- does the + resolved key work -- which needs the network, so it is opt-in. + + Exits 0 only when nothing it checked failed. A setting that is simply not + configured is a report, not a failure; a setting that points somewhere and + does not arrive -- an unset `env:` variable, an unknown profile, a probe the + service rejected -- exits non-zero, because a setup script branches on that. + """ + resolved = _resolved(obj) + problems: list[str] = [] + products: dict[str, Any] = {} + for product in PRODUCTS: + entry: dict[str, Any] = {} + for key in settings_for(product): + try: + entry[key] = resolved.resolution_source(product, key) + except ConfigError as exc: + entry[key] = {"resolved": False, "source": "unset", "detail": str(exc)} + if detail := entry[key].get("detail"): + problems.append(f"{product}.{key}: {detail}") + products[product] = entry + + try: + aliases = list(resolved.deployment_aliases()) + except ConfigError as exc: + aliases = [] + problems.append(str(exc)) + for alias in aliases: + # An alias carries a key of its own, so it is a second place a project + # file can name one -- and it falls back to the profile's key silently. + if detail := resolved.withheld_detail("deployments", alias, "api_key"): + problems.append(f"deployment alias {alias}: {detail}") + try: + # Resolved the way a run resolves it: that an alias is *listed* says + # nothing about whether the settings behind it arrive. + resolved.deployment(alias) + except ConfigError as exc: + problems.append(f"deployment alias {alias}: {exc}") + + report: dict[str, Any] = { + "active_profile": resolved.active_profile, + "config_path": str(resolved.file.path), + "config_exists": resolved.file.exists, + "products": products, + "deployment_aliases": aliases, + } + if any( + not entry["api_key"]["resolved"] + for entry in products.values() + if "api_key" in entry + ): + # The next question after "no key" is always where one comes from. The + # field name avoids the word the payload scrubber redacts on. + report["getting_started"] = KEY_SOURCES + if probe: + report["probe"] = _probe(resolved) + problems += [ + f"probe {name}: {result.get('detail')}" + for name, result in report["probe"].items() + if result["ok"] is False + ] + + if problems: + report["problems"] = problems + more = "" if len(problems) == 1 else f" (+{len(problems) - 1} more)" + raise CLIError( + f"{len(problems)} configuration check(s) failed: {problems[0]}{more}", + ExitCode.GENERIC, + details=report, + hint=( + "`details` carries the whole report, including where each setting " + "resolved from." + ), + ) + emit_result(report, _fmt(obj)) + + +def _loaded(obj: Any) -> ConfigFile: + """The config file, with its warnings reported. + + These commands load the file themselves rather than through the root + context, and they are the two a user runs *to understand* their config -- + reading it here without repeating what it warned about would make them the + quietest commands in the CLI about their own subject. + """ + cfg = load_config() + for warning in cfg.warnings: + diagnostic( + warning, + quiet=getattr(obj, "quiet", False), + verbosity=getattr(obj, "verbosity", 0), + ) + return cfg + + +def _resolved(obj: Any) -> ResolvedConfig: + """The root context's config, or a freshly loaded one when invoked standalone.""" + # Already loaded means the context already reported its warnings. + if (existing := getattr(obj, "_config", None)) is not None: + return existing + return ResolvedConfig(file=_loaded(obj), profile_name=getattr(obj, "profile", None)) + + +__all__ = ["config_group"] diff --git a/src/unstract_cli/commands/docstudio_cmd.py b/src/unstract_cli/commands/docstudio_cmd.py new file mode 100644 index 0000000..24656d3 --- /dev/null +++ b/src/unstract_cli/commands/docstudio_cmd.py @@ -0,0 +1,174 @@ +"""`unstract docstudio deployment ...` -- running a deployed API. + +The deployment client reports failure by returning a status code rather than +raising, and it has no polling loop of its own, so both are handled here. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any +from urllib.parse import parse_qs, urlparse + +import click +from unstract.api_deployments.client import APIDeploymentsClient + +from unstract_cli.app import Context, deployment_group, pass_context +from unstract_cli.commands.common import finish, raw_fields, wait_options +from unstract_cli.core.clients import ( + deployment, + naming_aliases, + raise_for_result, + translated, + translating, +) +from unstract_cli.core.errors import CLIError, ExitCode +from unstract_cli.core.params import requested, spec_options +from unstract_cli.core.poll import PollSpec, classify, preflight, wait_for_completion + +PRODUCT = "docstudio" + +#: The run POST and the status GET spell the state under different names, and +#: the API answers HTTP 422 while still executing -- only the body decides. +RUN_POLL = PollSpec( + handle_field="status_check_api_endpoint", + terminal_success=("COMPLETED", "SUCCESS"), + terminal_failure=("ERROR", "ERROR_EXCEPTION", "FAILED", "STOPPED"), + status_field=("execution_status", "status"), +) + +#: What `--output raw` prints, best answer first. A queued run answers with a +#: handle and no result, and a status read answers with a state until there is +#: one, so a single field would be wrong for two of the three shapes. +RUN_RAW = ("extraction_result", "execution_id") +STATUS_RAW = ("extraction_result", "execution_status") + +#: Parameters the run POST and the status GET share: what was asked for in the +#: run has to be asked for again when the result is read. +_SHARED_WITH_STATUS = ("include_metadata", "include_metrics", "include_extracted_text") + + +@raw_fields(*RUN_RAW) +@deployment_group.command("run") +@click.argument("target") +@click.argument("files", nargs=-1, required=True, type=click.Path(exists=True)) +@wait_options() +@spec_options( + PRODUCT, + "execute", + client_method=APIDeploymentsClient.structure_file, + # `files` is the FILES argument; `timeout` selects the server's own + # execution mode and would fight the CLI's polling for the same job. + exclude=("files", "timeout"), +) +@pass_context +def run( + ctx: Context, + target: str, + files: tuple[str, ...], + wait: bool, + interval: float, + wait_timeout: float, + save: str | None, + **params: Any, +) -> None: + """Run a deployment against one or more documents. + + TARGET is a deployment alias or an API name. With --wait (the default) this + polls until the execution finishes and returns its result. + """ + client = deployment(ctx.config, target, ctx.transport_timeout) + sent = requested(params) + if save: + preflight(save) + with naming_aliases(ctx.config, target), translated(endpoint=client.api_url): + # Queued execution, so the request returns a handle instead of holding + # the connection open for the length of the job. + started = client.structure_file(list(files), timeout=0, **sent) + raise_for_result(started, endpoint=client.api_url) + + if not wait: + # The ack names no execution of its own: the handle has to be read + # back out of the endpoint it hands you, and `meta` is where the + # CLI puts what it had to derive. + finish(ctx, started, raw_fields=RUN_RAW, meta=_handle_meta(started)) + return + + result = wait_for_completion( + initial=started, + spec=RUN_POLL, + poll=_status_poller( + client, {k: v for k, v in sent.items() if k in _SHARED_WITH_STATUS} + ), + save=save, + interval=interval, + timeout=wait_timeout, + on_status=lambda status: ( + click.echo(f"status: {status}", err=True) if not ctx.quiet else None + ), + ) + # A waited result names no execution, so the handle is returned as meta for + # correlation. + finish(ctx, result, raw_fields=RUN_RAW, meta=_handle_meta(started)) + + +def _handle_meta(started: dict[str, Any]) -> dict[str, Any]: + """The execution's identity, from wherever the run response carries it.""" + if execution_id := started.get("execution_id"): + return {"execution_id": execution_id} + endpoint = str(started.get("status_check_api_endpoint") or "") + found = parse_qs(urlparse(endpoint).query).get("execution_id") + return {"execution_id": found[0]} if found else {} + + +def _status_poller( + client: APIDeploymentsClient, params: dict[str, Any] +) -> Callable[[str], dict[str, Any]]: + """Poll one execution, failing on a status code the poll loop cannot use.""" + + def poll(endpoint: str) -> dict[str, Any]: + result = client.check_execution_status(endpoint, **params) + # A retryable status is left to the client's own retry policy, which has + # already run; the client reports those as still pending. + if not result.get("pending"): + raise_for_result(result, endpoint=client.api_url) + return result + + return translating(poll, client.api_url) + + +@raw_fields(*STATUS_RAW) +@deployment_group.command("status") +@click.argument("target") +@click.argument("execution_id") +@spec_options( + PRODUCT, + "status", + client_method=APIDeploymentsClient.check_execution_status, + exclude=("execution_id",), +) +@pass_context +def status(ctx: Context, target: str, execution_id: str, **params: Any) -> None: + """Report the state of a running or finished execution.""" + client = deployment(ctx.config, target, ctx.transport_timeout) + endpoint = f"{client.api_url}?execution_id={execution_id}" + with naming_aliases(ctx.config, target), translated(endpoint=client.api_url): + result = client.check_execution_status(endpoint, **requested(params)) + if not result.get("pending"): + raise_for_result(result, endpoint=client.api_url) + # A finished-and-failed execution is reported inside an HTTP 200, so the + # status code alone would call this a success. + if classify(result, RUN_POLL) == "failure": + raise CLIError( + f"Execution {execution_id} finished with status " + f"{result.get('execution_status')!r}.", + ExitCode.VALIDATION, + details=result, + endpoint=client.api_url, + hint="Inspect `details` for the per-file error, or check the execution logs.", + extra={"execution_id": execution_id}, + ) + finish(ctx, result, raw_fields=STATUS_RAW) + + +__all__ = ["run", "status"] diff --git a/src/unstract_cli/commands/whisper_cmd.py b/src/unstract_cli/commands/whisper_cmd.py new file mode 100644 index 0000000..2b3f1ff --- /dev/null +++ b/src/unstract_cli/commands/whisper_cmd.py @@ -0,0 +1,385 @@ +"""`unstract whisper ...` -- text and layout extraction. + +Every flag below the command name is derived from the committed spec, so this +module holds only what the spec cannot say: which parameter is an argument, +which the CLI owns, and how a result is polled for and retrieved. +""" + +from __future__ import annotations + +from typing import Any + +import click +from unstract.llmwhisperer.client_v2 import LLMWhispererClientV2 + +from unstract_cli.app import Context, pass_context, whisper_group +from unstract_cli.commands.common import finish, raw_fields, wait_options +from unstract_cli.core.clients import llmwhisperer, translated, translating +from unstract_cli.core.errors import CLIError, ExitCode, remember_secret +from unstract_cli.core.params import requested, spec_options +from unstract_cli.core.poll import ( + PollSpec, + classify, + extract_status, + persist, + preflight, + wait_for_completion, +) + +PRODUCT = "llmwhisperer" + +#: Terminal states as the body reports them. `unknown` is one: the service +#: returns it for a hash it no longer knows, which no amount of polling changes. +EXTRACT_POLL = PollSpec( + handle_field="whisper_hash", + terminal_success=("processed",), + terminal_failure=("error", "unknown"), + status_field="status", +) + +#: `--output raw` prints one field rather than the whole payload. Extraction +#: results carry the text under this name. +RAW_TEXT = ("result_text",) + + +def _is_url(source: str) -> bool: + return source.startswith(("http://", "https://")) + + +@raw_fields(*RAW_TEXT) +@whisper_group.command("extract") +@click.argument("source") +@wait_options() +@spec_options( + PRODUCT, + "extract", + client_method=LLMWhispererClientV2.whisper, + # `url` is the SOURCE argument when it looks like one. + exclude=("url",), +) +@pass_context +def extract( + ctx: Context, + source: str, + wait: bool, + interval: float, + wait_timeout: float, + save: str | None, + **params: Any, +) -> None: + """Extract text from a document, given a file path or a URL. + + With --wait (the default) this returns the extracted text. With --no-wait it + returns the whisper_hash, and `whisper status` and `whisper retrieve` take + it from there. + """ + client = llmwhisperer(ctx.config) + sent = requested(params) + if save: + preflight(save) + + if sent.get("use_webhook") and wait: + raise CLIError( + "--wait and --use-webhook are mutually exclusive.", + ExitCode.USAGE, + hint=( + "A webhook delivers the result itself; pass --no-wait to submit " + "and return immediately." + ), + ) + + with translated(endpoint="whisper"): + # The CLI's own poll loop is used over the client's so that waiting + # behaves the same for every product. + accepted = client.whisper( + **({"url": source} if _is_url(source) else {"file_path": source}), + **sent, + wait_for_completion=False, + ) + + if not wait: + finish(ctx, accepted) + return + + result = wait_for_completion( + initial=accepted, + spec=EXTRACT_POLL, + poll=translating(client.whisper_status, "whisper-status"), + retrieve=translating( + lambda handle: _extraction(client.whisper_retrieve(handle)), + "whisper-retrieve", + ), + save=save, + interval=interval, + timeout=wait_timeout, + on_status=lambda status: ( + click.echo(f"status: {status}", err=True) if not ctx.quiet else None + ), + ) + # Waiting returns the text, which identifies the job nowhere; the hash is + # what a later status, retrieve or highlights call needs. + finish( + ctx, + result, + raw_fields=RAW_TEXT, + meta={"whisper_hash": accepted.get("whisper_hash")} + if accepted.get("whisper_hash") + else None, + ) + + +def _extraction(payload: Any) -> Any: + """The extracted result out of a retrieve response. + + A retrieve is the acknowledging read, so an empty result here is a document + that was processed, billed and consumed for nothing -- reporting it as a + success would hide that. + """ + result = payload.get("extraction", payload) if isinstance(payload, dict) else payload + if not result: + raise CLIError( + "The service returned no extraction for a completed job.", + ExitCode.SERVER_ERROR, + details=payload, + hint=( + "The read has been acknowledged, so it cannot be repeated. " + "`details` carries the response exactly as it arrived." + ), + ) + return result + + +@whisper_group.command("status") +@click.argument("whisper_hash") +@pass_context +def status(ctx: Context, whisper_hash: str) -> None: + """Report the state of a submitted extraction.""" + client = llmwhisperer(ctx.config) + with translated(endpoint="whisper-status"): + result = client.whisper_status(whisper_hash) + # A failed extraction is reported inside an HTTP 200, so the status code + # alone would call this a success. + if classify(result, EXTRACT_POLL) == "failure": + raise CLIError( + f"Extraction finished with status {extract_status(result)!r}.", + ExitCode.VALIDATION, + details=result, + endpoint="whisper-status", + hint=( + "`details` carries the service's own message. An `unknown` status " + "means the service no longer holds this hash." + ), + extra={"whisper_hash": whisper_hash}, + ) + finish(ctx, result) + + +@raw_fields(*RAW_TEXT) +@whisper_group.command("retrieve") +@click.argument("whisper_hash") +@click.option( + "--save", + type=click.Path(dir_okay=False), + default=None, + help="Write the result here before printing it.", +) +@pass_context +def retrieve(ctx: Context, whisper_hash: str, save: str | None) -> None: + """Fetch a finished extraction. + + A result can be read exactly once, so --save writes it to disk before it is + printed: a broken pipe or a full terminal buffer after the read cannot be + recovered by asking again. + """ + client = llmwhisperer(ctx.config) + if save: + preflight(save) + with translated(endpoint="whisper-retrieve"): + payload = client.whisper_retrieve(whisper_hash) + result = _extraction(payload) + if save: + persist(save, result) + finish(ctx, result, raw_fields=RAW_TEXT) + + +@whisper_group.command("detail") +@click.argument("whisper_hash") +@pass_context +def detail(ctx: Context, whisper_hash: str) -> None: + """Report processing detail for one extraction.""" + client = llmwhisperer(ctx.config) + with translated(endpoint="whisper-detail"): + finish(ctx, client.whisper_detail(whisper_hash)) + + +@whisper_group.command("highlights") +@click.argument("whisper_hash") +@spec_options( + PRODUCT, + "highlights", + client_method=LLMWhispererClientV2.get_highlight_data, + exclude=("whisper_hash",), +) +@click.option( + "--target-width", + type=int, + default=None, + help="Width of the page as displayed. With --target-height, adds a bounding box per line.", +) +@click.option( + "--target-height", + type=int, + default=None, + help="Height of the page as displayed.", +) +@pass_context +def highlights( + ctx: Context, + whisper_hash: str, + target_width: int | None, + target_height: int | None, + **params: Any, +) -> None: + """Fetch line metadata, optionally scaled to a page you are rendering. + + The scaling is arithmetic on the metadata, not a second request, so it is + folded in here rather than being a command of its own. + """ + sent = requested(params) + if not sent.get("lines") and not sent.get("extract_all_lines"): + raise CLIError( + "Nothing to fetch: pass --lines, or --extract-all-lines for all of them.", + ExitCode.USAGE, + ) + # The client takes `lines` positionally whether or not the request needs it. + sent.setdefault("lines", "") + + client = llmwhisperer(ctx.config) + try: + with translated(endpoint="highlights"): + data = client.get_highlight_data(whisper_hash, **sent) + except CLIError as exc: + if exc.exit_code is ExitCode.VALIDATION: + # Line metadata is recorded during extraction or not at all, so the + # fix belongs to a call that has already been made and paid for. + exc.hint = ( + "Line metadata exists only for an extraction run with " + "--add-line-nos. It cannot be added to this call: re-run " + "`whisper extract --add-line-nos` for the document." + ) + raise + + if target_width and target_height: + data = { + "lines": data, + "rects": _bounding_boxes(client, data, target_width, target_height), + } + finish(ctx, data) + + +def _line_metadata(value: Any) -> list[int] | None: + """The `[page, base_y, height, page_height]` list the geometry needs. + + The service returns it as a named object carrying the list under `raw`, and + the client's geometry takes the bare list, so both shapes are read. + """ + if isinstance(value, dict): + value = value.get("raw") + if ( + isinstance(value, list) + and len(value) >= 4 + and all(isinstance(item, (int, float)) for item in value) + # The page height is a divisor in the scaling, and the service reports a + # line it has no geometry for as all zeros. + and value[3] + ): + return value + return None + + +def _bounding_boxes( + client: LLMWhispererClientV2, + data: Any, + target_width: int, + target_height: int, +) -> dict[str, list[int]]: + """(page, x1, y1, x2, y2) per line, for the lines that carry metadata.""" + if not isinstance(data, dict): + return {} + lines = {line: _line_metadata(value) for line, value in data.items()} + return { + str(line): list(client.get_highlight_rect(metadata, target_width, target_height)) + for line, metadata in lines.items() + if metadata is not None + } + + +@whisper_group.command("usage") +@pass_context +def usage(ctx: Context) -> None: + """Report this key's usage and remaining quota.""" + client = llmwhisperer(ctx.config) + with translated(endpoint="get-usage-info"): + finish(ctx, client.get_usage_info()) + + +@whisper_group.group("webhook") +def webhook_group() -> None: + """Manage the webhooks an extraction can deliver its result to.""" + + +@webhook_group.command("create") +@click.argument("name") +@click.option("--url", required=True, help="Where the result is delivered.") +@click.option("--auth-token", required=True, help="Token sent with the delivery.") +@pass_context +def webhook_create(ctx: Context, name: str, url: str, auth_token: str) -> None: + """Register a webhook.""" + remember_secret(auth_token) + client = llmwhisperer(ctx.config) + with translated(endpoint="whisper-manage-callback"): + finish(ctx, client.register_webhook(url, auth_token, name)) + + +@webhook_group.command("update") +@click.argument("name") +@click.option("--url", required=True, help="Where the result is delivered.") +@click.option("--auth-token", required=True, help="Token sent with the delivery.") +@pass_context +def webhook_update(ctx: Context, name: str, url: str, auth_token: str) -> None: + """Replace a webhook's URL and token.""" + remember_secret(auth_token) + client = llmwhisperer(ctx.config) + with translated(endpoint="whisper-manage-callback"): + finish(ctx, client.update_webhook_details(name, url, auth_token)) + + +@webhook_group.command("get") +@click.argument("name") +@pass_context +def webhook_get(ctx: Context, name: str) -> None: + """Show one webhook's configuration. + + The token is reported as redacted, including for a webhook registered + elsewhere: it authenticates deliveries wherever it was set, and this output + is as likely to land in a log as on a screen. + """ + client = llmwhisperer(ctx.config) + with translated(endpoint="whisper-manage-callback"): + details = client.get_webhook_details(name) + if isinstance(details, dict): + remember_secret(details.get("auth_token")) + finish(ctx, details) + + +@webhook_group.command("delete") +@click.argument("name") +@pass_context +def webhook_delete(ctx: Context, name: str) -> None: + """Remove a webhook.""" + client = llmwhisperer(ctx.config) + with translated(endpoint="whisper-manage-callback"): + finish(ctx, client.delete_webhook(name)) + + +__all__ = ["extract", "highlights", "retrieve", "status", "usage", "webhook_group"] diff --git a/src/unstract_cli/config.py b/src/unstract_cli/config.py new file mode 100644 index 0000000..c31ade6 --- /dev/null +++ b/src/unstract_cli/config.py @@ -0,0 +1,614 @@ +"""Profile-based configuration. + +Two products with different hosts, different keys, and `org_id` as a URL *path +segment* rather than a flag. Named profiles (kubectl/aws style) hold per-product +host, key and org, plus deployment aliases so a deployment can be named instead +of spelled out. + +The resolution chain -- **flag > env > profile > built-in default** -- is +implemented once here and used by every parameter. It is never re-implemented +per command. + +The CLI is fully usable with **no config file at all**, driven entirely by +environment variables; that is the expected mode in CI and agent sandboxes. +""" + +from __future__ import annotations + +import contextlib +import os +import stat +import tempfile +import tomllib +from copy import deepcopy +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import tomli_w + +from unstract_cli.core.errors import remember_secret + +LLMWHISPERER = "llmwhisperer" +DOCSTUDIO = "docstudio" +PRODUCTS: tuple[str, ...] = (LLMWHISPERER, DOCSTUDIO) + +#: Built-in defaults, lowest precedence. +DEFAULT_BASE_URLS: dict[str, str] = { + LLMWHISPERER: "https://llmwhisperer-api.us-central.unstract.com/api/v2", + DOCSTUDIO: "https://us-central.unstract.com", +} + +#: Environment variables per (product, setting), checked before the config file +#: and in the order given. The trailing names are the ones the published clients +#: themselves read: an environment already set up for a client must not leave +#: the CLI silently on its built-in default, which is production. +ENV_VARS: dict[tuple[str, str], tuple[str, ...]] = { + (LLMWHISPERER, "api_key"): ("LLMWHISPERER_API_KEY",), + (LLMWHISPERER, "base_url"): ("LLMWHISPERER_BASE_URL", "LLMWHISPERER_BASE_URL_V2"), + (DOCSTUDIO, "api_key"): ("UNSTRACT_DEPLOYMENT_KEY", "UNSTRACT_API_DEPLOYMENT_KEY"), + (DOCSTUDIO, "base_url"): ("UNSTRACT_BASE_URL",), + (DOCSTUDIO, "org_id"): ("UNSTRACT_ORG_ID",), +} + + +#: Where the two credentials are minted. Quoted wherever the CLI reports one as +#: missing: knowing a key is unset is no help without knowing where one is made. +KEY_SOURCES = ( + "Get an LLMWhisperer key from the LLMWhisperer console; a deployment key is " + "shown on the API deployment's own page in the Unstract UI, and a key " + "covering every deployment in the organisation is minted under " + "Settings -> API Key Manager." +) + + +def settings_for(product: str) -> tuple[str, ...]: + """The settings a product actually has. + + Products differ: `org_id` is a URL path segment for one and meaningless for + the other, and reporting a setting a user has no way to supply reads as a + misconfiguration they cannot fix. + """ + return tuple(sorted(key for prod, key in ENV_VARS if prod == product)) + + +#: Filename a project can commit to point the CLI at its own settings. +PROJECT_CONFIG_NAME = ".unstract.toml" + +#: Where the config lives when nothing else selects one. +HOME_CONFIG = Path("~/.unstract/config.toml") + + +class ConfigError(Exception): + """Configuration could not be loaded or resolved.""" + + +#: Set by the root `--config` flag. Highest precedence, matching the +#: flag > env > file ordering used for every other setting. +_config_override: Path | None = None + + +def set_config_path(path: str | Path | None) -> None: + """Point this process at a specific config file (the `--config` flag).""" + global _config_override + _config_override = Path(path).expanduser() if path else None + + +def find_project_config(start: Path | None = None) -> Path | None: + """Search upward from the working directory for ``.unstract.toml``. + + Mirrors how git and ruff resolve project settings: running the CLI inside a + project picks up that project's config with no flag. The search stops at the + filesystem root, and at ``$HOME`` so a stray file in a parent directory + cannot silently capture every invocation. + + A symlinked candidate is skipped rather than followed: the file it points at + is chosen by whoever wrote the link, and this path is written to as well as + read from -- `config set` and `config init --force` would rewrite the target. + """ + current = (start or Path.cwd()).resolve() + home = Path.home().resolve() + for directory in (current, *current.parents): + candidate = directory / PROJECT_CONFIG_NAME + if candidate.is_file() and not candidate.is_symlink(): + return candidate + if directory == home: + break + return None + + +def config_path() -> Path: + """Location of the config file. + + Resolution: ``--config``, then ``$UNSTRACT_CONFIG``, then a project-local + ``.unstract.toml`` found by upward search, then ``~/.unstract/config.toml``. + + Several config files coexisting is expected, not exceptional: a per-project + file checked into a repo, a throwaway one in CI, and a personal default, each + selected per invocation. + """ + return _resolve_config_path()[0] + + +def _resolve_config_path() -> tuple[Path, bool]: + """The config path, and whether it was *discovered* rather than named. + + The boolean is the trust signal: a path the user named (``--config`` or + ``$UNSTRACT_CONFIG``) is trusted, one found by walking up from the working + directory is not. See ``UNTRUSTED_PROJECT_KEYS``. + """ + if _config_override is not None: + return _config_override, False + if override := os.environ.get("UNSTRACT_CONFIG"): + return Path(override).expanduser(), False + if local := find_project_config(): + return local, True + return HOME_CONFIG.expanduser(), False + + +def _deref(value: Any) -> Any: + """Resolve ``env:VAR_NAME`` indirection so config files hold no secrets. + + An unset variable resolves to ``None`` rather than the literal string, so a + missing credential surfaces as "not configured" instead of being sent as the + nonsense value ``"env:FOO"``. + + An empty string resolves the same way: the placeholders a generated config + carries must not satisfy `require`. + """ + if isinstance(value, str): + if value.startswith("env:"): + return os.environ.get(value[4:].strip()) or None + return value or None + return value + + +#: Settings a *discovered* project-local file may not supply: a checkout the +#: user did not write must not choose the host their key is sent to. Everything +#: else -- org_id, profile selection, deployment aliases -- is still honoured. +UNTRUSTED_PROJECT_KEYS = frozenset({"api_key", "base_url"}) + + +@dataclass +class ConfigFile: + """Parsed contents of the config file.""" + + default_profile: str | None = None + profiles: dict[str, dict[str, Any]] = field(default_factory=dict) + path: Path | None = None + exists: bool = False + #: Non-fatal diagnostics (e.g. loose file permissions), surfaced on stderr. + warnings: tuple[str, ...] = () + #: True when `path` was found by walking up from the working directory rather + #: than named. Such a file is not trusted with credentials or hosts. + is_project_local: bool = False + #: Keys withheld from an untrusted file, as ``{(profile, *blocks, key): value}``. + #: Excluded from resolution, but kept so a write-back does not drop them. + withheld: dict[tuple[str, ...], Any] = field(default_factory=dict) + + +def _strip_untrusted(profiles: dict[str, Any]) -> dict[tuple[str, ...], Any]: + """Remove the untrusted keys from a profile tree, in place, reporting what went.""" + withheld: dict[tuple[str, ...], Any] = {} + + def walk(node: Any, trail: tuple[str, ...]) -> None: + if not isinstance(node, dict): + return + for key in list(node): + if key in UNTRUSTED_PROJECT_KEYS: + withheld[(*trail, key)] = node.pop(key) + else: + walk(node[key], (*trail, key)) + + walk(profiles, ()) + return withheld + + +def _is_discovered(path: Path) -> bool: + """Whether this path is the file an upward search would have found. + + Trust follows the file, not the call: naming the project-local file that + discovery would have picked anyway does not make its contents any more the + user's own. ``--config`` and ``$UNSTRACT_CONFIG`` are a deliberate choice and + are resolved before this, so they stay trusted. + """ + candidate = find_project_config() + return candidate is not None and candidate.resolve() == path.resolve() + + +def load_config(path: Path | None = None) -> ConfigFile: + """Load the config file. A missing file is normal, not an error.""" + if path is not None: + target, project_local = path, _is_discovered(path) + else: + target, project_local = _resolve_config_path() + if not target.exists(): + return ConfigFile(path=target, exists=False, is_project_local=project_local) + + try: + with target.open("rb") as fh: + raw = tomllib.load(fh) + except (OSError, tomllib.TOMLDecodeError) as exc: + raise ConfigError(f"Could not read config at {target}: {exc}") from exc + + warnings: list[str] = [] + try: + mode = target.stat().st_mode + if mode & (stat.S_IRWXG | stat.S_IRWXO): + warnings.append( + f"Config file {target} is readable by other users " + f"(mode {stat.filemode(mode)}); consider `chmod 600`." + ) + except OSError: # pragma: no cover - stat failure is not worth failing on + pass + + profiles = raw.get("profiles", {}) + if not isinstance(profiles, dict): + raise ConfigError(f"`profiles` in {target} must be a table.") + + # Said out loud rather than dropped in silence; the rest of the file still + # applies. + withheld: dict[tuple[str, ...], Any] = {} + if project_local: + withheld = _strip_untrusted(profiles) + if withheld: + names = ", ".join(sorted(".".join(trail) for trail in withheld)) + warnings.append( + f"Ignoring {names} from project config {target}: a discovered " + f"{PROJECT_CONFIG_NAME} may not supply credentials or base URLs. " + "Pass --config explicitly, or set the environment variable instead." + ) + + return ConfigFile( + default_profile=raw.get("default_profile"), + profiles=profiles, + path=target, + exists=True, + warnings=tuple(warnings), + is_project_local=project_local, + withheld=withheld, + ) + + +def _restored_profiles(cfg: ConfigFile, target: Path) -> dict[str, Any]: + """The profiles to write, with anything withheld put back. + + Withholding a key from resolution is the security property; deleting it from + the user's file is not, and `config set` loads, mutates and saves the whole + document. Restored **only** when writing back to the file they came from -- + into any other path this would copy untrusted values somewhere they are + trusted. + """ + if not cfg.withheld or cfg.path is None or target.resolve() != cfg.path.resolve(): + return cfg.profiles + + profiles = deepcopy(cfg.profiles) + for (*parents, leaf), value in cfg.withheld.items(): + node: dict[str, Any] = profiles + for segment in parents: + child = node.get(segment) + if not isinstance(child, dict): + child = node[segment] = {} + node = child + node.setdefault(leaf, value) + return profiles + + +def save_config(cfg: ConfigFile, path: Path | None = None) -> Path: + """Write the config file with owner-only permissions.""" + target = path or cfg.path or config_path() + target.parent.mkdir(parents=True, exist_ok=True) + + doc: dict[str, Any] = {} + if cfg.default_profile: + doc["default_profile"] = cfg.default_profile + doc["profiles"] = _restored_profiles(cfg, target) + + # The path is not always one the user chose, and replacing a symlink would + # silently turn a deliberate one into a regular file. + if target.is_symlink(): + raise ConfigError( + f"Refusing to write config through the symlink at {target}: it would " + f"overwrite {os.readlink(target)} instead. Pass --config with the path " + "of the real file." + ) + + # Written through a temporary file and renamed into place. Truncating the + # real one first would destroy a working config if anything below it failed, + # and `mkstemp` both names the temporary unpredictably -- a guessable + # sibling in a shared directory is a symlink waiting to be planted -- and + # creates it 0600, which is the mode the rename then gives the config, with + # no window in which the new credential is readable more widely. + try: + handle_fd, name = tempfile.mkstemp(dir=target.parent, suffix=".tmp") + except OSError as exc: + # Renaming into place is what makes the write atomic, and that needs the + # directory, not just the file. Writing the file in place instead would + # put back the truncate this replaced. + raise ConfigError( + f"Cannot write {target}: its directory {target.parent} is not " + f"writable, and the config is replaced rather than overwritten so a " + f"failed write cannot destroy it ({exc.strerror})." + ) from exc + tmp = Path(name) + try: + with os.fdopen(handle_fd, "wb") as fh: + tomli_w.dump(doc, fh) + # The rename only replaces one whole config with another if the new + # bytes are on the disk before it happens. Without this a crash can + # leave the rename standing over content that never landed. + fh.flush() + os.fsync(fh.fileno()) + os.replace(tmp, target) + # And the rename is itself a directory change that has to be persisted; + # syncing the file does not cover the entry that now points at it. + with contextlib.suppress(OSError): # not every platform syncs a directory + dir_fd = os.open(target.parent, os.O_RDONLY) + try: + os.fsync(dir_fd) + finally: + os.close(dir_fd) + except BaseException: + tmp.unlink(missing_ok=True) + raise + return target + + +@dataclass +class ResolvedConfig: + """Effective settings for one invocation. + + ``overrides`` holds command-line flags, which outrank everything else. + """ + + file: ConfigFile + profile_name: str | None = None + overrides: dict[str, Any] = field(default_factory=dict) + + @property + def active_profile(self) -> str | None: + """Profile selected by flag, ``UNSTRACT_PROFILE``, or the file default.""" + return ( + self.profile_name + or os.environ.get("UNSTRACT_PROFILE") + or self.file.default_profile + ) + + def _profile(self) -> dict[str, Any]: + name = self.active_profile + if not name: + return {} + profile = self.file.profiles.get(name) + if profile is None: + if self.file.exists and self.file.profiles: + known = ", ".join(sorted(self.file.profiles)) or "none" + raise ConfigError( + f"Profile {name!r} not found in {self.file.path}. " + f"Known profiles: {known}" + ) + return {} + return profile if isinstance(profile, dict) else {} + + def _product_block(self, product: str) -> dict[str, Any]: + # One accepted shape only, settings nested under the product name: a + # config that looks applied but is not fails later with no obvious cause. + block = self._profile().get(product) + return block if isinstance(block, dict) else {} + + def get(self, product: str, key: str, default: Any = None) -> Any: + """Resolve one setting: **flag > env > profile > built-in default**.""" + value = self._resolve(product, key, default) + if key == "api_key": + remember_secret(value) + return value + + def _resolve(self, product: str, key: str, default: Any = None) -> Any: + if (value := self.overrides.get(f"{product}.{key}")) is not None: + return value + if (value := self.overrides.get(key)) is not None: + return value + + for env_var in ENV_VARS.get((product, key), ()): + if value := os.environ.get(env_var): + return value + + if (value := _deref(self._product_block(product).get(key))) is not None: + return value + + if default is not None: + return default + if key == "base_url": + return DEFAULT_BASE_URLS.get(product) + return None + + def require(self, product: str, key: str) -> Any: + """Resolve a setting, or raise a message naming exactly how to supply it.""" + if (value := self.get(product, key)) is not None: + return value + + hints: list[str] = [] + if env_vars := ENV_VARS.get((product, key)): + hints.append(f"set ${env_vars[0]}") + hints.append(f"or add `{key}` to the [profiles..{product}] block") + # `--api-key` exists but is not suggested: a secret on the command line + # lands in shell history and in the process list. + if key != "api_key": + hints.append(f"or pass --{key.replace('_', '-')}") + raise ConfigError( + f"Missing required setting {product}.{key}. To fix: {'; '.join(hints)}." + ) + + def deployment(self, alias: str) -> dict[str, Any]: + """Resolve a deployment alias to its api_name, org and key. + + ``org_id`` and ``api_key`` are optional per alias and fall back to the + profile's Document Studio block, so the common case is one line per + deployment. + """ + aliases = self._profile().get("deployments") + entry = aliases.get(alias) if isinstance(aliases, dict) else None + if not isinstance(entry, dict): + known = ( + ", ".join(sorted(aliases)) + if isinstance(aliases, dict) and aliases + else "none" + ) + raise ConfigError( + f"Deployment alias {alias!r} not found in profile " + f"{self.active_profile!r}. Known aliases: {known}." + ) + if not entry.get("api_name"): + raise ConfigError(f"Deployment alias {alias!r} has no `api_name`.") + api_key = self._alias_setting(alias, entry, "api_key") + remember_secret(api_key) + return { + "api_name": entry["api_name"], + "org_id": self._alias_setting(alias, entry, "org_id"), + "api_key": api_key, + } + + def _alias_setting(self, alias: str, entry: dict[str, Any], key: str) -> Any: + """One alias setting, falling back to the profile only where the alias is silent. + + An ``env:`` reference that does not resolve is not silence. Falling back + there runs the deployment against the profile's organisation, with the + profile's key, and reports success. + """ + raw = entry.get(key) + if isinstance(raw, str) and raw.startswith("env:"): + if value := _deref(raw): + return value + raise ConfigError( + f"Deployment alias {alias!r} sets {key} to {raw!r}, and " + f"${raw[4:].strip()} is not set in this process's environment." + ) + return raw or self.get(DOCSTUDIO, key) + + def deployment_aliases(self) -> tuple[str, ...]: + """Names of the deployment aliases defined in the active profile.""" + aliases = self._profile().get("deployments") + return tuple(sorted(aliases)) if isinstance(aliases, dict) else () + + def resolution_source(self, product: str, key: str) -> dict[str, Any]: + """Report where a setting resolves from, without echoing a secret. + + `config doctor` uses this to answer the question that costs the most + time: "the CLI says the key is not configured, but I set it -- where is + it looking?" + """ + if ( + self.overrides.get(f"{product}.{key}") is not None + or self.overrides.get(key) is not None + ): + return {"resolved": True, "source": "flag/override"} + + for env_var in ENV_VARS.get((product, key), ()): + if os.environ.get(env_var): + return {"resolved": True, "source": f"env:{env_var}"} + + raw = self._product_block(product).get(key) + if isinstance(raw, str) and raw.startswith("env:"): + var = raw[4:].strip() + present = bool(os.environ.get(var)) + return { + "resolved": present, + "source": f"profile -> env:{var}", + "detail": None + if present + else f"${var} is not set in this process's environment", + } + if raw not in (None, ""): + return {"resolved": True, "source": "profile (literal)"} + + report: dict[str, Any] = ( + {"resolved": True, "source": "built-in default"} + if key == "base_url" and DEFAULT_BASE_URLS.get(product) + else {"resolved": False, "source": "unset"} + ) + if detail := self.withheld_detail(product, key): + report["detail"] = detail + return report + + def withheld_detail(self, *trail: str) -> str | None: + """Why a setting the config file plainly holds did not arrive, if that is why. + + Reporting only where a value came *from* would leave the user staring at + a setting they can see in the file. Takes a trail rather than a + product/key pair so a deployment alias's own key -- nested a level deeper + -- is answerable too. + """ + if (self.active_profile, *trail) not in self.file.withheld: + return None + return ( + f"{self.file.path} sets {trail[-1]}, and a discovered " + f"{PROJECT_CONFIG_NAME} is not trusted with it." + ) + + +def starter_profiles() -> dict[str, dict[str, Any]]: + """Profile stubs written by `config init`. + + Every credential uses ``env:`` indirection: the generated file is a map of + where secrets live, never a copy of them. + + One key on the product block, and aliases that carry only ``api_name``: a + key can cover every deployment in the organisation, so a key per alias is + the exception -- for an organisation whose deployments hold separate keys -- + rather than the shape to start from. + """ + return { + "cloud-us": { + LLMWHISPERER: { + "base_url": DEFAULT_BASE_URLS[LLMWHISPERER], + "api_key": "env:LLMWHISPERER_API_KEY", + }, + DOCSTUDIO: { + "base_url": DEFAULT_BASE_URLS[DOCSTUDIO], + "org_id": "", + "api_key": "env:UNSTRACT_DEPLOYMENT_KEY", + }, + "deployments": {"example": {"api_name": "your-api-deployment-name"}}, + }, + "cloud-eu": { + LLMWHISPERER: { + "base_url": "https://llmwhisperer-api.eu-west.unstract.com/api/v2", + "api_key": "env:LLMWHISPERER_API_KEY", + }, + }, + # A shape to copy for a self-hosted install, not a profile to select: + # its host is a placeholder and only the active profile is resolved. + "onprem-example": { + LLMWHISPERER: { + "base_url": "https://llmwhisperer.unstract.internal.example/api/v2", + "api_key": "env:LLMWHISPERER_API_KEY", + }, + DOCSTUDIO: { + "base_url": "https://unstract.internal.example", + "org_id": "", + "api_key": "env:UNSTRACT_DEPLOYMENT_KEY", + }, + }, + } + + +__all__ = [ + "DEFAULT_BASE_URLS", + "DOCSTUDIO", + "ENV_VARS", + "HOME_CONFIG", + "KEY_SOURCES", + "LLMWHISPERER", + "PRODUCTS", + "PROJECT_CONFIG_NAME", + "UNTRUSTED_PROJECT_KEYS", + "ConfigError", + "ConfigFile", + "ResolvedConfig", + "config_path", + "find_project_config", + "load_config", + "save_config", + "set_config_path", + "settings_for", + "starter_profiles", +] diff --git a/src/unstract_cli/core/__init__.py b/src/unstract_cli/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/unstract_cli/core/clients.py b/src/unstract_cli/core/clients.py new file mode 100644 index 0000000..66923e6 --- /dev/null +++ b/src/unstract_cli/core/clients.py @@ -0,0 +1,271 @@ +"""Building the product clients, and turning their failures into CLI errors. + +The entry point deliberately does not catch bare ``Exception``: an unexpected +crash should look like a crash. Everything a client raises on purpose is +expected, so it is translated here into a ``CLIError`` carrying an exit code, a +hint and the response detail. + +The two clients report failure differently -- LLMWhisperer raises with a status +code attached, the deployment client returns a dict containing one -- so both +shapes converge here rather than in each command. +""" + +from __future__ import annotations + +import socket +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from typing import Any + +from requests.exceptions import ConnectionError, Timeout +from unstract.api_deployments.client import ( + APIDeploymentsClient, + APIDeploymentsClientException, +) +from unstract.llmwhisperer.client_v2 import ( + LLMWhispererClientException, + LLMWhispererClientV2, +) + +from unstract_cli.config import DOCSTUDIO, LLMWHISPERER, ResolvedConfig +from unstract_cli.core.errors import CLIError, ExitCode, error_from_status +from unstract_cli.core.params import find_operation + + +def llmwhisperer(config: ResolvedConfig) -> LLMWhispererClientV2: + """Build the LLMWhisperer client from the resolved configuration.""" + return LLMWhispererClientV2( + base_url=config.require(LLMWHISPERER, "base_url"), + api_key=config.require(LLMWHISPERER, "api_key"), + logging_level="ERROR", + ) + + +def deployment_url(base_url: str, org_id: str, api_name: str) -> str: + """The deployment's full URL, laid out as the spec declares the route. + + The client takes the whole URL and reads the organisation and API name back + out of its last two segments, so the route is built from the spec rather + than from a format string that can disagree with it. + """ + path = find_operation(DOCSTUDIO, "execute")["path"] + path = path.format(org_name=org_id, api_name=api_name) + return base_url.rstrip("/") + path + + +def deployment( + config: ResolvedConfig, target: str, transport_timeout: float | None = None +) -> APIDeploymentsClient: + """Build a deployment client for an alias, or for a bare API name. + + An alias carries its own organisation and key; a bare name falls back to the + profile's, so an unconfigured caller can still name a deployment directly. + """ + if target in config.deployment_aliases(): + entry = config.deployment(target) + api_name, org_id, api_key = ( + entry["api_name"], + entry["org_id"], + entry["api_key"], + ) + else: + api_name = target + org_id = config.get(DOCSTUDIO, "org_id") + api_key = config.get(DOCSTUDIO, "api_key") + + missing = [ + name for name, value in (("org_id", org_id), ("api_key", api_key)) if not value + ] + if missing: + raise CLIError( + f"Deployment {target!r} is missing {' and '.join(missing)}.", + ExitCode.USAGE, + hint=_alias_hint(config, target) + or ( + "Define the deployment as an alias in the active profile, or set " + "$UNSTRACT_ORG_ID and $UNSTRACT_DEPLOYMENT_KEY." + ), + ) + + return APIDeploymentsClient( + api_url=deployment_url(config.require(DOCSTUDIO, "base_url"), org_id, api_name), + api_key=api_key, + logging_level="ERROR", + transport_timeout=transport_timeout, + ) + + +def _alias_hint(config: ResolvedConfig, target: str) -> str | None: + """What to say when a target is not one of the aliases that are configured. + + A bare API name is a supported way to name a deployment, so a target that is + not an alias cannot be rejected outright. It can still be a misspelt one, + and a caller who has defined aliases is likelier to have meant one of them + than to have typed a raw name, so the ones that exist are worth naming. + """ + if not (aliases := config.deployment_aliases()) or target in aliases: + return None + return ( + f"{target!r} is not one of the deployment aliases in the active profile " + f"({', '.join(aliases)}), so it was sent as an API name." + ) + + +@contextmanager +def naming_aliases(config: ResolvedConfig, target: str) -> Iterator[None]: + """Say which aliases exist when a bare API name is not found. + + Sending a misspelt alias as an API name is indistinguishable from sending a + real one until the service answers, so the correction belongs on the answer. + """ + try: + yield + except CLIError as exc: + if exc.exit_code is ExitCode.NOT_FOUND and (hint := _alias_hint(config, target)): + exc.hint = f"{exc.hint} {hint}" if exc.hint else hint + raise + + +def _message_and_details(value: Any) -> tuple[str, Any]: + """Split a client's error value into a one-line message and the raw detail. + + LLMWhisperer raises with either a string or the decoded error body, and the + body's own wording is better than anything invented here. + """ + if isinstance(value, dict): + for key in ("message", "error", "detail", "reason"): + if text := value.get(key): + return str(text), value + return str(value), value + return str(value), None + + +def _causes(exc: BaseException) -> Iterator[BaseException]: + """One failure and everything it was raised from, outermost first.""" + seen: BaseException | None = exc + while seen is not None: + yield seen + seen = seen.__cause__ or seen.__context__ + + +def _unresolved_host(exc: BaseException) -> str | None: + """The host a connection failed to resolve, or ``None`` if that is not why. + + A name that does not resolve is the one connection failure retrying cannot + fix. Read from the chain rather than from the outermost exception: the + clients re-raise transport failures as their ``requests`` equivalents + carrying only a message, so nothing structural survives at the top -- but + the original is still attached underneath, and `socket.gaierror` is the + resolver's own answer whichever transport asked it. + """ + if not any(isinstance(cause, socket.gaierror) for cause in _causes(exc)): + return None + for cause in _causes(exc): + # httpx keeps the request on the error it raises; urllib3 keeps the + # connection. Either names the host without parsing a message. + url = getattr(getattr(cause, "request", None), "url", None) + if host := getattr(url, "host", "") or getattr( + getattr(cause, "conn", None), "host", "" + ): + return host + return "" + + +@contextmanager +def translated(endpoint: str | None = None) -> Iterator[None]: + """Turn a client failure into a CLIError with an exit code and a hint.""" + try: + yield + except LLMWhispererClientException as exc: + message, details = _message_and_details(exc.value) + status = exc.status_code or ( + details.get("status_code") if isinstance(details, dict) else None + ) + if status: + raise error_from_status( + int(status), message, details=details, endpoint=endpoint + ) from exc + raise CLIError(message, details=details, endpoint=endpoint) from exc + except APIDeploymentsClientException as exc: + raise CLIError(str(exc), ExitCode.USAGE, endpoint=endpoint) from exc + except Timeout as exc: + raise CLIError( + str(exc), + ExitCode.TIMEOUT, + endpoint=endpoint, + retryable=True, + hint="The request timed out in transit; the job may still be running.", + ) from exc + except ConnectionError as exc: + if (host := _unresolved_host(exc)) is not None: + raise CLIError( + f"Could not resolve the host {host or endpoint or 'in the base URL'}.", + ExitCode.SERVER_ERROR, + endpoint=endpoint, + hint="Check the base URL for a typo. Retrying will not help.", + ) from exc + raise CLIError( + str(exc), + ExitCode.SERVER_ERROR, + endpoint=endpoint, + retryable=True, + hint="Could not reach the service. Check the base URL and connectivity.", + ) from exc + + +def translating( + call: Callable[..., Any], endpoint: str | None = None +) -> Callable[..., Any]: + """Wrap one call so its failures are CLIErrors where they happen. + + A ``with translated(...)`` around a loop converts nothing until the loop is + left, by which point what the loop knew -- the job handle above all -- is out + of scope. + """ + + def wrapped(*args: Any, **kwargs: Any) -> Any: + with translated(endpoint=endpoint): + return call(*args, **kwargs) + + return wrapped + + +def raise_for_result(result: dict[str, Any], endpoint: str | None = None) -> None: + """Fail on a deployment response that reports an error status. + + The deployment client returns its status code instead of raising, so a + failure would otherwise be reported as a successful run whose payload + happens to contain an error. + """ + status = int(result.get("status_code") or 0) + reported = result.get("error") + if status and not 200 <= status < 300: + raise error_from_status( + status, + str(reported or f"Request failed with status {status}"), + details=result, + endpoint=endpoint, + ) + if reported: + # HTTP success carrying a failure in the body. Not retryable: a re-run + # starts a second billed execution rather than retrying the first. + raise CLIError( + str(reported), + ExitCode.VALIDATION, + http_status=status or None, + details=result, + endpoint=endpoint, + hint="The request was accepted and the work was not done; `details` " + "carries the service's own report.", + ) + + +__all__ = [ + "deployment", + "deployment_url", + "llmwhisperer", + "naming_aliases", + "raise_for_result", + "translated", + "translating", +] diff --git a/src/unstract_cli/core/discover.py b/src/unstract_cli/core/discover.py new file mode 100644 index 0000000..ebec37e --- /dev/null +++ b/src/unstract_cli/core/discover.py @@ -0,0 +1,173 @@ +"""`--discover`: the CLI describing itself, in three tiers. + +An agent driving this CLI needs to know what exists before it can run anything, +and `--help` is prose scraped from a terminal. Discovery answers the same +question as JSON, at whichever depth the question needs: + +* ``groups`` -- what products are here at all +* ``summary`` -- what commands each group has +* ``full`` -- every flag with its type, default and allowed values, plus the + exit codes and the output contract, which is enough to construct a call and + read its answer without a second round trip + +Every tier is read back from Click itself. Describing commands from anywhere +else lets the description drift from what the parser accepts. +""" + +from __future__ import annotations + +from typing import Any + +import click + +from unstract_cli.core.errors import _ERROR_CODES, ExitCode +from unstract_cli.core.output import CONTRACT_VERSION + +TIERS = ("groups", "summary", "full") + +#: Click's marker for "no default was given". It stopped being `None` in 8.2 and +#: is not exported, so it is read off a bare option and tracks whichever version +#: is installed -- serialised, it would publish a string that reads as a value. +_NO_DEFAULT = click.Option(["--unset"]).default + +#: The same question for a paired on/off flag, which answers it differently: +#: given no default, some versions report `False` and others their own sentinel. +#: Read the same way, so neither is mistaken for a default the flag really has. +_NO_FLAG_DEFAULT = click.Option(["--unset/--no-unset"], default=None).default + + +def contract() -> dict[str, Any]: + """How to consume this CLI's output, published rather than assumed. + + Both halves of the compatibility bargain are written down here: what we + promise not to break, and what a consumer has to do for that promise to be + worth anything. + """ + return { + "version": CONTRACT_VERSION, + "envelope": ["ok", "data", "error", "meta"], + "rules": [ + "Pass `-o json`. The default format is for people and is free to " + "change; json is the parseable one and never varies with the " + "terminal, the config or the environment.", + "Ignore fields you do not recognise. New ones are added within a " + "major version.", + "Refuse a `meta.contract_version` whose value is greater than the " + "one you were written against: the shape has changed under you.", + "Branch on the exit code, not on the message text.", + "Read stdout for the envelope only. Diagnostics are on stderr.", + ], + } + + +def exit_codes() -> list[dict[str, Any]]: + """The exit-code table, which is part of the contract callers branch on.""" + return [ + { + "code": int(code), + "name": code.name.lower(), + "error_code": _ERROR_CODES.get(code, ""), + } + for code in ExitCode + ] + + +def _param(param: click.Parameter) -> dict[str, Any]: + """One flag or argument, in the terms a caller needs to supply it.""" + entry: dict[str, Any] = { + "name": param.name, + "kind": "argument" if isinstance(param, click.Argument) else "option", + "type": getattr(param.type, "name", "text"), + "required": bool(param.required), + } + if isinstance(param, click.Option): + entry["flags"] = list(param.opts) + list(param.secondary_opts) + entry["help"] = param.help or "" + entry["repeatable"] = bool(param.multiple) + if isinstance(param.type, click.Choice): + entry["choices"] = list(param.type.choices) + # What omitting the flag actually gets you, which is not what Click reports: + # the same declaration answers differently across the supported range, so + # reading `param.default` straight publishes a contract per version. + default = param.default + if param.secondary_opts and default is _NO_FLAG_DEFAULT: + # An on/off flag the CLI declares with no default means "not passed, so + # not sent". Publishing the `False` some versions report here would + # promise a value the CLI does not send. + default = None + elif default is _NO_DEFAULT: + default = False if getattr(param, "is_flag", False) else None + if default is not None and not isinstance(param, click.Argument): + entry["default"] = default + return entry + + +def _params(command: click.Command) -> list[dict[str, Any]]: + """The flags a caller can pass to one command, group or the root. + + A group carries the connection settings for everything beneath it, so + describing only the leaves describes a call nobody can make. + """ + return [_param(p) for p in command.params if p.name not in ("help", "discover")] + + +def _describe(command: click.Command, tier: str) -> dict[str, Any]: + entry: dict[str, Any] = {"help": (command.help or "").strip().split("\n")[0]} + if tier == "full": + entry["params"] = _params(command) + # What `--output raw` prints for this command, best answer first: the + # first of these the answer carries is the one printed, and an answer + # carrying none of them fails rather than printing something else. + if raw := getattr(command, "raw_fields", ()): + entry["raw_fields"] = list(raw) + if isinstance(command, click.Group): + entry["commands"] = { + name: _describe(sub, tier) for name, sub in sorted(command.commands.items()) + } + return entry + + +def discover(root: click.Group, tier: str) -> dict[str, Any]: + """Describe the CLI at one tier. + + ``groups`` stops at the top level rather than walking further, so the cheap + question stays cheap: an agent starts here and drills down only where it + needs to. + """ + if tier not in TIERS: + raise ValueError(f"Unknown discovery tier {tier!r}. One of: {', '.join(TIERS)}") + + if tier == "groups": + top = sorted(root.commands.items()) + + def summary(name: str, command: click.Command) -> dict[str, str]: + return {"name": name, "help": (command.help or "").strip().split("\n")[0]} + + return { + "tier": tier, + "groups": [ + summary(name, sub) for name, sub in top if isinstance(sub, click.Group) + ], + # Leaf commands are listed apart from the groups, so a consumer + # walking groups for their commands does not drop them. + "commands": [ + summary(name, sub) + for name, sub in top + if not isinstance(sub, click.Group) + ], + } + + payload: dict[str, Any] = { + "tier": tier, + "commands": { + name: _describe(sub, tier) for name, sub in sorted(root.commands.items()) + }, + } + if tier == "full": + payload["params"] = _params(root) + payload["exit_codes"] = exit_codes() + payload["contract"] = contract() + return payload + + +__all__ = ["TIERS", "contract", "discover", "exit_codes"] diff --git a/src/unstract_cli/core/errors.py b/src/unstract_cli/core/errors.py new file mode 100644 index 0000000..2edd158 --- /dev/null +++ b/src/unstract_cli/core/errors.py @@ -0,0 +1,288 @@ +"""Exit codes, structured errors, and secret redaction. + +Exit codes are a stable API: a caller branches on them without parsing prose. +Every failure also carries `hint` and `retryable` so the caller can self-correct +rather than retry blindly. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from enum import IntEnum +from typing import Any + + +class ExitCode(IntEnum): + SUCCESS = 0 + GENERIC = 1 + USAGE = 2 + AUTH = 3 + NOT_FOUND = 4 + VALIDATION = 5 + RATE_LIMITED = 6 + TIMEOUT = 7 + SERVER_ERROR = 8 + ALREADY_CONSUMED = 9 + SAVE_FAILED = 10 + #: 128 + SIGINT, the value a shell and every job runner already read as + #: "the user stopped it" rather than as a failure of the command. + INTERRUPTED = 130 + + +#: HTTP status -> exit code. An in-progress 422 never reaches here: the poll +#: engine branches on the response body first. +_STATUS_MAP: dict[int, ExitCode] = { + 400: ExitCode.VALIDATION, + 401: ExitCode.AUTH, + 403: ExitCode.AUTH, + 404: ExitCode.NOT_FOUND, + # Only the deployment status endpoint answers 406; the whisper equivalent + # is a 400 whose body says so, which is prose we do not translate on. + 406: ExitCode.ALREADY_CONSUMED, + 408: ExitCode.TIMEOUT, + 409: ExitCode.VALIDATION, + 422: ExitCode.VALIDATION, + 429: ExitCode.RATE_LIMITED, +} + +_ERROR_CODES: dict[ExitCode, str] = { + ExitCode.GENERIC: "error", + ExitCode.USAGE: "usage_error", + ExitCode.AUTH: "auth_error", + ExitCode.NOT_FOUND: "not_found", + ExitCode.VALIDATION: "validation_error", + ExitCode.RATE_LIMITED: "rate_limited", + ExitCode.TIMEOUT: "timeout", + ExitCode.SERVER_ERROR: "server_error", + ExitCode.ALREADY_CONSUMED: "already_consumed", + ExitCode.SAVE_FAILED: "save_failed", + ExitCode.INTERRUPTED: "interrupted", +} + + +def exit_code_for_status(status: int) -> ExitCode: + """Map an HTTP status onto its exit code.""" + if code := _STATUS_MAP.get(status): + return code + if 500 <= status < 600: + return ExitCode.SERVER_ERROR + # A 3xx that was not followed, or a status no spec declares, is still a + # failure: never fall through to SUCCESS. + return ExitCode.GENERIC + + +def is_retryable(status: int) -> bool: + """Retry only on rate limiting and server faults -- never on 4xx. + + Retrying a 4xx re-sends a request the server already rejected on its merits, + and for one-shot reads a blind retry can consume a result the first attempt + already delivered. + """ + return status == 429 or 500 <= status < 600 + + +# --------------------------------------------------------------------------- # +# Redaction +# --------------------------------------------------------------------------- # + +_SECRET_HEADERS = {"unstract-key", "authorization", "apikey"} +_SECRET_HEADER_PREFIXES = ("x-",) +_SECRET_KEY_HINTS = ("key", "token", "secret", "password", "credential", "auth") +REDACTED = "***REDACTED***" + +#: Credentials resolved during this run. Registered where they are resolved, so +#: no emitter has to remember to opt into scrubbing. +_KNOWN_SECRETS: set[str] = set() + + +def remember_secret(value: Any) -> None: + """Record a resolved credential so no stream can print it later.""" + if isinstance(value, str) and len(value) >= 8: + _KNOWN_SECRETS.add(value) + + +def known_secrets() -> list[str]: + """Every credential resolved so far, longest first. + + Longest first so a key that contains another as a prefix is replaced whole + rather than leaving its tail behind. + """ + return sorted(_KNOWN_SECRETS, key=len, reverse=True) + + +def redact_headers(headers: dict[str, Any]) -> dict[str, Any]: + """Redact credential-bearing headers.""" + out: dict[str, Any] = {} + for key, value in headers.items(): + low = key.lower() + secret = low in _SECRET_HEADERS or ( + low.startswith(_SECRET_HEADER_PREFIXES) + and any(hint in low for hint in _SECRET_KEY_HINTS) + ) + out[key] = REDACTED if secret else value + return out + + +def redact_value(value: Any) -> Any: + """Recursively redact secret-looking keys in a payload.""" + if isinstance(value, dict): + return { + k: ( + REDACTED + if any(hint in str(k).lower() for hint in _SECRET_KEY_HINTS) + and isinstance(v, str) + else redact_value(v) + ) + for k, v in value.items() + } + if isinstance(value, list): + return [redact_value(v) for v in value] + return value + + +def scrub(text: str, secrets: list[str]) -> str: + """Remove known secret literals from free text. + + Last line of defence: a credential that reaches a message body via an + upstream error string still must not be printed. Short values are skipped -- + redacting a 3-character "key" would mangle unrelated text. + """ + for secret in secrets: + if secret and len(secret) >= 8: + text = re.sub(re.escape(secret), REDACTED, text) + return text + + +# --------------------------------------------------------------------------- # +# CLIError +# --------------------------------------------------------------------------- # + + +@dataclass +class CLIError(Exception): + """A failure that maps onto an exit code and a structured error payload.""" + + message: str + exit_code: ExitCode = ExitCode.GENERIC + http_status: int | None = None + details: Any = None + endpoint: str | None = None + hint: str | None = None + retryable: bool = False + code: str | None = None + extra: dict[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + super().__init__(self.message) + if self.exit_code is ExitCode.SUCCESS: + raise ValueError("a CLIError cannot carry the success exit code") + + def to_dict(self) -> dict[str, Any]: + payload: dict[str, Any] = { + "code": self.code or _ERROR_CODES.get(self.exit_code, "error"), + "message": self.message, + "exit_code": int(self.exit_code), + "retryable": self.retryable, + } + if self.http_status is not None: + payload["http_status"] = self.http_status + if self.details is not None: + # Structural, not opt-in: the details come from a server body that + # can echo the request, headers and key included. + payload["details"] = redact_value(self.details) + if self.endpoint: + payload["endpoint"] = self.endpoint + if self.hint: + payload["hint"] = self.hint + payload.update(self.extra) + return payload + + +def error_from_status( + status: int, message: str, *, details: Any = None, endpoint: str | None = None +) -> CLIError: + """Build a CLIError from an HTTP status, with its exit code, hint and retryability.""" + return CLIError( + message, + exit_code_for_status(status), + http_status=status, + details=details, + endpoint=endpoint, + hint=hint_for(status), + retryable=is_retryable(status), + ) + + +def undeclared_status_error( + status: int, body: Any, endpoint: str | None = None +) -> CLIError: + """Report a status the spec does not declare, verbatim. + + A guessed message for an unknown status is worse than none: it sends the + reader after the wrong cause. The body is passed through untouched. + """ + return CLIError( + f"Undeclared status {status} with body {body!r}", + exit_code_for_status(status), + http_status=status, + details=body, + endpoint=endpoint, + retryable=is_retryable(status), + ) + + +def hint_for(status: int) -> str | None: + """A short, actionable next step for a common failure.""" + match status: + case 400: + return ( + "The service rejected the request. Check the ids and parameter " + "values passed; `details` carries the service's own response." + ) + case 401 | 403: + # Wrong, revoked and not-permitted all arrive as the same response, + # so the hint cannot settle on one of them. A key from another + # organisation is not among them: the resource is resolved within + # its organisation first, so that answers 404 instead. + return ( + "The key was rejected. Keys are per-product: `unstract config " + "doctor` reports which one resolved and from where. A key that " + "works elsewhere can still be rejected here if it does not cover " + "this deployment." + ) + case 404: + return ( + "Verify the resource id, and that the organisation matches the " + "resource's own. For deployments, confirm the API name." + ) + case 406: + return ( + "This execution result was already retrieved. A deployment serves " + "its result exactly once; re-running the status call cannot " + "recover it. Pass --save to `deployment run` to keep the next one." + ) + case 409: + return "The resource is in use, or conflicts with an existing one." + case 429: + return "Rate limited. Back off and retry." + if 500 <= status < 600: + return "Server-side failure. If it persists, check service status." + return None + + +__all__ = [ + "REDACTED", + "CLIError", + "ExitCode", + "known_secrets", + "remember_secret", + "error_from_status", + "exit_code_for_status", + "hint_for", + "is_retryable", + "redact_headers", + "redact_value", + "scrub", + "undeclared_status_error", +] diff --git a/src/unstract_cli/core/output.py b/src/unstract_cli/core/output.py new file mode 100644 index 0000000..edcc4d1 --- /dev/null +++ b/src/unstract_cli/core/output.py @@ -0,0 +1,361 @@ +"""Output rendering, and choosing which rendering to use. + +The contract a caller depends on: + +* ``-o json`` writes **one envelope to stdout and nothing else** -- ``{ok, data, + error, meta}`` -- on success and on failure alike, so a failed run still + yields a valid object rather than an empty stream. +* What ``-o json`` produces depends on nothing but the command and its + arguments: not on a terminal, not on configuration, not on who is calling. +* Human-facing notes, warnings and progress all go to stderr. +* Without ``-o`` the output is ``table``, which is for people to read and is + free to change. Anything parsing this CLI passes ``-o json`` explicitly. + +The one thing an unflagged run reads from its environment is which *default* to +use: a coding agent gets ``json``, because an agent that has to be told twice is +an agent that parses a table. Detection is never allowed to reach past the +default -- see ``resolve_format``. +""" + +from __future__ import annotations + +import json +import os +import shutil +import sys +import textwrap +from collections.abc import Mapping +from enum import StrEnum +from fnmatch import fnmatch +from typing import Any + +from unstract_cli.core.errors import CLIError, ExitCode, known_secrets, scrub + +#: Major version of the stdout envelope, published in every ``meta``. +CONTRACT_VERSION = 1 + +#: Environment markers the coding agents set for the tools they drive. Patterns, +#: so a family of variables can be named once. +AGENT_ENV = ("CLAUDECODE", "CURSOR_AGENT", "CODEX_*", "AI_AGENT") + + +class OutputFormat(StrEnum): + JSON = "json" + TABLE = "table" + RAW = "raw" + + +class AgentMode(StrEnum): + AUTO = "auto" + YES = "yes" + NO = "no" + + +def agent_detected(env: Mapping[str, str] | None = None) -> bool: + """Whether the environment looks like a coding agent's.""" + names = os.environ if env is None else env + return any( + names[name] and fnmatch(name, pattern) for name in names for pattern in AGENT_ENV + ) + + +def resolve_format( + explicit: str | None, + agent: str = AgentMode.AUTO, + env: Mapping[str, str] | None = None, +) -> OutputFormat: + """The format to render in. + + An explicit ``-o`` wins outright, so detection can only ever pick the + default: two runs of ``-o json`` in different environments render the same + bytes, which is the property a script is relying on. + """ + if explicit: + return OutputFormat(explicit) + if agent == AgentMode.YES or (agent == AgentMode.AUTO and agent_detected(env)): + return OutputFormat.JSON + return OutputFormat.TABLE + + +def envelope( + *, + data: Any = None, + error: dict[str, Any] | None = None, + meta: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Build the stdout envelope. ``ok`` is derived, never passed in.""" + return { + "ok": error is None, + "data": data, + "error": error, + "meta": {**(meta or {}), "contract_version": CONTRACT_VERSION}, + } + + +def _flatten(value: Any) -> str: + """Render a cell. Nested structures become compact JSON, not Python reprs.""" + if value is None: + return "" + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (dict, list)): + return json.dumps(value, default=str) + return str(value) + + +def _rows_and_columns( + data: Any, columns: tuple[str, ...] = () +) -> tuple[list[str], list[list[str]]]: + """Derive table columns and rows from arbitrary JSON. + + List of objects -> columns from the union of keys in first-seen order; + single object -> a two-column key/value listing; anything else -> one + ``value`` column. ``columns`` overrides the selection where the generic rule + reads poorly. + """ + if isinstance(data, dict): + # Unwrap a single list-valued envelope, e.g. {"results": [...]}. + for key in ("results", "message", "members", "data", "highlights"): + inner = data.get(key) + if isinstance(inner, list) and inner: + data = inner + break + + if isinstance(data, list): + if not data: + return [], [] + if all(isinstance(item, dict) for item in data): + if columns: + headers = list(columns) + else: + headers = [] + for item in data: + headers.extend(k for k in item if k not in headers) + return headers, [[_flatten(item.get(h)) for h in headers] for item in data] + return ["value"], [[_flatten(item)] for item in data] + + if isinstance(data, dict): + keys = list(columns) if columns else list(data) + return ["key", "value"], [[k, _flatten(data.get(k))] for k in keys] + + return ["value"], [[_flatten(data)]] + + +def _terminal_width(default: int = 100) -> int: + try: + return max(shutil.get_terminal_size((default, 24)).columns, 40) + except Exception: # pragma: no cover - detached terminal + return default + + +def render_table( + data: Any, columns: tuple[str, ...] = (), *, max_width: int | None = None +) -> str: + """Render as an aligned plain-text table. + + Plain text rather than box drawing: tables end up in logs and terminals of + varying width, and ASCII survives both. + + Long cells are **wrapped, never truncated**: a table is a view of the data, + not a lossy summary, and a silently dropped tail is the kind of thing you + only notice after acting on it. + """ + headers, rows = _rows_and_columns(data, columns) + if not headers: + return "(no results)" + + gutter = 2 + total_width = max_width or _terminal_width() + + natural = [len(h) for h in headers] + for row in rows: + for i, cell in enumerate(row): + if i < len(natural): + natural[i] = max( + natural[i], max((len(p) for p in cell.split("\n")), default=0) + ) + + # Shrink only the widest columns, and only as far as the terminal requires, + # so a narrow column is never squeezed on behalf of a wide neighbour. + widths = list(natural) + budget = total_width - gutter * (len(headers) - 1) + while sum(widths) > budget and max(widths) > 8: + widest = widths.index(max(widths)) + widths[widest] -= 1 + + def fmt(cells: list[str]) -> list[str]: + """Lay one logical row out over as many physical lines as it needs.""" + wrapped = [ + textwrap.wrap(cell, width=w, break_long_words=True, break_on_hyphens=False) + or [""] + for cell, w in zip(cells, widths, strict=False) + ] + height = max(len(parts) for parts in wrapped) + lines = [] + for line_no in range(height): + pieces = [ + (parts[line_no] if line_no < len(parts) else "").ljust(w) + for parts, w in zip(wrapped, widths, strict=False) + ] + lines.append((" " * gutter).join(pieces).rstrip()) + return lines + + out = fmt(headers) + out.append((" " * gutter).join("-" * w for w in widths).rstrip()) + for row in rows: + out.extend(fmt(row)) + return "\n".join(out) + + +def raw_value(env: dict[str, Any], fields: tuple[str, ...]) -> Any: + """The first declared field this answer actually carries. + + Commands declare several because one call has several shapes: a queued run + answers with a handle and no result, and a status read answers with a state + until there is a result to answer with. Each is a value the caller asked + for, so raw prints the first one present rather than the first one declared. + + ``meta`` is searched too, because a handle the CLI had to derive rather than + read off the response lands there and is still what the caller wants. + + Nothing present is a failure, not empty output. Raw is one value on stdout + and nothing else, so it cannot say "not this time" inside itself: printing + the whole payload would answer a question nobody asked, and printing the + field's own ``null`` is worse, because a caller polling for a result cannot + tell it apart from a finished job that produced nothing. + """ + payload = env["data"] if env["ok"] else env["error"] + if not fields or not isinstance(payload, dict): + return payload + for name in fields: + for source in (payload, env.get("meta") or {}): + if isinstance(source, dict) and source.get(name) is not None: + return source[name] + raise CLIError( + f"This answer carries none of {', '.join(fields)}, so there is nothing " + "to print as raw output.", + ExitCode.GENERIC, + hint="Read it with `-o json`, which prints whatever the answer does carry.", + ) + + +def render( + env: dict[str, Any], + fmt: OutputFormat = OutputFormat.JSON, + *, + columns: tuple[str, ...] = (), + raw_fields: tuple[str, ...] = (), +) -> str: + """Render an envelope. ``table`` and ``raw`` show ``data``, or the error.""" + if fmt is OutputFormat.JSON: + return json.dumps(env, indent=2, default=str) + + payload = env["data"] if env["ok"] else env["error"] + if fmt is OutputFormat.TABLE: + return render_table(payload, columns) + + payload = raw_value(env, raw_fields) + if isinstance(payload, bytes): + return payload.decode("utf-8", errors="replace") + if isinstance(payload, str): + return payload + return json.dumps(payload, indent=2, default=str) + + +def emit( + env: dict[str, Any], + fmt: OutputFormat = OutputFormat.JSON, + *, + columns: tuple[str, ...] = (), + raw_fields: tuple[str, ...] = (), + secrets: list[str] | None = None, +) -> None: + """Write one envelope to stdout -- and nothing else to stdout. + + Every credential resolved during the run is scrubbed whether or not the + caller passed one: an emitter that has to remember is an emitter that + eventually forgets. + """ + emit_text(render(env, fmt, columns=columns, raw_fields=raw_fields), secrets=secrets) + + +def emit_text(text: str, *, secrets: list[str] | None = None) -> None: + """Write already-rendered text to stdout, scrubbed the way an envelope is. + + A command that renders its own table is still writing to the stream no + credential may reach, and scrubbing it by hand is the arrangement that + eventually forgets. + """ + if to_hide := [*(secrets or []), *known_secrets()]: + text = scrub(text, to_hide) + print(text) + + +def emit_result( + data: Any, + fmt: OutputFormat = OutputFormat.JSON, + *, + meta: dict[str, Any] | None = None, + columns: tuple[str, ...] = (), + raw_fields: tuple[str, ...] = (), + secrets: list[str] | None = None, +) -> None: + """Write a successful result.""" + emit( + envelope(data=data, meta=meta), + fmt, + columns=columns, + raw_fields=raw_fields, + secrets=secrets, + ) + + +def emit_error( + error: CLIError, + fmt: OutputFormat = OutputFormat.JSON, + *, + meta: dict[str, Any] | None = None, + secrets: list[str] | None = None, +) -> ExitCode: + """Write a failure envelope to stdout and a one-line summary to stderr. + + Returns the exit code so the caller can hand it straight to the shell. + """ + emit(envelope(error=error.to_dict(), meta=meta), fmt, secrets=secrets) + summary = error.message + if to_hide := [*(secrets or []), *known_secrets()]: + summary = scrub(summary, to_hide) + print(f"error: {summary}", file=sys.stderr) + return error.exit_code + + +def diagnostic( + message: str, *, quiet: bool = False, verbosity: int = 0, level: int = 0 +) -> None: + """Write a human-facing note to **stderr**, keeping stdout parseable. + + ``level`` is the minimum ``-v`` count required: 0 always shows (unless + ``--quiet``), 1 needs ``-v``, 2 needs ``-vv``. + """ + if quiet or verbosity < level: + return + print(message, file=sys.stderr) + + +__all__ = [ + "AGENT_ENV", + "CONTRACT_VERSION", + "AgentMode", + "OutputFormat", + "agent_detected", + "diagnostic", + "emit", + "emit_error", + "emit_result", + "emit_text", + "envelope", + "raw_value", + "render", + "render_table", + "resolve_format", +] diff --git a/src/unstract_cli/core/overlay.py b/src/unstract_cli/core/overlay.py new file mode 100644 index 0000000..337f1d6 --- /dev/null +++ b/src/unstract_cli/core/overlay.py @@ -0,0 +1,38 @@ +"""What the specs cannot say about a flag. + +The committed specs are generated from server code, so they carry names, types +and defaults but no allowed-value lists, no short flags and, today, no parameter +descriptions. Those live here rather than in the derivation, so adding one is an +edit to a data file instead of a special case in code. + +TOML, read with the stdlib, for the same reason the config file is TOML: no +parser dependency, and the file stays editable without a code change. + +Anything not overridden falls through to the spec, so an empty overlay is a +valid overlay. +""" + +from __future__ import annotations + +import tomllib +from functools import cache +from importlib import resources +from typing import Any + +OVERLAY_FILE = "overlay.toml" + + +@cache +def load_overlay() -> dict[str, Any]: + """Read the packaged overlay.""" + text = (resources.files("unstract_cli") / OVERLAY_FILE).read_text(encoding="utf-8") + return tomllib.loads(text) + + +def overlay_for(product: str, operation_id: str) -> dict[str, dict[str, Any]]: + """Per-parameter overrides for one operation, keyed by parameter name.""" + entries = load_overlay().get(product, {}).get(operation_id, {}) + return {name: entry for name, entry in entries.items() if isinstance(entry, dict)} + + +__all__ = ["OVERLAY_FILE", "load_overlay", "overlay_for"] diff --git a/src/unstract_cli/core/params.py b/src/unstract_cli/core/params.py new file mode 100644 index 0000000..971876f --- /dev/null +++ b/src/unstract_cli/core/params.py @@ -0,0 +1,425 @@ +"""Command flags, derived from the committed API specs. + +The specs are the same artifacts the published clients are generated from, so a +parameter the API gains reaches the CLI by refreshing a JSON file rather than by +hand-editing a flag list that drifts the moment nobody looks at it. + +Two rules make the derivation safe to hand to a caller: + +* **A flag not passed is not sent.** Every option defaults to ``None``, which + means "absent", and the server's own default applies. Writing the spec's + default into the request instead would pin a value the server would otherwise + choose, and the two diverge the moment the server's default moves. +* **A falsy value is a choice, not an absence.** ``0``, ``false`` and ``""`` all + travel; only ``None`` is filtered. + +What the spec cannot express -- allowed values, short flags, wording -- comes +from the overlay, never from a guess made here. +""" + +from __future__ import annotations + +import inspect +import json +import re +from collections.abc import Callable +from dataclasses import dataclass, replace +from functools import cache +from importlib import resources +from typing import Any + +import click + +from unstract_cli.core.overlay import overlay_for + +#: Spec file per product, vendored so flags derive with no network and no +#: dependency on where the client happens to be installed from. +SPEC_FILES = {"llmwhisperer": "llmwhisperer.json", "docstudio": "docstudio.json"} + +_HTTP_METHODS = frozenset({"get", "post", "put", "patch", "delete"}) + +#: OpenAPI type -> Click type. `array` is handled separately, as repetition. +_TYPES: dict[str, click.ParamType] = { + "string": click.STRING, + "integer": click.INT, + "number": click.FLOAT, +} + + +@cache +def load_spec(product: str) -> dict[str, Any]: + """Read one vendored spec.""" + try: + filename = SPEC_FILES[product] + except KeyError: + raise KeyError(f"No spec vendored for product {product!r}") from None + text = (resources.files("unstract_cli.specs") / filename).read_text(encoding="utf-8") + return json.loads(text) + + +def find_operation(product: str, operation_id: str) -> dict[str, Any]: + """Look one operation up by its operationId.""" + for path, methods in load_spec(product)["paths"].items(): + for method, operation in methods.items(): + if method in _HTTP_METHODS and operation.get("operationId") == operation_id: + return {"path": path, "method": method, **operation} + raise KeyError(f"{product} spec declares no operation {operation_id!r}") + + +@dataclass(frozen=True) +class Param: + """One request parameter, as the spec describes it.""" + + name: str + type: str = "string" + default: Any = None + description: str = "" + array: bool = False + nullable: bool = False + required: bool = False + choices: tuple[str, ...] = () + + @property + def flag(self) -> str: + return "--" + self.name.replace("_", "-") + + +def _from_schema( + name: str, schema: dict[str, Any], description: str, *, required: bool = False +) -> Param: + """Read one parameter out of its JSON schema. + + Nullability has two spellings -- a `null` branch in a type union, and 3.0's + `nullable` keyword, which is what the deployment spec uses. The null branch + carries no information for a flag, so the other branch decides the type. + """ + types = schema.get("type") + if isinstance(types, list): + nullable = "null" in types + remaining = [t for t in types if t != "null"] + type_name = remaining[0] if remaining else "string" + else: + nullable = bool(schema.get("nullable")) + type_name = types or "string" + + array = type_name == "array" + if array: + item = schema.get("items") or {} + type_name = item.get("type", "string") + + return Param( + name=name, + type=type_name, + default=schema.get("default"), + description=(description or schema.get("description") or "").strip(), + array=array, + nullable=nullable, + required=required, + choices=tuple(schema.get("enum") or ()), + ) + + +def operation_params(product: str, operation_id: str) -> list[Param]: + """Every parameter one operation accepts: query, then request body. + + Path parameters are excluded: they are the route, supplied by the command + from configuration, not by the caller as a flag. So are deprecated ones: a + superseded spelling the client still accepts would otherwise become a second + flag for the same value. + """ + operation = find_operation(product, operation_id) + params = [ + _from_schema( + p["name"], + p.get("schema") or {}, + p.get("description", ""), + required=bool(p.get("required")), + ) + for p in operation.get("parameters", []) + if p.get("in") == "query" and not p.get("deprecated") + ] + + body = operation.get("requestBody", {}).get("content", {}) + for media_type, content in body.items(): + # A binary body is the document itself, passed as an argument. + if media_type == "application/octet-stream": + continue + schema = content.get("schema") or {} + if ref := schema.get("$ref"): + schema = _resolve_ref(product, ref) + mandatory = set(schema.get("required") or ()) + for name, prop in (schema.get("properties") or {}).items(): + if prop.get("deprecated"): + continue + params.append( + _from_schema( + name, + prop, + prop.get("description", ""), + required=name in mandatory, + ) + ) + + return params + + +def client_params(method: Callable[..., Any]) -> dict[str, inspect.Parameter]: + """The parameters a client method accepts, by name. + + The published clients are frozen, so a spec parameter the client's signature + does not name cannot be reached at all: passing it raises ``TypeError`` + rather than sending it. Flags are intersected with this to keep the CLI's + surface equal to what actually works. + """ + return { + name: p + for name, p in inspect.signature(method).parameters.items() + if name not in ("self", "cls") + } + + +#: Python annotation -> OpenAPI type. A source-derived spec describes the wire, +#: which can differ from what the client method takes. +_ANNOTATIONS: dict[Any, str] = { + bool: "boolean", + int: "integer", + float: "number", + str: "string", +} + + +def _is_unset(value: Any) -> bool: + """Whether a default is a generated client's "absent" sentinel. + + Matched by name rather than by import: each client ships its own ``Unset`` + inside its generated tree, and that path is regenerated wholesale. + """ + return type(value).__name__ == "Unset" + + +def _from_signature(param: Param, signature: inspect.Parameter) -> Param: + """Reconcile a spec parameter with the client signature that will carry it.""" + updates: dict[str, Any] = {} + if (mapped := _ANNOTATIONS.get(signature.annotation)) is not None: + updates["type"] = mapped + # Whether a flag is mandatory is the spec's answer, not the signature's: a + # signature with no default says only that the *call* cannot omit the + # argument, which the command answers by supplying one. + if signature.default is not inspect.Parameter.empty and not _is_unset( + signature.default + ): + # What omitting the flag gets you: an `Unset` default sends nothing, so + # the spec's default is the one that applies. + updates["default"] = signature.default + return replace(param, **updates) + + +#: `name (type, optional): description` -- the Args entry of a Google-style +#: docstring, which is how both clients document their parameters. +_ARG_LINE = re.compile(r"^\s*(\w+)\s*(\([^)]*\))?\s*:\s*(.*)$") + +#: Sentences a description restates from elsewhere, stripped in the order they +#: appear. Each pattern ends at its own sentence, so prose after it survives. +_RESTATED = ( + re.compile(r"\s*Defaults to .*?\.(?=\s+[A-Z]|\s*$)"), + re.compile(r'\s*Can be ".*?"\s*\.'), +) + + +def docstring_params(method: Callable[..., Any]) -> dict[str, str]: + """Parameter descriptions from a client method's own docstring. + + The specs are generated from server code and carry no parameter + descriptions, while the published clients document every parameter. Reading + the docstring keeps one description per parameter, maintained where the + parameter is implemented, instead of a second copy here that goes stale + quietly. + """ + doc = inspect.getdoc(method) or "" + _, _, args = doc.partition("Args:") + if not args: + return {} + + out: dict[str, str] = {} + current: str | None = None + for line in args.splitlines(): + if not line.strip(): + continue + if line[:1] not in " \t" or re.match(r"^\s{0,4}(Returns|Raises|Yields):", line): + break + if (match := _ARG_LINE.match(line)) and (match.group(2) or current is None): + current = match.group(1) + out[current] = match.group(3).strip() + elif current: + out[current] = f"{out[current]} {line.strip()}".strip() + # Default and allowed values are rendered from the signature and the spec; + # the docstring's own copy of them would disagree as soon as either moves. + return {name: _strip_restated(text) for name, text in out.items() if text} + + +def _strip_restated(text: str) -> str: + text = " ".join(text.split()) + for pattern in _RESTATED: + text = pattern.sub("", text) + return text.strip() + + +def _resolve_ref(product: str, ref: str) -> dict[str, Any]: + node: Any = load_spec(product) + for part in ref.lstrip("#/").split("/"): + node = node[part] + return node + + +def _help_text(param: Param, choices: tuple[str, ...]) -> str: + """Help for one flag: what it does, what it accepts, what omitting it means. + + The default is reported but never applied. It answers "what happens if I + leave this out", which is the only question a default can honestly answer + here: the CLI does not resend it, the client or the server does. + """ + parts = [param.description] if param.description else [] + if choices: + parts.append(f"One of: {', '.join(choices)}.") + if param.default not in (None, "") and not param.required: + rendered = ( + str(param.default).lower() + if isinstance(param.default, bool) + else str(param.default) + ) + parts.append(f"[default: {rendered}]") + return " ".join(parts) + + +def click_option(param: Param, spec_overlay: dict[str, Any]) -> click.Option: + """Build one Click option from a spec parameter and its overlay entry.""" + entry = spec_overlay.get(param.name, {}) + # Falling back to the spec's own enum, so a hand-written list is needed only + # to narrow one on purpose -- a copy of it goes stale as the service grows. + choices = tuple(entry.get("choices", ())) or param.choices + help_text = entry.get("help") or _help_text(param, choices) + short = entry.get("short") + + # A required option is left without one: from Click 8.2 an explicit default + # counts as a value the caller supplied, and `required` stops being enforced. + absent: dict[str, Any] = {} if param.required else {"default": None} + + if param.type == "boolean": + # A paired flag, not `is_flag`: a default-true parameter cannot be + # turned off by an on-only flag, and `None` keeps "not passed" apart + # from "passed false". + decls = [f"{param.flag}/--no-{param.name.replace('_', '-')}"] + if short: + decls.insert(0, short) + return click.Option(decls, required=param.required, help=help_text, **absent) + + decls = [param.flag] + if short: + decls.insert(0, short) + return click.Option( + decls, + type=click.Choice(choices) if choices else _TYPES.get(param.type, click.STRING), + required=param.required, + multiple=param.array, + help=help_text, + **absent, + ) + + +def derive_params( + product: str, + operation_id: str, + *, + client_method: Callable[..., Any] | None = None, + exclude: tuple[str, ...] = (), +) -> list[Param]: + """The parameters one command exposes, in spec order. + + With ``client_method``, the spec is intersected with what that method + accepts and the method's own defaults win, because that is the value the + caller gets by omitting the flag. A spec parameter the method does not name + is dropped rather than offered and then rejected at the call. + """ + spec_overlay = overlay_for(product, operation_id) + hidden = {name for name, entry in spec_overlay.items() if entry.get("hidden")} + accepted = client_params(client_method) if client_method is not None else None + described = docstring_params(client_method) if client_method is not None else {} + + out: list[Param] = [] + for param in operation_params(product, operation_id): + if param.name in exclude or param.name in hidden: + continue + if accepted is not None: + if param.name not in accepted: + continue + param = _from_signature(param, accepted[param.name]) + if not param.description and (text := described.get(param.name)): + param = replace(param, description=text) + out.append(param) + return out + + +def spec_options( + product: str, + operation_id: str, + *, + client_method: Callable[..., Any] | None = None, + exclude: tuple[str, ...] = (), +) -> Callable[[Any], Any]: + """Decorator: hang one operation's parameters off a command as options. + + ``exclude`` drops parameters the command supplies itself -- the document to + extract is an argument, not a flag, and the CLI owns the polling that + ``use_webhook`` would bypass. + + Applies either above or below ``@group.command()``: above it decorates a + built command, below it a bare function that Click has yet to build. + """ + spec_overlay = overlay_for(product, operation_id) + + def decorate(target: Any) -> Any: + options = [ + click_option(param, spec_overlay) + for param in derive_params( + product, operation_id, client_method=client_method, exclude=exclude + ) + ] + if isinstance(target, click.Command): + target.params.extend(options) + else: + # Click reads this list back in reverse, so the help lists the + # parameters in the order the spec declares them. + pending = getattr(target, "__click_params__", []) + target.__click_params__ = list(reversed(options)) + pending + return target + + return decorate + + +def requested(values: dict[str, Any], *, drop: tuple[str, ...] = ()) -> dict[str, Any]: + """Keep the parameters the caller actually passed. + + ``None`` is the only absence. An empty tuple from a repeatable option is one + too -- Click spells "not passed" that way for ``multiple=True`` -- but ``0``, + ``False`` and ``""`` are values the caller chose and must survive. + """ + return { + name: list(value) if isinstance(value, tuple) else value + for name, value in values.items() + if name not in drop and value is not None and value != () + } + + +__all__ = [ + "SPEC_FILES", + "Param", + "click_option", + "client_params", + "derive_params", + "docstring_params", + "find_operation", + "load_spec", + "operation_params", + "requested", + "spec_options", +] diff --git a/src/unstract_cli/core/poll.py b/src/unstract_cli/core/poll.py new file mode 100644 index 0000000..02e380d --- /dev/null +++ b/src/unstract_cli/core/poll.py @@ -0,0 +1,310 @@ +"""`--wait` state machine and one-shot result persistence. + +Both products follow execute -> poll -> retrieve, and a caller should not have to +script that loop. + +**The load-bearing rule:** terminal state is decided by the ``status`` field in +the *response body*, never by the HTTP status code. The deployment API returns +HTTP 422 for the in-progress states, so reading the body means this behaves +identically before and after that is fixed server-side. + +The engine takes callables rather than owning any transport: the clients issue +every request, and the clock is injected so the whole thing tests offline. +""" + +from __future__ import annotations + +import json +import os +import tempfile +import time +from collections.abc import Callable +from contextlib import suppress +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from unstract_cli.core.errors import CLIError, ExitCode + + +@dataclass(frozen=True) +class PollSpec: + """How to read progress out of one operation's responses.""" + + #: Where the job handle lives in the initial response. Echoed back on + #: timeout so a caller can resume rather than reprocess the document. + handle_field: str + terminal_success: tuple[str, ...] + terminal_failure: tuple[str, ...] + #: One name, or candidates tried in order: the run POST and the status GET + #: spell the state differently. + status_field: str | tuple[str, ...] = "status" + + +def _dig(payload: Any, field: str) -> Any: + """Find a field, looking one level into the common envelopes.""" + if not isinstance(payload, dict): + return None + if field in payload: + return payload[field] + for envelope in ("message", "data", "result"): + inner = payload.get(envelope) + if isinstance(inner, dict) and field in inner: + return inner[field] + return None + + +def extract_status(payload: Any, field: str | tuple[str, ...] = "status") -> str | None: + """Read the status from a response body; first candidate that resolves wins.""" + fields = (field,) if isinstance(field, str) else field + for candidate in fields: + value = _dig(payload, candidate) + if value is not None: + return str(value) + return None + + +def extract_handle(payload: Any, field: str) -> str | None: + """Read the job handle out of a response body.""" + value = _dig(payload, field) + return str(value) if value is not None else None + + +def preflight(path: str | Path) -> Path: + """Prove the save target is writable, before anything destructive runs. + + `--save` exists to protect a read the server serves exactly once, so + discovering an unwritable path *after* that read is the one failure the + flag must not have. + """ + target = Path(path).expanduser() + # Saving here would replace the link itself, so it stops being a link and + # whatever it stands for stops being updated. + if target.is_symlink(): + raise CLIError( + f"--save target {path!r} is a symlink to {os.readlink(target)}: the " + "result would replace the link rather than update what it points at.", + ExitCode.USAGE, + hint=( + "Pass the path of the real file; nothing has been read yet, so " + "nothing is lost." + ), + ) + try: + target.parent.mkdir(parents=True, exist_ok=True) + existed = target.exists() + with target.open("a", encoding="utf-8"): + pass + if not existed: + target.unlink() + except OSError as exc: + raise CLIError( + f"Cannot write to --save target {path!r}: {exc}.", + ExitCode.USAGE, + hint="Pick a writable path; nothing has been read yet, so nothing is lost.", + ) from exc + return target + + +def persist(path: str | Path, payload: Any) -> Path: + """Write a result to disk and return where it landed. + + Some results can be read exactly once. Callers must persist **before** the + read is acknowledged to the user, so a crash between the two cannot destroy + a result the server will not serve again. + + Written through a temporary file so a full disk leaves the previous copy + intact rather than a truncated one. A failure here raises with the payload + attached: by this point the only surviving copy is in memory, and it has to + reach stdout somehow. + """ + target = Path(path).expanduser() + text = ( + payload + if isinstance(payload, str) + else json.dumps(payload, indent=2, default=str) + ) + tmp: Path | None = None + try: + target.parent.mkdir(parents=True, exist_ok=True) + # A predictable sibling in a directory someone else can write is a + # symlink waiting to be planted, and the write would follow it. `mkstemp` + # names it unpredictably and creates it exclusively; the 0600 it opens + # with is what `os.replace` then gives the result. + handle_fd, name = tempfile.mkstemp(dir=target.parent, suffix=".tmp") + tmp = Path(name) + with os.fdopen(handle_fd, "w", encoding="utf-8") as handle: + handle.write(text) + handle.flush() + os.fsync(handle.fileno()) + # Re-checked rather than taken on trust from the preflight: the target + # can become a link while the request that produced this result is in + # flight, and the rename would then destroy it. + if target.is_symlink(): + with suppress(OSError): + tmp.unlink(missing_ok=True) + raise CLIError( + f"{path!r} became a symlink to {os.readlink(target)} while the " + "result was being fetched: saving would replace the link rather " + "than update what it points at.", + ExitCode.SAVE_FAILED, + details=payload, + hint=( + "`details` carries the result. Save it to the real path -- it " + "has been read already, and will not be served again." + ), + ) + os.replace(tmp, target) + except OSError as exc: + if tmp is not None: + with suppress(OSError): + tmp.unlink(missing_ok=True) + raise CLIError( + f"The result could not be written to {path!r}: {exc}.", + ExitCode.SAVE_FAILED, + details=payload, + hint=( + "`details` carries the result. It has already been read from the " + "service, which will not serve it again -- save it from here." + ), + ) from exc + return target + + +def classify(payload: Any, spec: PollSpec) -> str: + """`success`, `failure`, `pending` or `unknown` for one poll response. + + Shared with the standalone status commands: a finished-and-failed execution + is reported inside an HTTP 200, so a command that only checks the status + code calls it a success. + """ + status = (extract_status(payload, spec.status_field) or "").lower() + if status in {state.lower() for state in spec.terminal_failure}: + return "failure" + if status in {state.lower() for state in spec.terminal_success}: + return "success" + if not status or _dig(payload, "error"): + # Not progress: polling on regardless reports a server fault as "still + # running" until the deadline. + return "unknown" + return "pending" + + +def wait_for_completion( + *, + initial: Any, + spec: PollSpec, + poll: Callable[[str], Any], + retrieve: Callable[[str], Any] | None = None, + save: str | Path | None = None, + interval: float = 3.0, + timeout: float = 300.0, + on_status: Callable[[str | None], None] | None = None, + #: Called with the path once a result is on disk, before the caller sees + #: anything. The ordering it observes is the whole point of --save. + on_saved: Callable[[Path], None] | None = None, + sleep: Callable[[float], None] = time.sleep, + now: Callable[[], float] = time.monotonic, +) -> Any: + """Poll until terminal, then retrieve if the operation has a retrieve step. + + On timeout, raises with the job handle attached, so a caller can resume with + a plain status/retrieve call rather than resubmitting the document. + """ + handle = extract_handle(initial, spec.handle_field) + if not handle: + return initial + + deadline = now() + timeout + last_status: str | None = None + payload: Any = initial + + def naming_the_job(call: Callable[[str], Any]) -> Any: + """Run one step of the loop, ensuring any failure names the job. + + The handle is the difference between resuming and paying to process the + document a second time, so it is attached here rather than left to + whatever the caller wrapped the loop in. + """ + try: + return call(handle) + except CLIError as exc: + exc.extra.setdefault(spec.handle_field, handle) + raise + except Exception as exc: + raise CLIError( + str(exc) or type(exc).__name__, + ExitCode.SERVER_ERROR, + retryable=True, + extra={spec.handle_field: handle}, + ) from exc + + while True: + payload = naming_the_job(poll) + status = extract_status(payload, spec.status_field) + + if status != last_status: + if on_status is not None: + on_status(status) + last_status = status + + state = classify(payload, spec) + if state == "failure": + raise CLIError( + f"Operation finished with status {status!r}.", + ExitCode.VALIDATION, + details=payload, + hint="Inspect `details` for the per-file error, or check the execution logs.", + extra={spec.handle_field: handle}, + ) + if state == "unknown": + raise CLIError( + "The service answered with neither a status nor progress.", + ExitCode.SERVER_ERROR, + details=payload, + retryable=True, + hint=( + "The response carries no usable state, so polling on would " + "only repeat it. Retry with the handle below." + ), + extra={spec.handle_field: handle}, + ) + if state == "success": + break + + remaining = deadline - now() + if remaining <= 0: + raise CLIError( + f"Timed out after {timeout:g}s waiting for completion " + f"(last status: {status!r}).", + ExitCode.TIMEOUT, + retryable=True, + hint=( + f"The job is still running. Resume with the {spec.handle_field} " + f"below rather than resubmitting the document." + ), + extra={spec.handle_field: handle, "last_status": status}, + ) + + # Never sleep past the deadline: --wait 30 that returns at 35s has lied, + # and the last poll should land on the deadline, not after it. + sleep(min(interval, remaining)) + + if retrieve is not None: + payload = naming_the_job(retrieve) + if save is not None: + written = persist(save, payload) + if on_saved is not None: + on_saved(written) + return payload + + +__all__ = [ + "PollSpec", + "classify", + "extract_handle", + "extract_status", + "persist", + "preflight", + "wait_for_completion", +] diff --git a/src/unstract_cli/overlay.toml b/src/unstract_cli/overlay.toml new file mode 100644 index 0000000..f563cac --- /dev/null +++ b/src/unstract_cli/overlay.toml @@ -0,0 +1,7 @@ +# Per-flag overrides for spec-derived options: [..]. +# +# Only what the spec cannot express belongs here. Names, types, defaults and +# allowed values are read from the spec, and help text falls back to the +# published client's own docstring, so an entry is needed only to add a short +# flag, narrow a value list on purpose, hide a parameter the CLI owns, or reword +# help the client states poorly. diff --git a/src/unstract_cli/specs/README.md b/src/unstract_cli/specs/README.md new file mode 100644 index 0000000..6831d27 --- /dev/null +++ b/src/unstract_cli/specs/README.md @@ -0,0 +1,24 @@ +# Vendored API specs + +Copies of the specs the two published clients are generated from, kept here so +flags derive with no network and no assumption about where a client was +installed from. Each one is produced by the service that serves it, never edited +by hand: + +| file | source | +|---|---| +| `llmwhisperer.json` | `specs/llmwhisperer.json` in the LLMWhisperer service repo, generated by `tools/gen_spec.py` | +| `docstudio.json` | `specs/docstudio-oss.json` in the backend, generated by `manage.py generate_docstudio_spec` | + +`provenance.json` pins the commit each copy was taken from and its sha256, and +`tests/test_specs.py` fails if a vendored file stops matching its pin. + +Refresh one by copying it byte-for-byte from the commit the client pinned in +`pyproject.toml` was generated from, then updating `provenance.json` to match. +Refreshing it against any other commit is what `tests/test_contract.py` guards: +a spec parameter the pinned client has no argument for cannot become a flag, and +that test names the ones that already cannot. + +A refresh that changes which flags a command offers fails against +`tests/derived_flags.json`. Read the difference before refreshing that file -- +a flag missing from it is a flag the CLI has stopped offering. diff --git a/src/unstract_cli/specs/docstudio.json b/src/unstract_cli/specs/docstudio.json new file mode 100644 index 0000000..c7f8ebb --- /dev/null +++ b/src/unstract_cli/specs/docstudio.json @@ -0,0 +1,722 @@ +{ + "components": { + "schemas": { + "AcknowledgedResponse": { + "description": "The execution's result was handed to an earlier call and discarded.", + "properties": { + "message": { + "type": "string" + }, + "status": { + "type": "string" + } + }, + "required": [ + "message", + "status" + ], + "type": "object" + }, + "ErrorDetail": { + "description": "One problem found with the request.", + "properties": { + "attr": { + "description": "The request field the problem belongs to, when it belongs to one.", + "nullable": true, + "type": "string" + }, + "code": { + "description": "Machine-readable problem identifier.", + "type": "string" + }, + "detail": { + "description": "Human-readable description.", + "type": "string" + } + }, + "required": [ + "attr", + "code", + "detail" + ], + "type": "object" + }, + "ErrorResponse": { + "description": "The body of a rejected request.\n\nProduced by the project-wide exception handler, so its shape is the same\nfor every failure listed against an operation.", + "properties": { + "errors": { + "items": { + "$ref": "#/components/schemas/ErrorDetail" + }, + "type": "array" + }, + "type": { + "$ref": "#/components/schemas/ErrorType" + } + }, + "required": [ + "errors", + "type" + ], + "type": "object" + }, + "ErrorType": { + "description": "* `validation_error` - validation_error\n* `client_error` - client_error\n* `server_error` - server_error", + "enum": [ + "validation_error", + "client_error", + "server_error" + ], + "type": "string" + }, + "ExecuteRequest": { + "description": "The documents to run, and the options that shape the result.\n\nSupply `files`, `presigned_urls`, or both.", + "properties": { + "custom_data": { + "nullable": true + }, + "files": { + "items": { + "format": "binary", + "type": "string" + }, + "type": "array" + }, + "hitl_packet_id": { + "description": "Groups documents reviewed together into one packet. Requires the enterprise manual-review capability; an installation without it rejects the request with 400.", + "nullable": true, + "type": "string" + }, + "hitl_queue_name": { + "description": "Document class name for the manual review queue. Requires the enterprise manual-review capability; an installation without it rejects the request with 400.", + "nullable": true, + "type": "string" + }, + "include_extracted_text": { + "default": false, + "type": "boolean" + }, + "include_metadata": { + "default": false, + "type": "boolean" + }, + "include_metrics": { + "default": false, + "type": "boolean" + }, + "llm_profile_id": { + "nullable": true, + "type": "string" + }, + "presigned_urls": { + "items": { + "format": "uri", + "type": "string" + }, + "type": "array" + }, + "tags": { + "default": "", + "description": "Comma-separated list of tag names (EX:'tag1,tag2-name,tag3_name')", + "type": "string" + }, + "timeout": { + "default": -1, + "maximum": 300, + "minimum": -1, + "type": "integer" + }, + "use_file_history": { + "default": false, + "type": "boolean" + } + }, + "type": "object" + }, + "ExecuteResponse": { + "properties": { + "message": { + "$ref": "#/components/schemas/ExecutionMessage" + } + }, + "required": [ + "message" + ], + "type": "object" + }, + "ExecutionMessage": { + "description": "The execution's identity and, once it has finished, its per-file\nresults.", + "properties": { + "error": { + "nullable": true, + "type": "string" + }, + "execution_id": { + "type": "string" + }, + "execution_status": { + "type": "string" + }, + "result": { + "items": { + "$ref": "#/components/schemas/FileResult" + }, + "nullable": true, + "type": "array" + }, + "status_api": { + "nullable": true, + "type": "string" + } + }, + "required": [ + "execution_id", + "execution_status" + ], + "type": "object" + }, + "FileResult": { + "description": "One input document's outcome.\n\nEvery key is present on every item; the ones that depend on the request\noptions or on the outcome are sent as `null` when they do not apply.", + "properties": { + "error": { + "nullable": true, + "type": "string" + }, + "extracted_text": { + "description": "The document's full extracted text. Sent only when the request set `include_extracted_text`.", + "nullable": true, + "type": "string" + }, + "file": { + "type": "string" + }, + "file_execution_id": { + "nullable": true, + "type": "string" + }, + "metadata": { + "nullable": true + }, + "result": { + "nullable": true + }, + "status": { + "type": "string" + } + }, + "required": [ + "file" + ], + "type": "object" + }, + "StatusResponse": { + "properties": { + "message": { + "items": { + "$ref": "#/components/schemas/FileResult" + }, + "nullable": true, + "type": "array" + }, + "status": { + "type": "string" + } + }, + "required": [ + "status" + ], + "type": "object" + } + }, + "securitySchemes": { + "deploymentKey": { + "description": "The API deployment's own key.", + "scheme": "bearer", + "type": "http" + } + } + }, + "info": { + "title": "Unstract API", + "version": "v1" + }, + "openapi": "3.0.3", + "paths": { + "/deployment/api/{org_name}/{api_name}/": { + "get": { + "description": "Read the result of a previously started execution.\n\nThis read is one-shot: the first call that observes a completed execution acknowledges it and the stored result is discarded, so every later call for that execution answers 406. Poll while the execution is pending, and keep the payload of the call that returns it \u2014 it cannot be fetched again.\n\nA still-running execution answers 422 carrying its current `status`, so a polling loop should treat 422 as the normal reply and stop on 200. Clients that raise on any non-2xx need to allow for that.", + "operationId": "status", + "parameters": [ + { + "description": "API deployment name.", + "in": "path", + "name": "api_name", + "required": true, + "schema": { + "pattern": "^[\\w-]+$", + "type": "string" + } + }, + { + "in": "query", + "name": "execution_id", + "required": true, + "schema": { + "minLength": 1, + "type": "string" + } + }, + { + "in": "query", + "name": "include_extracted_text", + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "in": "query", + "name": "include_metadata", + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "in": "query", + "name": "include_metrics", + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "description": "Organization identifier.", + "in": "path", + "name": "org_name", + "required": true, + "schema": { + "pattern": "^[\\w-]+$", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/StatusResponse" + } + } + }, + "description": "" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The request failed validation." + }, + "401": { + "content": { + "application/json": { + "examples": { + "AuthenticationFailed": { + "value": { + "errors": [ + { + "attr": null, + "code": "authentication_failed", + "detail": "Incorrect authentication credentials." + } + ], + "type": "client_error" + } + }, + "NotAuthenticated": { + "value": { + "errors": [ + { + "attr": null, + "code": "not_authenticated", + "detail": "Authentication credentials were not provided." + } + ], + "type": "client_error" + } + } + }, + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "No usable API key was supplied for the deployment." + }, + "403": { + "content": { + "application/json": { + "examples": { + "PermissionDenied": { + "value": { + "errors": [ + { + "attr": null, + "code": "permission_denied", + "detail": "You do not have permission to perform this action." + } + ], + "type": "client_error" + } + } + }, + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The request was refused as unauthorized." + }, + "404": { + "content": { + "application/json": { + "examples": { + "NotFound": { + "value": { + "errors": [ + { + "attr": null, + "code": "not_found", + "detail": "Not found." + } + ], + "type": "client_error" + } + } + }, + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "No active deployment, or a referenced document, was found." + }, + "406": { + "content": { + "application/json": { + "examples": { + "NotAcceptable": { + "value": { + "errors": [ + { + "attr": null, + "code": "not_acceptable", + "detail": "Could not satisfy the request Accept header." + } + ], + "type": "client_error" + } + } + }, + "schema": { + "$ref": "#/components/schemas/AcknowledgedResponse" + } + } + }, + "description": "The result was already consumed by an earlier call." + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/StatusResponse" + } + } + }, + "description": "The execution is still running, or it finished with an error; read `status` to tell them apart." + }, + "500": { + "content": { + "application/json": { + "examples": { + "APIException": { + "value": { + "errors": [ + { + "attr": null, + "code": "error", + "detail": "A server error occurred." + } + ], + "type": "server_error" + } + } + }, + "schema": { + "$ref": "#/components/schemas/StatusResponse" + } + } + }, + "description": "The execution could not be completed; the body carries its last known state." + } + }, + "security": [ + { + "deploymentKey": [] + } + ], + "tags": [ + "deployment" + ] + }, + "post": { + "description": "Execute an API deployment against one or more documents.\n\nSupply the documents either as `files` (multipart upload) or as `presigned_urls` (HTTPS S3 URLs), or both \u2014 a request carrying neither is rejected, and the two together may not exceed 32 documents.\n\nWith the default `timeout` of -1 the call returns as soon as the execution is queued; read the outcome from the status endpoint.", + "operationId": "execute", + "parameters": [ + { + "description": "API deployment name.", + "in": "path", + "name": "api_name", + "required": true, + "schema": { + "pattern": "^[\\w-]+$", + "type": "string" + } + }, + { + "description": "Organization identifier.", + "in": "path", + "name": "org_name", + "required": true, + "schema": { + "pattern": "^[\\w-]+$", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "$ref": "#/components/schemas/ExecuteRequest" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExecuteResponse" + } + } + }, + "description": "" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The request failed validation." + }, + "401": { + "content": { + "application/json": { + "examples": { + "AuthenticationFailed": { + "value": { + "errors": [ + { + "attr": null, + "code": "authentication_failed", + "detail": "Incorrect authentication credentials." + } + ], + "type": "client_error" + } + }, + "NotAuthenticated": { + "value": { + "errors": [ + { + "attr": null, + "code": "not_authenticated", + "detail": "Authentication credentials were not provided." + } + ], + "type": "client_error" + } + } + }, + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "No usable API key was supplied for the deployment." + }, + "403": { + "content": { + "application/json": { + "examples": { + "PermissionDenied": { + "value": { + "errors": [ + { + "attr": null, + "code": "permission_denied", + "detail": "You do not have permission to perform this action." + } + ], + "type": "client_error" + } + } + }, + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The request was refused as unauthorized." + }, + "404": { + "content": { + "application/json": { + "examples": { + "NotFound": { + "value": { + "errors": [ + { + "attr": null, + "code": "not_found", + "detail": "Not found." + } + ], + "type": "client_error" + } + } + }, + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "No active deployment, or a referenced document, was found." + }, + "413": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "A referenced document is larger than the limit." + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExecuteResponse" + } + } + }, + "description": "The execution finished with an error." + }, + "429": { + "content": { + "application/json": { + "examples": { + "Throttled": { + "value": { + "errors": [ + { + "attr": null, + "code": "throttled", + "detail": "Request was throttled." + } + ], + "type": "client_error" + } + } + }, + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "Too many concurrent executions; retry later." + }, + "500": { + "content": { + "application/json": { + "examples": { + "APIException": { + "value": { + "errors": [ + { + "attr": null, + "code": "error", + "detail": "A server error occurred." + } + ], + "type": "server_error" + } + } + }, + "schema": { + "$ref": "#/components/schemas/ExecuteResponse" + } + } + }, + "description": "The deployment could not be run; the body carries the execution that failed." + }, + "502": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "A referenced document could not be fetched." + }, + "504": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "Fetching a referenced document timed out." + } + }, + "security": [ + { + "deploymentKey": [] + } + ], + "tags": [ + "deployment" + ] + } + } + }, + "tags": [ + { + "description": "Run an API deployment against one or more documents and poll the result.", + "name": "deployment" + } + ] +} diff --git a/src/unstract_cli/specs/llmwhisperer.json b/src/unstract_cli/specs/llmwhisperer.json new file mode 100644 index 0000000..bef1fdd --- /dev/null +++ b/src/unstract_cli/specs/llmwhisperer.json @@ -0,0 +1,2709 @@ +{ + "components": { + "schemas": { + "Error": { + "properties": { + "message": { + "type": "string" + } + }, + "type": "object" + }, + "WebhookConfig": { + "properties": { + "auth_token": { + "type": "string" + }, + "url": { + "format": "uri", + "type": "string" + }, + "webhook_name": { + "type": "string" + } + }, + "required": [ + "url", + "auth_token", + "webhook_name" + ], + "type": "object" + }, + "WhisperAccepted": { + "properties": { + "format": { + "type": "string" + }, + "message": { + "type": "string" + }, + "status": { + "type": "string" + }, + "whisper_hash": { + "type": "string" + } + }, + "type": "object" + }, + "WhisperResult": { + "properties": { + "confidence_metadata": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array" + }, + "line_metadata": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array" + }, + "metadata": { + "additionalProperties": true, + "type": "object" + }, + "result_text": { + "type": "string" + }, + "webhook_metadata": { + "type": "string" + }, + "whisper_metadata": { + "additionalProperties": true, + "type": "object" + } + }, + "type": "object" + }, + "WhisperStatus": { + "properties": { + "detail": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array" + }, + "message": { + "type": "string" + }, + "status": { + "type": "string" + } + }, + "type": "object" + } + }, + "securitySchemes": { + "unstract_key": { + "in": "header", + "name": "unstract-key", + "type": "apiKey" + } + } + }, + "info": { + "description": "The hosted regions are listed under `servers`; a self-hosted deployment serves the same API from its own URL, which every client takes as a configuration option.", + "title": "Unstract LLMWhisperer", + "version": "v2" + }, + "openapi": "3.0.3", + "paths": { + "/api/v2/convert-to-pdf": { + "post": { + "operationId": "convert_to_pdf", + "parameters": [ + { + "in": "query", + "name": "url", + "required": false, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "url_in_post", + "required": false, + "schema": { + "default": false, + "type": "boolean" + } + } + ], + "requestBody": { + "content": { + "application/octet-stream": { + "schema": { + "format": "binary", + "type": "string" + } + } + }, + "required": false + }, + "responses": { + "200": { + "content": { + "application/pdf": { + "schema": { + "format": "binary", + "type": "string" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "402": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Quota exhausted, or the licence does not cover this request." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + }, + "415": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The uploaded file's type is not supported." + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Unhandled server error." + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "A dependency this operation needs is not configured or reachable." + } + }, + "summary": "Convert a document to PDF", + "tags": [ + "convert" + ] + } + }, + "/api/v2/convert-xlsb-to-xlsx": { + "post": { + "operationId": "convert_xlsb_to_xlsx", + "parameters": [ + { + "in": "query", + "name": "url", + "required": false, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "url_in_post", + "required": false, + "schema": { + "default": false, + "type": "boolean" + } + } + ], + "requestBody": { + "content": { + "application/octet-stream": { + "schema": { + "format": "binary", + "type": "string" + } + } + }, + "required": false + }, + "responses": { + "200": { + "content": { + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": { + "schema": { + "format": "binary", + "type": "string" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "402": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Quota exhausted, or the licence does not cover this request." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + }, + "415": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The uploaded file's type is not supported." + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Unhandled server error." + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "A dependency this operation needs is not configured or reachable." + } + }, + "summary": "Convert an XLSB workbook to XLSX", + "tags": [ + "convert" + ] + } + }, + "/api/v2/document-insights": { + "post": { + "operationId": "document_insights", + "parameters": [ + { + "in": "query", + "name": "file_name", + "required": false, + "schema": { + "default": "sample.pdf", + "type": "string" + } + }, + { + "in": "query", + "name": "pages_to_extract", + "required": false, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "tag", + "required": false, + "schema": { + "default": "default", + "type": "string" + } + }, + { + "in": "query", + "name": "url", + "required": false, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "url_in_post", + "required": false, + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "in": "query", + "name": "use_webhook", + "required": false, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "webhook_metadata", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/octet-stream": { + "schema": { + "format": "binary", + "type": "string" + } + } + }, + "required": false + }, + "responses": { + "202": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WhisperAccepted" + } + } + }, + "description": "Accepted" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "402": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Quota exhausted, or the licence does not cover this request." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + }, + "415": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The uploaded file's type is not supported." + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Unhandled server error." + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "A dependency this operation needs is not configured or reachable." + } + }, + "summary": "Run document insights over a file", + "tags": [ + "insights" + ] + } + }, + "/api/v2/document-insights-retrieve": { + "get": { + "operationId": "document_insights_retrieve", + "parameters": [ + { + "in": "query", + "name": "whisper_hash", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "402": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Quota exhausted, or the licence does not cover this request." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + }, + "415": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The uploaded file's type is not supported." + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Unhandled server error." + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "A dependency this operation needs is not configured or reachable." + } + }, + "summary": "Retrieve document insights result (destructive \u2014 one shot)", + "tags": [ + "insights" + ] + } + }, + "/api/v2/get-usage-info": { + "get": { + "operationId": "usage_info", + "parameters": [], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "402": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Quota exhausted, or the licence does not cover this request." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + }, + "415": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The uploaded file's type is not supported." + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Unhandled server error." + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "A dependency this operation needs is not configured or reachable." + } + }, + "summary": "Subscription usage summary", + "tags": [ + "account" + ] + } + }, + "/api/v2/highlights": { + "get": { + "operationId": "highlights", + "parameters": [ + { + "in": "query", + "name": "extract_all_lines", + "required": false, + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "description": "Line numbers or ranges, e.g. `1-5,9`. Required unless `extract_all_lines=true`.", + "in": "query", + "name": "lines", + "required": false, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "whisper_hash", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "402": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Quota exhausted, or the licence does not cover this request." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + }, + "415": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The uploaded file's type is not supported." + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Unhandled server error." + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "A dependency this operation needs is not configured or reachable." + } + }, + "summary": "Line-level highlight geometry for an extraction", + "tags": [ + "whisper" + ] + } + }, + "/api/v2/pdf-to-images": { + "post": { + "operationId": "pdf_to_images", + "parameters": [ + { + "in": "query", + "name": "file_name", + "required": false, + "schema": { + "default": "sample.pdf", + "type": "string" + } + }, + { + "in": "query", + "name": "format", + "required": false, + "schema": { + "default": "png", + "enum": [ + "png", + "jpeg" + ], + "type": "string" + } + }, + { + "in": "query", + "name": "tag", + "required": false, + "schema": { + "default": "default", + "type": "string" + } + }, + { + "in": "query", + "name": "url", + "required": false, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "url_in_post", + "required": false, + "schema": { + "default": false, + "type": "boolean" + } + } + ], + "requestBody": { + "content": { + "application/octet-stream": { + "schema": { + "format": "binary", + "type": "string" + } + } + }, + "required": false + }, + "responses": { + "202": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WhisperAccepted" + } + } + }, + "description": "Accepted" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "402": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Quota exhausted, or the licence does not cover this request." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + }, + "415": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The uploaded file's type is not supported." + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Unhandled server error." + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "A dependency this operation needs is not configured or reachable." + } + }, + "summary": "Render a PDF's pages as images", + "tags": [ + "convert" + ] + } + }, + "/api/v2/pdf-to-images-retrieve": { + "get": { + "operationId": "pdf_to_images_retrieve", + "parameters": [ + { + "in": "query", + "name": "whisper_hash", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/zip": { + "schema": { + "format": "binary", + "type": "string" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "402": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Quota exhausted, or the licence does not cover this request." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + }, + "415": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The uploaded file's type is not supported." + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Unhandled server error." + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "A dependency this operation needs is not configured or reachable." + } + }, + "summary": "Retrieve rendered images as a zip (destructive \u2014 one shot)", + "tags": [ + "convert" + ] + } + }, + "/api/v2/pdf-to-images-status": { + "get": { + "operationId": "pdf_to_images_status", + "parameters": [ + { + "in": "query", + "name": "whisper_hash", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WhisperStatus" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "402": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Quota exhausted, or the licence does not cover this request." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + }, + "415": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The uploaded file's type is not supported." + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Unhandled server error." + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "A dependency this operation needs is not configured or reachable." + } + }, + "summary": "Poll PDF-to-images status", + "tags": [ + "convert" + ] + } + }, + "/api/v2/test-connection": { + "get": { + "operationId": "test_connection", + "parameters": [], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "402": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Quota exhausted, or the licence does not cover this request." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + }, + "415": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The uploaded file's type is not supported." + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Unhandled server error." + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "A dependency this operation needs is not configured or reachable." + } + }, + "summary": "Verify credentials", + "tags": [ + "account" + ] + } + }, + "/api/v2/usage": { + "get": { + "operationId": "usage", + "parameters": [ + { + "in": "query", + "name": "from_date", + "required": false, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "tag", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "to_date", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "402": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Quota exhausted, or the licence does not cover this request." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + }, + "415": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The uploaded file's type is not supported." + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Unhandled server error." + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "A dependency this operation needs is not configured or reachable." + } + }, + "summary": "Detailed usage statistics", + "tags": [ + "account" + ] + } + }, + "/api/v2/whisper": { + "post": { + "operationId": "extract", + "parameters": [ + { + "in": "query", + "name": "add_line_nos", + "required": false, + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "in": "query", + "name": "allow_rotated_text", + "required": false, + "schema": { + "default": true, + "type": "boolean" + } + }, + { + "in": "query", + "name": "checkbox_confidence_threshold", + "required": false, + "schema": { + "default": 0.3, + "type": "number" + } + }, + { + "in": "query", + "name": "derotate_threshold", + "required": false, + "schema": { + "default": 10.0, + "type": "number" + } + }, + { + "in": "query", + "name": "file_name", + "required": false, + "schema": { + "default": "sample.pdf", + "type": "string" + } + }, + { + "in": "query", + "name": "gaussian_blur_radius", + "required": false, + "schema": { + "default": 0, + "type": "number" + } + }, + { + "in": "query", + "name": "horizontal_stretch_factor", + "required": false, + "schema": { + "default": 1.0, + "type": "number" + } + }, + { + "in": "query", + "name": "ignore_vertical_text", + "required": false, + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "in": "query", + "name": "include_line_confidence", + "required": false, + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "in": "query", + "name": "lang", + "required": false, + "schema": { + "default": "eng", + "type": "string" + } + }, + { + "in": "query", + "name": "line_splitter_strategy", + "required": false, + "schema": { + "default": "left-priority", + "enum": [ + "left-priority", + "mid-priority", + "right-priority" + ], + "type": "string" + } + }, + { + "in": "query", + "name": "line_splitter_tolerance", + "required": false, + "schema": { + "default": 0.75, + "type": "number" + } + }, + { + "in": "query", + "name": "mark_horizontal_lines", + "required": false, + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "in": "query", + "name": "mark_vertical_lines", + "required": false, + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "in": "query", + "name": "median_filter_size", + "required": false, + "schema": { + "default": 0, + "type": "integer" + } + }, + { + "in": "query", + "name": "min_table_width", + "required": false, + "schema": { + "default": 0.0, + "type": "number" + } + }, + { + "in": "query", + "name": "mode", + "required": false, + "schema": { + "default": "form", + "enum": [ + "document_insights", + "excel", + "form", + "high_quality", + "low_cost", + "native_text", + "table" + ], + "type": "string" + } + }, + { + "in": "query", + "name": "output_mode", + "required": false, + "schema": { + "default": "layout_preserving", + "enum": [ + "dump-text", + "layout_preserving", + "line-printer", + "text" + ], + "type": "string" + } + }, + { + "in": "query", + "name": "page_separator", + "required": false, + "schema": { + "default": "<<<", + "type": "string" + } + }, + { + "deprecated": true, + "description": "Deprecated misspelling of `page_separator`, read only when that one is absent. Send both to stay compatible with older deployments.", + "in": "query", + "name": "page_seperator", + "required": false, + "schema": { + "default": "<<<", + "type": "string" + } + }, + { + "in": "query", + "name": "pages_to_extract", + "required": false, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "tag", + "required": false, + "schema": { + "default": "default", + "type": "string" + } + }, + { + "description": "Fetch the document from this URL instead of sending a body.", + "in": "query", + "name": "url", + "required": false, + "schema": { + "format": "uri", + "type": "string" + } + }, + { + "description": "Read the URL to fetch from the request body.", + "in": "query", + "name": "url_in_post", + "required": false, + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "in": "query", + "name": "use_webhook", + "required": false, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "watermark_angle_threshold", + "required": false, + "schema": { + "default": 25.0, + "type": "number" + } + }, + { + "in": "query", + "name": "webhook_metadata", + "required": false, + "schema": { + "type": "string" + } + }, + { + "description": "Minimum per-word OCR confidence to report. Defaults to 0.05 unless the deployment overrides it.", + "in": "query", + "name": "word_confidence_threshold", + "required": false, + "schema": { + "type": "number" + } + } + ], + "requestBody": { + "content": { + "application/octet-stream": { + "schema": { + "format": "binary", + "type": "string" + } + } + }, + "required": false + }, + "responses": { + "202": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WhisperAccepted" + } + } + }, + "description": "Accepted" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "402": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Quota exhausted, or the licence does not cover this request." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + }, + "415": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The uploaded file's type is not supported." + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Unhandled server error." + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "A dependency this operation needs is not configured or reachable." + } + }, + "summary": "Submit a document for text extraction", + "tags": [ + "whisper" + ] + } + }, + "/api/v2/whisper-detail": { + "get": { + "operationId": "detail", + "parameters": [ + { + "in": "query", + "name": "whisper_hash", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "402": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Quota exhausted, or the licence does not cover this request." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + }, + "415": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The uploaded file's type is not supported." + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Unhandled server error." + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "A dependency this operation needs is not configured or reachable." + } + }, + "summary": "Metadata about a whisper job", + "tags": [ + "whisper" + ] + } + }, + "/api/v2/whisper-manage-callback": { + "delete": { + "operationId": "webhook_delete", + "parameters": [ + { + "in": "query", + "name": "webhook_name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "402": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Quota exhausted, or the licence does not cover this request." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + }, + "415": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The uploaded file's type is not supported." + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Unhandled server error." + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "A dependency this operation needs is not configured or reachable." + } + }, + "summary": "Manage extraction webhooks", + "tags": [ + "webhook" + ] + }, + "get": { + "operationId": "webhook_get", + "parameters": [ + { + "in": "query", + "name": "webhook_name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "402": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Quota exhausted, or the licence does not cover this request." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + }, + "415": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The uploaded file's type is not supported." + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Unhandled server error." + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "A dependency this operation needs is not configured or reachable." + } + }, + "summary": "Manage extraction webhooks", + "tags": [ + "webhook" + ] + }, + "post": { + "operationId": "webhook_post", + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhookConfig" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object" + } + } + }, + "description": "Created" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "402": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Quota exhausted, or the licence does not cover this request." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + }, + "415": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The uploaded file's type is not supported." + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Unhandled server error." + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "A dependency this operation needs is not configured or reachable." + } + }, + "summary": "Manage extraction webhooks", + "tags": [ + "webhook" + ] + }, + "put": { + "operationId": "webhook_put", + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhookConfig" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "402": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Quota exhausted, or the licence does not cover this request." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + }, + "415": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The uploaded file's type is not supported." + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Unhandled server error." + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "A dependency this operation needs is not configured or reachable." + } + }, + "summary": "Manage extraction webhooks", + "tags": [ + "webhook" + ] + } + }, + "/api/v2/whisper-retrieve": { + "get": { + "operationId": "retrieve", + "parameters": [ + { + "in": "query", + "name": "text_only", + "required": false, + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "in": "query", + "name": "whisper_hash", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WhisperResult" + } + }, + "text/plain": { + "schema": { + "type": "string" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "402": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Quota exhausted, or the licence does not cover this request." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + }, + "415": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The uploaded file's type is not supported." + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Unhandled server error." + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "A dependency this operation needs is not configured or reachable." + } + }, + "summary": "Retrieve extraction result (destructive \u2014 one shot)", + "tags": [ + "whisper" + ] + } + }, + "/api/v2/whisper-status": { + "get": { + "operationId": "status", + "parameters": [ + { + "in": "query", + "name": "whisper_hash", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WhisperStatus" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "402": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Quota exhausted, or the licence does not cover this request." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + }, + "415": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The uploaded file's type is not supported." + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "Unhandled server error." + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "A dependency this operation needs is not configured or reachable." + } + }, + "summary": "Poll extraction status", + "tags": [ + "whisper" + ] + } + } + }, + "security": [ + { + "unstract_key": [] + } + ], + "servers": [ + { + "description": "US region (the default of the published clients).", + "url": "https://llmwhisperer-api.us-central.unstract.com" + }, + { + "description": "EU region.", + "url": "https://llmwhisperer-api.eu-west.unstract.com" + } + ] +} diff --git a/src/unstract_cli/specs/provenance.json b/src/unstract_cli/specs/provenance.json new file mode 100644 index 0000000..22719d3 --- /dev/null +++ b/src/unstract_cli/specs/provenance.json @@ -0,0 +1,14 @@ +{ + "docstudio.json": { + "repo": "https://github.com/Zipstack/unstract", + "commit": "0c5f36dabf497220f82917a5f4f92f2cd396b5a5", + "path": "specs/docstudio-oss.json", + "sha256": "e453d4f7444d3757a24a1da73373b11c3d362ceb2d7e13e8658a5b3c068b86f5" + }, + "llmwhisperer.json": { + "repo": "https://github.com/Zipstack/unstract-llm-whisperer", + "commit": "750f941ee229e12cc05d8bd85edaab6a337a8758", + "path": "specs/llmwhisperer.json", + "sha256": "88ecc01e92443ba5ba6079db7f57cf3038f97670cb796f13c326268a3d79f366" + } +} diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..5d33b03 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import os +from fnmatch import fnmatch + +import pytest + +from unstract_cli import config as config_mod +from unstract_cli.core.output import AGENT_ENV + +#: Every variable the loader consults. Cleared per test so a developer's real +#: shell environment cannot change a result. +_ENV_VARS = sorted( + {var for vars_ in config_mod.ENV_VARS.values() for var in vars_} + | {"UNSTRACT_CONFIG", "UNSTRACT_PROFILE"} +) + + +@pytest.fixture(autouse=True) +def clean_env(monkeypatch, tmp_path): + for var in _ENV_VARS: + monkeypatch.delenv(var, raising=False) + # These decide the default output format, and this suite is as likely to be + # run by an agent as by a person. + for var in [ + name + for name in os.environ + if any(fnmatch(name, pattern) for pattern in AGENT_ENV) + ]: + monkeypatch.delenv(var, raising=False) + config_mod.set_config_path(None) + # Both discovery fallbacks are redirected into the tmp dir: an upward search + # from a real cwd could otherwise find a developer's own .unstract.toml. + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(config_mod, "HOME_CONFIG", tmp_path / "home" / "config.toml") + yield + config_mod.set_config_path(None) + + +@pytest.fixture +def write_config(tmp_path, monkeypatch): + """Write a config file and point the CLI at it.""" + + def _write(text: str): + path = tmp_path / "config.toml" + path.write_text(text, encoding="utf-8") + monkeypatch.setenv("UNSTRACT_CONFIG", str(path)) + return path + + return _write diff --git a/tests/derived_flags.json b/tests/derived_flags.json new file mode 100644 index 0000000..2c6277c --- /dev/null +++ b/tests/derived_flags.json @@ -0,0 +1,457 @@ +{ + "llmwhisperer:extract": { + "--add-line-nos": { + "name": "add_line_nos", + "type": "boolean", + "default": false, + "description": "Adds line numbers to the extracted text and saves line metadata, which can be queried later using the highlights API.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--allow-rotated-text": { + "name": "allow_rotated_text", + "type": "boolean", + "default": null, + "description": "Whether to keep words whose own orientation is rotated. With this off, a word angled further than watermark_angle_threshold is treated as a watermark and excluded.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--checkbox-confidence-threshold": { + "name": "checkbox_confidence_threshold", + "type": "number", + "default": null, + "description": "The minimum confidence a detected checkbox mark must have to be reported as marked. Accepts a value in the range [0.0, 1.0].", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--derotate-threshold": { + "name": "derotate_threshold", + "type": "number", + "default": null, + "description": "The page rotation in degrees beyond which the page is straightened and re-read.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--file-name": { + "name": "file_name", + "type": "string", + "default": null, + "description": "The name of the file to store in reports.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--gaussian-blur-radius": { + "name": "gaussian_blur_radius", + "type": "integer", + "default": 0, + "description": "The radius of the Gaussian blur.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--horizontal-stretch-factor": { + "name": "horizontal_stretch_factor", + "type": "number", + "default": 1.0, + "description": "The horizontal stretch factor.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--ignore-vertical-text": { + "name": "ignore_vertical_text", + "type": "boolean", + "default": null, + "description": "Whether to drop vertically oriented text instead of extracting it.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--include-line-confidence": { + "name": "include_line_confidence", + "type": "boolean", + "default": false, + "description": "Adds line confidence to the line metadata returned by the highlights API. Requires add_line_nos to be enabled.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--lang": { + "name": "lang", + "type": "string", + "default": "eng", + "description": "The language of the document.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--line-splitter-strategy": { + "name": "line_splitter_strategy", + "type": "string", + "default": null, + "description": "The line splitter strategy.", + "array": false, + "nullable": false, + "required": false, + "choices": [ + "left-priority", + "mid-priority", + "right-priority" + ] + }, + "--line-splitter-tolerance": { + "name": "line_splitter_tolerance", + "type": "number", + "default": 0.4, + "description": "The line splitter tolerance. This client pins its own default below the service's, and has always sent it, so the two are expected to differ.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--mark-horizontal-lines": { + "name": "mark_horizontal_lines", + "type": "boolean", + "default": false, + "description": "Whether to mark horizontal lines.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--mark-vertical-lines": { + "name": "mark_vertical_lines", + "type": "boolean", + "default": false, + "description": "Whether to mark vertical lines.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--median-filter-size": { + "name": "median_filter_size", + "type": "integer", + "default": 0, + "description": "The size of the median filter.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--min-table-width": { + "name": "min_table_width", + "type": "number", + "default": null, + "description": "The minimum width a table must span, as a fraction of the page width, to be extracted as a table.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--mode": { + "name": "mode", + "type": "string", + "default": "form", + "description": "The processing mode.", + "array": false, + "nullable": false, + "required": false, + "choices": [ + "document_insights", + "excel", + "form", + "high_quality", + "low_cost", + "native_text", + "table" + ] + }, + "--output-mode": { + "name": "output_mode", + "type": "string", + "default": "layout_preserving", + "description": "The output mode.", + "array": false, + "nullable": false, + "required": false, + "choices": [ + "dump-text", + "layout_preserving", + "line-printer", + "text" + ] + }, + "--page-separator": { + "name": "page_separator", + "type": "string", + "default": null, + "description": "The page separator.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--pages-to-extract": { + "name": "pages_to_extract", + "type": "string", + "default": "", + "description": "The pages to extract.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--tag": { + "name": "tag", + "type": "string", + "default": "default", + "description": "The tag for the document.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--url": { + "name": "url", + "type": "string", + "default": "", + "description": "Fetch the document from this URL instead of sending a body.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--use-webhook": { + "name": "use_webhook", + "type": "string", + "default": "", + "description": "Webhook name to call. If not provided, then no webhook will be called.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--watermark-angle-threshold": { + "name": "watermark_angle_threshold", + "type": "number", + "default": null, + "description": "The angle in degrees beyond which a rotated word counts as a watermark. Only applies when allow_rotated_text is off.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--webhook-metadata": { + "name": "webhook_metadata", + "type": "string", + "default": "", + "description": "The webhook metadata. This data will be passed to the webhook if webhooks are used", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--word-confidence-threshold": { + "name": "word_confidence_threshold", + "type": "number", + "default": 0.3, + "description": "Minimum per-word OCR confidence to report. Defaults to 0.05 unless the deployment overrides it.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + } + }, + "llmwhisperer:highlights": { + "--extract-all-lines": { + "name": "extract_all_lines", + "type": "boolean", + "default": false, + "description": "", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--lines": { + "name": "lines", + "type": "string", + "default": null, + "description": "Line numbers or ranges, e.g. `1-5,9`. Required unless `extract_all_lines=true`.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--whisper-hash": { + "name": "whisper_hash", + "type": "string", + "default": null, + "description": "The hash of the whisper operation.", + "array": false, + "nullable": false, + "required": true, + "choices": [] + } + }, + "docstudio:execute": { + "--custom-data": { + "name": "custom_data", + "type": "string", + "default": null, + "description": "Arbitrary JSON. The service returns it under each result item's ``metadata.custom_data``, which is server behaviour: the spec carries the field on the request only, so the round trip is not declared and nothing here pins it. Anything that is not already a string is serialised to JSON before it is sent.", + "array": false, + "nullable": true, + "required": false, + "choices": [] + }, + "--hitl-packet-id": { + "name": "hitl_packet_id", + "type": "string", + "default": null, + "description": "Groups documents reviewed together into one packet. Requires the enterprise manual-review capability; an installation without it rejects the request with 400.", + "array": false, + "nullable": true, + "required": false, + "choices": [] + }, + "--hitl-queue-name": { + "name": "hitl_queue_name", + "type": "string", + "default": null, + "description": "Document class name for the manual review queue. Requires the enterprise manual-review capability; an installation without it rejects the request with 400.", + "array": false, + "nullable": true, + "required": false, + "choices": [] + }, + "--include-extracted-text": { + "name": "include_extracted_text", + "type": "boolean", + "default": false, + "description": "Include the extracted text.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--include-metadata": { + "name": "include_metadata", + "type": "boolean", + "default": false, + "description": "Include metadata in the result.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--include-metrics": { + "name": "include_metrics", + "type": "boolean", + "default": false, + "description": "Include metrics in the result.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--llm-profile-id": { + "name": "llm_profile_id", + "type": "string", + "default": null, + "description": "LLM profile to override the deployment's.", + "array": false, + "nullable": true, + "required": false, + "choices": [] + }, + "--presigned-urls": { + "name": "presigned_urls", + "type": "string", + "default": null, + "description": "URLs to fetch the inputs from.", + "array": true, + "nullable": false, + "required": false, + "choices": [] + }, + "--tags": { + "name": "tags", + "type": "string", + "default": "", + "description": "Comma-separated list of tag names (EX:'tag1,tag2-name,tag3_name')", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--timeout": { + "name": "timeout", + "type": "integer", + "default": -1, + "description": "Execution mode \u2014 ``0`` or below queues the execution and returns immediately; above it the call runs synchronously.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--use-file-history": { + "name": "use_file_history", + "type": "boolean", + "default": false, + "description": "Reuse a previous result for the same file.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + } + }, + "docstudio:status": { + "--include-extracted-text": { + "name": "include_extracted_text", + "type": "boolean", + "default": false, + "description": "Include the extracted text.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--include-metadata": { + "name": "include_metadata", + "type": "boolean", + "default": false, + "description": "Include metadata in the result.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--include-metrics": { + "name": "include_metrics", + "type": "boolean", + "default": false, + "description": "Include metrics in the result.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + } + } +} diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..7184189 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,203 @@ +"""End-to-end through the entry point: exit codes reach the shell, stdout parses.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from unstract_cli import app +from unstract_cli.__main__ import main +from unstract_cli.app import cli, command_tree +from unstract_cli.core.errors import ExitCode + + +def run(capsys, *args): + """Invoke the CLI as the console script does, returning (code, stdout json). + + `-o json` is passed the way any consumer has to pass it: the default format + is human-facing, and a test that relied on it would be pinning the wrong + thing. + """ + code = main(["-o", "json", *args]) + captured = capsys.readouterr() + payload = json.loads(captured.out) if captured.out.strip() else None + return code, payload, captured.err + + +def test_v1_groups_are_registered(): + tree = command_tree() + assert set(tree) >= {"config", "whisper", "docstudio"} + assert "deployment" in tree["docstudio"]["commands"] + assert set(tree["config"]["commands"]) == {"doctor", "get", "init", "list", "set"} + + +def test_help_exits_zero(capsys): + assert main(["--help"]) == int(ExitCode.SUCCESS) + + +def test_unknown_command_is_a_usage_error_with_an_envelope(capsys): + code, payload, err = run(capsys, "nope") + assert code == int(ExitCode.USAGE) + assert payload["ok"] is False + assert payload["error"]["exit_code"] == int(ExitCode.USAGE) + assert err.startswith("error:") + + +def test_an_interrupt_exits_one_thirty_with_an_envelope(capsys, monkeypatch): + """Ctrl-C is not a failure of the command. Reporting it as a generic error + tells a supervisor to retry what the user deliberately stopped.""" + + def interrupted(): + raise KeyboardInterrupt + + monkeypatch.setattr("unstract_cli.commands.config_cmd.load_config", interrupted) + + code, payload, _ = run(capsys, "config", "doctor") + + assert code == int(ExitCode.INTERRUPTED) == 130 + assert payload["ok"] is False + assert payload["error"]["code"] == "interrupted" + + +def test_unknown_config_target_exits_two(capsys): + code, payload, _ = run(capsys, "config", "get", "nosuchproduct", "base_url") + assert code == int(ExitCode.USAGE) + assert "llmwhisperer" in payload["error"]["hint"] + + +def test_set_then_get_round_trip(capsys, tmp_path, monkeypatch): + monkeypatch.setenv("UNSTRACT_CONFIG", str(tmp_path / "c.toml")) + + code, payload, _ = run(capsys, "config", "set", "docstudio", "org_id", "org_A") + assert code == 0 and payload["ok"] is True + + code, payload, _ = run(capsys, "config", "get", "docstudio", "org_id") + assert code == 0 + assert payload["data"]["value"] == "org_A" + + +def test_set_warns_when_a_credential_is_stored_literally(capsys, tmp_path, monkeypatch): + monkeypatch.setenv("UNSTRACT_CONFIG", str(tmp_path / "c.toml")) + _, payload, _ = run(capsys, "config", "set", "llmwhisperer", "api_key", "literal-key") + assert "env:VAR_NAME" in payload["data"]["warning"] + + +def test_get_never_echoes_a_credential(capsys, tmp_path, monkeypatch): + monkeypatch.setenv("UNSTRACT_CONFIG", str(tmp_path / "c.toml")) + run(capsys, "config", "set", "llmwhisperer", "api_key", "super-secret-value") + _, payload, _ = run(capsys, "config", "get", "llmwhisperer", "api_key") + assert payload["data"]["value"] == "***SET***" + assert "super-secret-value" not in json.dumps(payload) + + +def test_init_refuses_to_clobber_without_force(capsys, tmp_path, monkeypatch): + monkeypatch.setenv("UNSTRACT_CONFIG", str(tmp_path / "c.toml")) + assert run(capsys, "config", "init")[0] == 0 + + code, payload, _ = run(capsys, "config", "init") + assert code == int(ExitCode.USAGE) + assert "--force" in payload["error"]["hint"] + + assert run(capsys, "config", "init", "--force")[0] == 0 + + +def test_doctor_reports_sources_without_leaking_values(capsys, monkeypatch): + monkeypatch.setenv("LLMWHISPERER_API_KEY", "super-secret-value") + code, payload, _ = run(capsys, "config", "doctor") + assert code == 0 + products = payload["data"]["products"] + assert products["llmwhisperer"]["api_key"] == { + "resolved": True, + "source": "env:LLMWHISPERER_API_KEY", + } + assert products["docstudio"]["api_key"]["resolved"] is False + assert "super-secret-value" not in json.dumps(payload) + + +def doctor(capsys, *args) -> str: + """`config doctor` -- a command with no network -- and its raw stdout.""" + main([*args, "config", "doctor"]) + return capsys.readouterr().out + + +def is_table(out: str) -> bool: + try: + json.loads(out) + except json.JSONDecodeError: + return "active_profile" in out + return False + + +class TestOutputFormatEndToEnd: + """One rule: `-o` decides, and where it is absent the environment picks the + default only. Everything here is a way of getting that wrong.""" + + def test_the_default_is_a_table_in_a_terminal_and_in_a_pipe( + self, capsys, monkeypatch + ): + monkeypatch.setattr("sys.stdout.isatty", lambda: True, raising=False) + assert is_table(doctor(capsys)) + monkeypatch.setattr("sys.stdout.isatty", lambda: False, raising=False) + assert is_table(doctor(capsys)) + + def test_no_isatty_call_decides_a_format(self): + """A format that depends on a terminal makes a script's output depend on + how it was launched.""" + source = Path(app.__file__).parent + offenders = [ + path.name + for path in source.rglob("*.py") + if "isatty" in path.read_text(encoding="utf-8") + ] + assert offenders == [] + + def test_an_agent_environment_makes_json_the_default(self, capsys, monkeypatch): + monkeypatch.setenv("CLAUDECODE", "1") + assert json.loads(doctor(capsys))["ok"] is True + + def test_an_explicit_format_wins_over_a_detected_agent(self, capsys, monkeypatch): + monkeypatch.setenv("CLAUDECODE", "1") + assert is_table(doctor(capsys, "-o", "table")) + + def test_agent_no_forces_the_human_default(self, capsys, monkeypatch): + monkeypatch.setenv("CLAUDECODE", "1") + assert is_table(doctor(capsys, "--agent", "no")) + + def test_json_is_byte_identical_however_it_was_asked_for(self, capsys, monkeypatch): + monkeypatch.setattr("sys.stdout.isatty", lambda: True, raising=False) + on_a_tty = doctor(capsys, "-o", "json") + + monkeypatch.setattr("sys.stdout.isatty", lambda: False, raising=False) + monkeypatch.setenv("CLAUDECODE", "1") + piped_under_an_agent = doctor(capsys, "-o", "json") + + assert on_a_tty == piped_under_an_agent + + def test_every_envelope_carries_the_contract_version(self, capsys): + assert run(capsys, "config", "doctor")[1]["meta"]["contract_version"] == 1 + assert run(capsys, "nope")[1]["meta"]["contract_version"] == 1 + + +def test_the_config_group_says_what_it_withheld(capsys, tmp_path, monkeypatch): + """`config list` is one of the commands run *to understand* the config. + + It loads the file itself rather than through the root context, so it has to + report the file's warnings on its own or stay silent about its own subject. + """ + work = tmp_path / "checkout" + work.mkdir() + (work / ".unstract.toml").write_text( + '[profiles.p.llmwhisperer]\napi_key = "planted"\n', encoding="utf-8" + ) + monkeypatch.chdir(work) + + _, _, err = run(capsys, "config", "list") + assert err.count("Ignoring p.llmwhisperer.api_key") == 1 + + +def test_click_parameter_info_dict_keeps_the_keys_discovery_reads(): + # Discovery derives flags from Click's own introspection; a Click bump that + # reshaped this dict would silently degrade it. + param = next(p for p in cli.params if p.name == "output") + info = param.to_info_dict() + assert {"name", "opts", "help", "type", "required"} <= set(info) diff --git a/tests/test_commands.py b/tests/test_commands.py new file mode 100644 index 0000000..7cca255 --- /dev/null +++ b/tests/test_commands.py @@ -0,0 +1,1273 @@ +"""The product commands, with the clients replaced. No network. + +The seam is the client factory, not the transport: what matters here is which +arguments a command hands the client, what it does with the reply, and what a +caller sees on stdout and in the exit code. +""" + +from __future__ import annotations + +import json +import os +import socket + +import httpx +import pytest +from requests.exceptions import ConnectionError +from unstract.clone.report import CloneReport, Endpoint, PhaseResult +from unstract.llmwhisperer import client_v2 +from unstract.llmwhisperer.client_v2 import ( + LLMWhispererClientException, + LLMWhispererClientV2, +) + +from unstract_cli.__main__ import main +from unstract_cli.app import command_tree +from unstract_cli.commands import clone_cmd, docstudio_cmd, whisper_cmd +from unstract_cli.config import LLMWHISPERER +from unstract_cli.core.errors import CLIError, ExitCode + + +def run(capsys, *args): + """Invoke the CLI as the console script does, returning (code, stdout, stderr). + + `-o json` explicitly: these assert on the parseable output, which is what a + caller opts into rather than what an unflagged run happens to print. + """ + code = main(["-o", "json", *args]) + captured = capsys.readouterr() + return code, captured.out, captured.err + + +def envelope(out: str) -> dict: + return json.loads(out) + + +def _name_resolution_error(host: str) -> ConnectionError: + """The failure the pinned client raises when a host does not resolve. + + Built by putting a transport error through the client's own translation + rather than assembled here: the client re-raises with only a message, so a + hand-made stand-in can keep passing long after the client has stopped + producing anything like it. + """ + + def fail(): + request = httpx.Request("GET", f"https://{host}/api/v2/get-usage-info") + raise httpx.ConnectError( + "[Errno -2] Name or service not known", request=request + ) from socket.gaierror(-2, "Name or service not known") + + try: + client_v2._translate_transport_errors(fail) + except ConnectionError as exc: + return exc + raise AssertionError("the pinned client no longer translates a connect error") + + +class FakeWhisper: + """Records calls; returns whatever the test queued.""" + + def __init__(self, **replies): + self.replies = replies + self.calls: list[tuple[str, tuple, dict]] = [] + + def _reply(self, name, *args, **kwargs): + self.calls.append((name, args, kwargs)) + reply = self.replies.get(name) + if isinstance(reply, Exception): + raise reply + if isinstance(reply, list): + return reply.pop(0) if len(reply) > 1 else reply[0] + return reply + + #: Pure geometry on a reply, so the real implementation is used rather than + #: a queued answer. + get_highlight_rect = LLMWhispererClientV2.get_highlight_rect + + def __getattr__(self, name): + def call(*args, **kwargs): + return self._reply(name, *args, **kwargs) + + return call + + def kwargs_for(self, name) -> dict: + return next(kw for called, _, kw in self.calls if called == name) + + +@pytest.fixture +def whisper_client(monkeypatch): + """Install a fake LLMWhisperer client and hand it back to the test.""" + + def install(**replies): + client = FakeWhisper(**replies) + # Resolving the credential is what registers it for scrubbing, so the + # fake factory has to do it too or the seam hides a production path. + monkeypatch.setattr( + whisper_cmd, + "llmwhisperer", + lambda config: (config.get(LLMWHISPERER, "api_key"), client)[1], + ) + return client + + return install + + +@pytest.fixture +def deployment_client(monkeypatch): + """Install a fake deployment client and hand it back to the test.""" + + def install(**replies): + client = FakeWhisper(**replies) + client.api_url = "https://api.example.com/deployment/api/org/api-name/" + client.built_with = {} + + def build(_config, _target, transport_timeout=None): + client.built_with["transport_timeout"] = transport_timeout + return client + + monkeypatch.setattr(docstudio_cmd, "deployment", build) + return client + + return install + + +# --------------------------------------------------------------------------- # +# The command surface +# --------------------------------------------------------------------------- # + + +def test_the_v1_commands_are_registered(): + tree = command_tree() + assert set(tree["whisper"]["commands"]) == { + "detail", + "extract", + "highlights", + "retrieve", + "status", + "usage", + "webhook", + } + assert set(tree["whisper"]["commands"]["webhook"]["commands"]) == { + "create", + "delete", + "get", + "update", + } + assert set(tree["docstudio"]["commands"]["deployment"]["commands"]) == { + "run", + "status", + } + + +# --------------------------------------------------------------------------- # +# whisper extract +# --------------------------------------------------------------------------- # + + +def test_extract_without_wait_returns_the_handle(capsys, whisper_client, tmp_path): + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + client = whisper_client(whisper={"whisper_hash": "h1", "status_code": 202}) + + code, out, _ = run(capsys, "whisper", "extract", str(doc), "--no-wait") + + assert code == int(ExitCode.SUCCESS) + assert envelope(out)["data"]["whisper_hash"] == "h1" + assert client.kwargs_for("whisper")["file_path"] == str(doc) + + +def test_only_the_flags_that_were_passed_reach_the_client( + capsys, whisper_client, tmp_path +): + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + client = whisper_client(whisper={"whisper_hash": "h1"}) + + run(capsys, "whisper", "extract", str(doc), "--no-wait", "--mode", "table") + + sent = client.kwargs_for("whisper") + assert sent["mode"] == "table" + assert "lang" not in sent and "median_filter_size" not in sent + + +def test_a_falsy_flag_still_reaches_the_client(capsys, whisper_client, tmp_path): + """`--median-filter-size 0` is a choice; a truthiness filter would drop it + and silently leave the client's own default in place.""" + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + client = whisper_client(whisper={"whisper_hash": "h1"}) + + run( + capsys, + "whisper", + "extract", + str(doc), + "--no-wait", + "--median-filter-size", + "0", + "--no-add-line-nos", + ) + + sent = client.kwargs_for("whisper") + assert sent["median_filter_size"] == 0 + assert sent["add_line_nos"] is False + + +def test_a_url_source_is_sent_as_a_url(capsys, whisper_client): + client = whisper_client(whisper={"whisper_hash": "h1"}) + run(capsys, "whisper", "extract", "https://example.com/a.pdf", "--no-wait") + sent = client.kwargs_for("whisper") + assert sent["url"] == "https://example.com/a.pdf" and "file_path" not in sent + + +def test_the_cli_owns_the_wait_loop(capsys, whisper_client, tmp_path): + """The client has a blocking loop of its own; using it would make --interval, + --timeout and the handle-on-timeout behaviour product-specific.""" + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + client = whisper_client( + whisper={"whisper_hash": "h1"}, + whisper_status=[{"status": "processing"}, {"status": "processed"}], + whisper_retrieve={"extraction": {"result_text": "hello"}}, + ) + + code, out, _ = run(capsys, "-q", "whisper", "extract", str(doc), "--interval", "0") + + assert code == int(ExitCode.SUCCESS) + assert client.kwargs_for("whisper")["wait_for_completion"] is False + assert envelope(out)["data"] == {"result_text": "hello"} + + +def test_raw_output_prints_the_extracted_text(capsys, whisper_client, tmp_path): + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + whisper_client( + whisper={"whisper_hash": "h1"}, + whisper_status={"status": "processed"}, + whisper_retrieve={"extraction": {"result_text": "hello"}}, + ) + + _, out, _ = run( + capsys, "-q", "-o", "raw", "whisper", "extract", str(doc), "--interval", "0" + ) + assert out.strip() == "hello" + + +def test_wait_and_use_webhook_are_mutually_exclusive(capsys, whisper_client, tmp_path): + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + whisper_client(whisper={"whisper_hash": "h1"}) + + code, out, _ = run( + capsys, "whisper", "extract", str(doc), "--use-webhook", "wh1", "--wait" + ) + assert code == int(ExitCode.USAGE) + assert "webhook" in envelope(out)["error"]["hint"] + + +def test_a_failed_extraction_carries_the_handle(capsys, whisper_client, tmp_path): + """A caller can resume from the handle rather than resubmitting.""" + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + whisper_client( + whisper={"whisper_hash": "h1"}, + whisper_status={"status": "error", "message": "bad scan"}, + ) + + code, out, _ = run(capsys, "-q", "whisper", "extract", str(doc), "--interval", "0") + assert code == int(ExitCode.VALIDATION) + assert envelope(out)["error"]["whisper_hash"] == "h1" + + +def test_a_transport_failure_mid_poll_carries_the_handle( + capsys, whisper_client, tmp_path +): + """The document is submitted and billed by this point. Without the handle the + only way on is to send it again and pay for it twice.""" + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + whisper_client( + whisper={"whisper_hash": "h1"}, + whisper_status=ConnectionError("connection dropped"), + ) + + code, out, _ = run(capsys, "-q", "whisper", "extract", str(doc), "--interval", "0") + assert code == int(ExitCode.SERVER_ERROR) + assert envelope(out)["error"]["whisper_hash"] == "h1" + + +def test_a_failed_retrieve_carries_the_handle(capsys, whisper_client, tmp_path): + """Retrieve is the acknowledging read: a failure here can lose the text and + the handle at once, and the handle is the only way back to either.""" + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + whisper_client( + whisper={"whisper_hash": "h1"}, + whisper_status={"status": "processed"}, + whisper_retrieve=ConnectionError("connection dropped"), + ) + + code, out, _ = run(capsys, "-q", "whisper", "extract", str(doc), "--interval", "0") + assert code == int(ExitCode.SERVER_ERROR) + assert envelope(out)["error"]["whisper_hash"] == "h1" + + +# --------------------------------------------------------------------------- # +# Retrieval is one-shot +# --------------------------------------------------------------------------- # + + +def test_retrieve_saves_before_it_prints(capsys, whisper_client, tmp_path): + """A result can be read once. Persisting after printing loses it to a broken + pipe or a full terminal buffer.""" + target = tmp_path / "out" / "result.json" + whisper_client(whisper_retrieve={"extraction": {"result_text": "hello"}}) + + code, out, _ = run(capsys, "whisper", "retrieve", "h1", "--save", str(target)) + + assert code == int(ExitCode.SUCCESS) + assert json.loads(target.read_text())["result_text"] == "hello" + assert envelope(out)["data"]["result_text"] == "hello" + + +def test_an_already_consumed_result_has_its_own_exit_code(capsys, whisper_client): + whisper_client(whisper_retrieve=LLMWhispererClientException("already retrieved", 406)) + code, out, _ = run(capsys, "whisper", "retrieve", "h1") + assert code == int(ExitCode.ALREADY_CONSUMED) + assert "once" in envelope(out)["error"]["hint"] + + +# --------------------------------------------------------------------------- # +# Errors from the client +# --------------------------------------------------------------------------- # + + +def test_an_auth_failure_maps_onto_its_exit_code(capsys, whisper_client): + whisper_client(get_usage_info=LLMWhispererClientException("bad key", 401)) + code, out, _ = run(capsys, "whisper", "usage") + assert code == int(ExitCode.AUTH) + assert envelope(out)["error"]["message"] == "bad key" + + +def test_an_error_body_keeps_its_own_wording(capsys, whisper_client): + whisper_client( + whisper_detail=LLMWhispererClientException( + {"message": "no such hash", "status_code": 404} + ) + ) + code, out, _ = run(capsys, "whisper", "detail", "h1") + assert code == int(ExitCode.NOT_FOUND) + error = envelope(out)["error"] + assert error["message"] == "no such hash" + assert error["details"]["status_code"] == 404 + + +# --------------------------------------------------------------------------- # +# highlights +# --------------------------------------------------------------------------- # + + +def test_highlights_scales_line_metadata_when_a_page_size_is_given( + capsys, whisper_client +): + """Pure arithmetic on the reply, so it is folded into this command rather + than being a command that makes no request.""" + whisper_client(get_highlight_data={"1": [1, 100, 20, 1000]}) + code, out, _ = run( + capsys, + "whisper", + "highlights", + "h1", + "--lines", + "1-5", + "--target-width", + "600", + "--target-height", + "800", + ) + assert code == int(ExitCode.SUCCESS) + data = envelope(out)["data"] + assert data["rects"]["1"] == [1, 0, 64, 600, 80] + + +def test_highlights_reads_the_named_metadata_object(capsys, whisper_client): + """The service returns the list inside an object; the client's geometry takes + the bare list.""" + whisper_client(get_highlight_data={"1": {"raw": [1, 100, 20, 1000], "page": 1}}) + _, out, _ = run( + capsys, + "whisper", + "highlights", + "h1", + "--lines", + "1-5", + "--target-width", + "600", + "--target-height", + "800", + ) + assert envelope(out)["data"]["rects"]["1"] == [1, 0, 64, 600, 80] + + +def test_a_line_without_geometry_gets_no_box(capsys, whisper_client): + """The service reports a line it has no geometry for as all zeros, and the + page height is a divisor in the scaling.""" + whisper_client( + get_highlight_data={"1": {"raw": [0, 0, 0, 0]}, "2": {"raw": [1, 100, 20, 1000]}} + ) + code, out, _ = run( + capsys, + "whisper", + "highlights", + "h1", + "--lines", + "1-5", + "--target-width", + "600", + "--target-height", + "800", + ) + assert code == int(ExitCode.SUCCESS) + assert set(envelope(out)["data"]["rects"]) == {"2"} + + +def test_highlights_returns_the_metadata_alone_without_a_page_size( + capsys, whisper_client +): + whisper_client(get_highlight_data={"1": [1, 100, 20, 1000]}) + _, out, _ = run(capsys, "whisper", "highlights", "h1", "--lines", "1-5") + assert envelope(out)["data"] == {"1": [1, 100, 20, 1000]} + + +def test_a_host_that_does_not_resolve_is_not_worth_retrying(capsys, whisper_client): + """Every other connection failure is transient; a name that does not resolve + is a typo, and a caller told to retry retries against it forever.""" + whisper_client(get_usage_info=_name_resolution_error("nope.invalid")) + code, out, _ = run(capsys, "whisper", "usage") + assert code == int(ExitCode.SERVER_ERROR) + error = envelope(out)["error"] + assert error["retryable"] is False + assert "nope.invalid" in error["message"] + + +@pytest.mark.skipif( + not os.environ.get("UNSTRACT_CLI_LIVE"), + reason="asks the resolver about a host; set UNSTRACT_CLI_LIVE=1 to run it", +) +def test_a_real_resolver_failure_reaches_the_same_answer(capsys, monkeypatch): + """The offline stand-in is built by hand, however carefully. This one asks + the pinned client to reach a name no resolver will answer for.""" + monkeypatch.setenv("LLMWHISPERER_API_KEY", "k") + monkeypatch.setenv("LLMWHISPERER_BASE_URL", "https://unresolvable.invalid/api/v2") + code, out, _ = run(capsys, "whisper", "usage") + assert code == int(ExitCode.SERVER_ERROR) + error = envelope(out)["error"] + assert error["retryable"] is False + assert "unresolvable.invalid" in error["message"] + + +def test_an_unreachable_service_is_worth_retrying(capsys, whisper_client): + whisper_client(get_usage_info=ConnectionError("connection refused")) + code, out, _ = run(capsys, "whisper", "usage") + assert code == int(ExitCode.SERVER_ERROR) + assert envelope(out)["error"]["retryable"] is True + + +def test_highlights_needs_lines_or_all_of_them(capsys, whisper_client): + """The API takes either; asking for neither is a usage error, not a call.""" + whisper_client(get_highlight_data={}) + code, out, _ = run(capsys, "whisper", "highlights", "h1") + assert code == int(ExitCode.USAGE) + assert "--extract-all-lines" in envelope(out)["error"]["message"] + + +def test_extract_all_lines_stands_in_for_a_line_range(capsys, whisper_client): + """The client takes `lines` positionally even when the request does not need + it, so omitting the flag would raise inside the client rather than answer.""" + client = whisper_client(get_highlight_data={"1": [1, 100, 20, 1000]}) + code, out, _ = run(capsys, "whisper", "highlights", "h1", "--extract-all-lines") + assert code == int(ExitCode.SUCCESS) + sent = client.kwargs_for("get_highlight_data") + assert sent == {"lines": "", "extract_all_lines": True} + + +# --------------------------------------------------------------------------- # +# Deployments +# --------------------------------------------------------------------------- # + + +def test_run_queues_the_execution_and_polls_it(capsys, deployment_client, tmp_path): + """`timeout=0` queues, so the CLI holds the poll loop instead of the request + holding a connection open for the length of the job.""" + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + client = deployment_client( + structure_file={ + "status_code": 200, + "pending": True, + "execution_status": "PENDING", + "status_check_api_endpoint": "/status?execution_id=e1", + }, + check_execution_status=[ + {"status_code": 200, "pending": True, "execution_status": "EXECUTING"}, + { + "status_code": 200, + "pending": False, + "execution_status": "COMPLETED", + "extraction_result": [{"file": "doc.pdf"}], + }, + ], + ) + + code, out, _ = run( + capsys, + "-q", + "docstudio", + "deployment", + "run", + "my-api", + str(doc), + "--interval", + "0", + ) + + assert code == int(ExitCode.SUCCESS) + assert client.kwargs_for("structure_file")["timeout"] == 0 + assert envelope(out)["data"]["execution_status"] == "COMPLETED" + + +@pytest.mark.parametrize( + ("flag", "expected"), [([], None), (["--transport-timeout", "12.5"], 12.5)] +) +def test_the_transport_timeout_flag_reaches_the_client( + capsys, deployment_client, tmp_path, flag, expected +): + """Unset means a stalled connection is never given up on, which is what + the client has always done.""" + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + client = deployment_client( + structure_file={"status_code": 200, "execution_status": "COMPLETED"} + ) + + code, _out, _err = run( + capsys, "-q", "docstudio", *flag, "deployment", "run", "my-api", str(doc) + ) + + assert code == int(ExitCode.SUCCESS) + assert client.built_with["transport_timeout"] == expected + + +def test_run_passes_only_the_flags_that_were_given(capsys, deployment_client, tmp_path): + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + client = deployment_client( + structure_file={"status_code": 200, "execution_status": "COMPLETED"} + ) + + run( + capsys, + "docstudio", + "deployment", + "run", + "my-api", + str(doc), + "--no-wait", + "--tags", + "a,b", + "--no-include-metrics", + ) + + sent = client.kwargs_for("structure_file") + assert sent["tags"] == "a,b" + assert sent["include_metrics"] is False + assert "llm_profile_id" not in sent + + +def test_a_queued_run_reports_the_handle_it_started(capsys, deployment_client, tmp_path): + """Without --wait the answer is an acknowledgement, so the only thing worth + printing is what the caller polls with.""" + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + deployment_client( + structure_file={ + "status_code": 200, + "execution_status": "PENDING", + "execution_id": "e-1", + "extraction_result": None, + } + ) + + code, out, _ = run( + capsys, "docstudio", "deployment", "run", "my-api", str(doc), "--no-wait" + ) + assert code == int(ExitCode.SUCCESS) + assert envelope(out)["meta"]["execution_id"] == "e-1" + + code = main( + [ + "-o", + "raw", + "docstudio", + "deployment", + "run", + "my-api", + str(doc), + "--no-wait", + ] + ) + assert code == int(ExitCode.SUCCESS) + assert capsys.readouterr().out.strip() == "e-1" + + +def test_a_target_that_is_not_a_configured_alias_names_the_ones_that_are( + capsys, deployment_client, write_config +): + """A misspelt alias is sent as an API name and comes back not-found, which + says nothing about the aliases sitting in the profile.""" + write_config( + 'default_profile = "p"\n' + "[profiles.p.docstudio]\n" + 'org_id = "org"\n' + 'api_key = "k"\n' + "[profiles.p.deployments.invoices]\n" + 'api_name = "invoice-parser"\n' + ) + deployment_client(check_execution_status={"status_code": 404, "error": "not found"}) + code, out, _ = run(capsys, "docstudio", "deployment", "status", "invoces", "e-1") + assert code == int(ExitCode.NOT_FOUND) + hint = envelope(out)["error"]["hint"] + assert "invoces" in hint and "invoices" in hint + + +def test_highlights_on_an_extraction_without_line_numbers_says_where_to_fix_it( + capsys, whisper_client +): + """The call that can be fixed is the extract, which has already been paid + for; a hint about this call sends the caller nowhere.""" + whisper_client( + get_highlight_data=LLMWhispererClientException( + {"message": "no line metadata", "status_code": 400}, 400 + ) + ) + code, out, _ = run(capsys, "whisper", "highlights", "h1", "--lines", "1-5") + assert code == int(ExitCode.VALIDATION) + assert "--add-line-nos" in envelope(out)["error"]["hint"] + + +ACK = { + "status_code": 200, + "execution_status": "PENDING", + "extraction_result": None, + "status_check_api_endpoint": "/deployment/api/status?execution_id=e-1", +} + +PENDING_STATUS = { + "status_code": 422, + "pending": True, + "execution_status": "EXECUTING", + "extraction_result": None, +} + +DONE_STATUS = { + "status_code": 200, + "execution_status": "COMPLETED", + "extraction_result": "the answer", +} + + +def _raw(capsys, *args) -> str: + assert main(["-o", "raw", *args]) == int(ExitCode.SUCCESS) + return capsys.readouterr().out.strip() + + +def test_a_queued_run_renders_the_handle_it_had_to_derive( + capsys, deployment_client, tmp_path +): + """The ack names no execution of its own -- the id is only in the endpoint + it hands back -- so raw would otherwise have nothing true to print.""" + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + deployment_client(structure_file=ACK) + + code, out, _ = run( + capsys, "docstudio", "deployment", "run", "my-api", str(doc), "--no-wait" + ) + assert code == int(ExitCode.SUCCESS) + assert envelope(out)["meta"]["execution_id"] == "e-1" + + deployment_client(structure_file=ACK) + assert ( + _raw(capsys, "docstudio", "deployment", "run", "my-api", str(doc), "--no-wait") + == "e-1" + ) + + +def test_a_still_running_status_never_renders_as_an_empty_result( + capsys, deployment_client +): + """`extraction_result` is present and null while the job runs. Printing that + tells a polling caller the same thing as a finished job with no output.""" + deployment_client(check_execution_status=PENDING_STATUS) + code, out, _ = run(capsys, "docstudio", "deployment", "status", "my-api", "e-1") + assert code == int(ExitCode.SUCCESS) + assert envelope(out)["data"]["execution_status"] == "EXECUTING" + + deployment_client(check_execution_status=PENDING_STATUS) + assert ( + _raw(capsys, "docstudio", "deployment", "status", "my-api", "e-1") == "EXECUTING" + ) + + +def test_a_finished_status_renders_its_result(capsys, deployment_client): + deployment_client(check_execution_status=DONE_STATUS) + code, out, _ = run(capsys, "docstudio", "deployment", "status", "my-api", "e-1") + assert envelope(out)["data"]["extraction_result"] == "the answer" + + deployment_client(check_execution_status=DONE_STATUS) + assert ( + _raw(capsys, "docstudio", "deployment", "status", "my-api", "e-1") == "the answer" + ) + + +def test_raw_fails_rather_than_printing_something_else(capsys, deployment_client): + """An answer carrying none of the declared fields has no raw form. Dumping + the whole payload answers a question the caller did not ask.""" + deployment_client(check_execution_status={"status_code": 200, "unexpected": 1}) + code = main(["-o", "raw", "docstudio", "deployment", "status", "my-api", "e-1"]) + out, err = capsys.readouterr() + assert code == int(ExitCode.GENERIC) + assert "unexpected" not in out + assert "extraction_result" in out or "extraction_result" in err + + +def test_an_error_status_from_a_run_is_a_failure(capsys, deployment_client, tmp_path): + """The client reports the status code instead of raising, so an error would + otherwise be reported as a successful run with an error inside it.""" + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + deployment_client( + structure_file={ + "status_code": 422, + "pending": False, + "execution_status": "ERROR", + "error": "no such API", + } + ) + + code, out, _ = run( + capsys, "docstudio", "deployment", "run", "my-api", str(doc), "--no-wait" + ) + assert code == int(ExitCode.VALIDATION) + assert envelope(out)["error"]["message"] == "no such API" + + +def test_deployment_status_reports_a_running_execution(capsys, deployment_client): + client = deployment_client( + check_execution_status={ + "status_code": 200, + "pending": True, + "execution_status": "EXECUTING", + } + ) + code, out, _ = run(capsys, "docstudio", "deployment", "status", "my-api", "e1") + assert code == int(ExitCode.SUCCESS) + assert envelope(out)["data"]["execution_status"] == "EXECUTING" + assert "execution_id=e1" in client.calls[0][1][0] + + +@pytest.mark.parametrize( + ("flag", "name", "value"), + [ + ("--include-metadata", "include_metadata", True), + ("--no-include-metadata", "include_metadata", False), + ("--include-metrics", "include_metrics", True), + ("--no-include-metrics", "include_metrics", False), + ("--include-extracted-text", "include_extracted_text", True), + ("--no-include-extracted-text", "include_extracted_text", False), + ], +) +def test_a_status_flag_reaches_the_client(capsys, deployment_client, flag, name, value): + """A derived flag that is collected and never forwarded is indistinguishable + from one that works: the command still succeeds and the payload still parses.""" + client = deployment_client( + check_execution_status={"status_code": 200, "execution_status": "COMPLETED"} + ) + run(capsys, "docstudio", "deployment", "status", flag, "my-api", "e1") + assert client.kwargs_for("check_execution_status")[name] is value + + +def test_status_sends_only_the_flags_that_were_given(capsys, deployment_client): + client = deployment_client( + check_execution_status={"status_code": 200, "execution_status": "COMPLETED"} + ) + run(capsys, "docstudio", "deployment", "status", "my-api", "e1") + assert client.kwargs_for("check_execution_status") == {} + + +def test_a_waited_run_reads_its_result_with_the_flags_it_was_given( + capsys, deployment_client, tmp_path +): + """Otherwise --wait silently returns less than the same flags return without + it: the run is asked for metrics and the read that fetches them is not.""" + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + client = deployment_client( + structure_file={ + "status_code": 200, + "pending": True, + "execution_status": "PENDING", + "status_check_api_endpoint": "/status?execution_id=e1", + }, + check_execution_status={ + "status_code": 200, + "pending": False, + "execution_status": "COMPLETED", + }, + ) + + run( + capsys, + "-q", + "docstudio", + "deployment", + "run", + "my-api", + str(doc), + "--interval", + "0", + "--include-metrics", + "--no-include-metadata", + ) + + polled = client.kwargs_for("check_execution_status") + assert polled["include_metrics"] is True + assert polled["include_metadata"] is False + # `tags` is a run-time parameter the status endpoint does not accept. + assert "tags" not in polled + + +def test_a_waited_run_reports_which_execution_it_was(capsys, deployment_client, tmp_path): + """The waited payload names the execution nowhere, so without this a caller + has no id to correlate the result against the service.""" + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + deployment_client( + structure_file={ + "status_code": 200, + "pending": True, + "execution_status": "PENDING", + "status_check_api_endpoint": "/status?execution_id=e1", + }, + check_execution_status={ + "status_code": 200, + "pending": False, + "execution_status": "COMPLETED", + }, + ) + + _, out, _ = run( + capsys, + "-q", + "docstudio", + "deployment", + "run", + "my-api", + str(doc), + "--interval", + "0", + ) + assert envelope(out)["meta"]["execution_id"] == "e1" + + +def test_a_run_only_parameter_is_not_forwarded_to_the_status_read( + capsys, deployment_client, tmp_path +): + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + client = deployment_client( + structure_file={ + "status_code": 200, + "pending": True, + "execution_status": "PENDING", + "status_check_api_endpoint": "/status?execution_id=e1", + }, + check_execution_status={ + "status_code": 200, + "pending": False, + "execution_status": "COMPLETED", + }, + ) + + run( + capsys, + "-q", + "docstudio", + "deployment", + "run", + "my-api", + str(doc), + "--interval", + "0", + "--tags", + "a,b", + ) + + assert client.kwargs_for("structure_file")["tags"] == "a,b" + assert client.kwargs_for("check_execution_status") == {} + + +# --------------------------------------------------------------------------- # +# The flag tier of flag > env > profile > default +# --------------------------------------------------------------------------- # + + +def test_a_connection_flag_beats_the_environment(capsys, monkeypatch, tmp_path): + monkeypatch.setenv("LLMWHISPERER_BASE_URL", "https://from-env.test") + monkeypatch.setenv("LLMWHISPERER_API_KEY", "env-key") + seen = {} + monkeypatch.setattr( + whisper_cmd, + "llmwhisperer", + lambda config: ( + seen.update( + base_url=config.get("llmwhisperer", "base_url"), + api_key=config.get("llmwhisperer", "api_key"), + ) + or FakeWhisper(get_usage_info={}) + ), + ) + + code, _, err = run( + capsys, + "whisper", + "--base-url", + "https://from-flag.test", + "--api-key", + "flag-key", + "usage", + ) + + assert code == int(ExitCode.SUCCESS) + assert seen == {"base_url": "https://from-flag.test", "api_key": "flag-key"} + # A key on the command line lands in shell history and the process list. + assert "shell history" in err + + +def test_the_environment_still_wins_over_a_profile(capsys, monkeypatch, write_config): + write_config( + """ + default_profile = "p" + [profiles.p.llmwhisperer] + base_url = "https://from-profile.test" + """ + ) + monkeypatch.setenv("LLMWHISPERER_BASE_URL", "https://from-env.test") + seen = {} + monkeypatch.setattr( + whisper_cmd, + "llmwhisperer", + lambda config: ( + seen.update(base_url=config.get("llmwhisperer", "base_url")) + or FakeWhisper(get_usage_info={}) + ), + ) + + run(capsys, "whisper", "usage") + assert seen == {"base_url": "https://from-env.test"} + + +def test_a_deployment_org_can_come_from_a_flag(capsys, monkeypatch): + monkeypatch.setenv("UNSTRACT_DEPLOYMENT_KEY", "key") + seen = {} + monkeypatch.setattr( + docstudio_cmd, + "deployment", + lambda config, target, transport_timeout=None: ( + seen.update(org=config.get("docstudio", "org_id")) or _deployment_fake() + ), + ) + run(capsys, "docstudio", "--org-id", "org_A", "deployment", "status", "api", "e1") + assert seen == {"org": "org_A"} + + +def _deployment_fake(): + client = FakeWhisper( + check_execution_status={"status_code": 200, "execution_status": "COMPLETED"} + ) + client.api_url = "https://api.example.com/deployment/api/org/api-name/" + return client + + +# --------------------------------------------------------------------------- # +# The one-shot data path +# --------------------------------------------------------------------------- # + + +def test_a_waited_extract_keeps_a_result_that_is_not_wrapped( + capsys, whisper_client, tmp_path +): + """A bare `.get("extraction")` returned None here and printed + `ok: true, data: null` for a document that had been processed and billed.""" + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + whisper_client( + whisper={"whisper_hash": "h1", "status_code": 202}, + whisper_status={"status": "processed"}, + # No `extraction` key -- the shape the sibling command already tolerated. + whisper_retrieve={"status_code": 200, "result_text": "THE REAL TEXT"}, + ) + + code, out, _ = run(capsys, "whisper", "extract", str(doc), "--interval", "0") + + assert code == int(ExitCode.SUCCESS) + assert envelope(out)["data"]["result_text"] == "THE REAL TEXT" + + +def test_a_waited_extract_calls_an_empty_result_a_failure( + capsys, whisper_client, tmp_path +): + """The read is acknowledged either way, so an empty result is a consumed + document with nothing to show for it.""" + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + whisper_client( + whisper={"whisper_hash": "h1", "status_code": 202}, + whisper_status={"status": "processed"}, + whisper_retrieve={"extraction": {}}, + ) + + code, out, _ = run(capsys, "whisper", "extract", str(doc), "--interval", "0") + + assert code == int(ExitCode.SERVER_ERROR) + assert envelope(out)["ok"] is False + + +def test_a_waited_extract_reads_the_result_when_it_is_not_wrapped( + capsys, whisper_client, tmp_path +): + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + whisper_client( + whisper={"whisper_hash": "h1", "status_code": 202}, + whisper_status={"status": "processed"}, + whisper_retrieve={"extraction": {"result_text": "hello"}}, + ) + + code, out, _ = run(capsys, "whisper", "extract", str(doc), "--interval", "0") + + assert code == int(ExitCode.SUCCESS) + assert envelope(out)["data"]["result_text"] == "hello" + + +def test_retrieve_writes_the_result_before_it_prints( + capsys, whisper_client, tmp_path, monkeypatch +): + """Ordering, not outcome: asserting after the command returns passes for + either order, which is how this went unnoticed.""" + order: list[str] = [] + target = tmp_path / "out" / "result.json" + whisper_client(whisper_retrieve={"extraction": {"result_text": "hello"}}) + + real_persist = whisper_cmd.persist + monkeypatch.setattr( + whisper_cmd, + "persist", + lambda path, payload: (order.append("persist"), real_persist(path, payload))[1], + ) + real_finish = whisper_cmd.finish + monkeypatch.setattr( + whisper_cmd, + "finish", + lambda *a, **kw: (order.append("finish"), real_finish(*a, **kw))[1], + ) + + run(capsys, "whisper", "retrieve", "h1", "--save", str(target)) + + assert order == ["persist", "finish"] + + +def test_retrieve_refuses_an_unwritable_target_before_reading( + capsys, whisper_client, tmp_path +): + """Nothing has been consumed yet at this point, so this failure is cheap -- + the same failure after the read is not recoverable at all.""" + blocker = tmp_path / "not-a-dir" + blocker.write_text("") + client = whisper_client(whisper_retrieve={"extraction": {"result_text": "hello"}}) + + code, out, _ = run( + capsys, "whisper", "retrieve", "h1", "--save", str(blocker / "r.json") + ) + + assert code == int(ExitCode.USAGE) + assert client.calls == [] + + +def test_a_save_failure_after_the_read_still_emits_the_result( + capsys, whisper_client, tmp_path, monkeypatch +): + target = tmp_path / "result.json" + whisper_client(whisper_retrieve={"extraction": {"result_text": "IRREPLACEABLE"}}) + + def explode(path, payload): + # What `persist` itself raises when the write fails: the payload rides + # out on the error because there is no other copy left. + raise CLIError( + "The result could not be written.", + ExitCode.SAVE_FAILED, + details=payload, + ) + + monkeypatch.setattr(whisper_cmd, "persist", explode) + + code, out, _ = run(capsys, "whisper", "retrieve", "h1", "--save", str(target)) + + assert code == int(ExitCode.SAVE_FAILED) + assert envelope(out)["error"]["details"]["result_text"] == "IRREPLACEABLE" + + +def test_a_failed_execution_inside_a_200_is_not_a_success(capsys, deployment_client): + deployment_client( + check_execution_status={ + "status_code": 200, + "pending": False, + "execution_status": "ERROR", + "error": "tool crashed", + } + ) + + code, out, _ = run(capsys, "docstudio", "deployment", "status", "api", "e1") + + assert code != int(ExitCode.SUCCESS) + assert envelope(out)["ok"] is False + + +def test_the_key_never_reaches_stdout_or_stderr(capsys, whisper_client, monkeypatch): + """Scrubbing is not a keyword argument a call site can forget.""" + key = "lw-live-ABCDEF0123456789" + monkeypatch.setenv("LLMWHISPERER_API_KEY", key) + whisper_client( + whisper_retrieve=LLMWhispererClientException( + {"message": f"invalid key {key}", "status_code": 401}, 401 + ) + ) + + code, out, err = run(capsys, "whisper", "retrieve", "h1") + + assert code == int(ExitCode.AUTH) + assert key not in out + assert key not in err + + +def test_clone_maps_its_flags_and_reports_a_partial_failure(capsys, monkeypatch): + """Migration flags decide what is copied where, with two admin keys in play.""" + captured: dict = {} + + def fake_clone(source, target, options): + captured.update(source=source, target=target, options=options) + return CloneReport( + source=Endpoint(source.base_url, source.organization_id), + target=Endpoint(target.base_url, target.organization_id), + phases=[ + PhaseResult(name="adapters", created=1, failed=2), + PhaseResult(name="files", created=1, skipped=3), + ], + oversize_files=[{"name": "big.pdf"}, {"name": "bigger.pdf"}], + ) + + monkeypatch.setattr(clone_cmd, "run_clone", fake_clone) + monkeypatch.setenv("UNSTRACT_SRC_PLATFORM_KEY", "src-key-0123456789") + monkeypatch.setenv("UNSTRACT_TGT_PLATFORM_KEY", "tgt-key-0123456789") + + code, out, err = run( + capsys, + "clone", + "--source-url", + "https://dev.example.com", + "--source-org", + "org_dev", + "--target-url", + "https://qa.example.com", + "--target-org", + "org_qa", + "--dry-run", + "--exclude", + "files, groups", + "--skip-files", + "--max-file-size", + "2MB", + "--api-prefix", + "api/v2", + "--on-name-conflict", + "abort", + ) + + assert captured["source"].platform_key == "src-key-0123456789" + assert captured["target"].organization_id == "org_qa" + assert captured["target"].api_path_prefix == "api/v2" + assert captured["options"].dry_run is True + assert captured["options"].exclude == ("files", "groups") + assert captured["options"].file_strategy == "skip" + assert captured["options"].max_file_size == 2 * 1024 * 1024 + # adopt and abort decide what is written into a live target organisation. + assert captured["options"].on_name_conflict == "abort" + + # A phase that failed is not a successful migration, whatever else worked. + assert code == int(ExitCode.GENERIC) + body = envelope(out) + assert body["ok"] is False + assert "adapters" in body["error"]["message"] + # Documents that never arrived are counted where a consumer reads first. + assert body["error"]["details"]["skipped"] == { + "total": 3, + "by_phase": {"files": 3}, + "oversize_files": 2, + "unsupported_files": 0, + } + for key in ("src-key-0123456789", "tgt-key-0123456789"): + assert key not in out and key not in err + + +def test_a_key_quoted_in_a_clone_report_does_not_survive_the_table(capsys, monkeypatch): + """The table is the output a person gets, and the report renders itself. + + A platform key quoted back by a failing service lands in a terminal buffer + and in whatever scrapes one, so the rendered report is scrubbed on the same + path as every envelope rather than by hand. + """ + key = "src-key-0123456789" + + def fake_clone(source, target, options): + return CloneReport( + source=Endpoint(source.base_url, source.organization_id), + target=Endpoint(target.base_url, target.organization_id), + phases=[PhaseResult(name="adapters", created=1)], + warnings=[f"target refused the request for {key}"], + ) + + monkeypatch.setattr(clone_cmd, "run_clone", fake_clone) + monkeypatch.setenv("UNSTRACT_SRC_PLATFORM_KEY", key) + monkeypatch.setenv("UNSTRACT_TGT_PLATFORM_KEY", "tgt-key-0123456789") + + code = main( + [ + "-o", + "table", + "clone", + "--source-url", + "https://dev.example.com", + "--source-org", + "org_dev", + "--target-url", + "https://qa.example.com", + "--target-org", + "org_qa", + ] + ) + captured = capsys.readouterr() + + assert code == int(ExitCode.SUCCESS) + assert "adapters" in captured.out + assert key not in captured.out and key not in captured.err diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..3190652 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,463 @@ +"""Config resolution: flag > env > profile > built-in default.""" + +from __future__ import annotations + +import os +import stat +from pathlib import Path + +import pytest + +from unstract_cli import config as config_module +from unstract_cli.config import ( + DEFAULT_BASE_URLS, + DOCSTUDIO, + LLMWHISPERER, + PROJECT_CONFIG_NAME, + ConfigError, + ConfigFile, + ResolvedConfig, + config_path, + find_project_config, + load_config, + save_config, + set_config_path, + starter_profiles, +) + +PROFILE_TOML = """ +default_profile = "p" + +[profiles.p.llmwhisperer] +base_url = "https://profile.example/api/v2" +api_key = "profile-key" + +[profiles.p.docstudio] +org_id = "org_from_profile" +api_key = "env:UNSTRACT_DEPLOYMENT_KEY" + +[profiles.p.deployments.invoices] +api_name = "invoice-parser" + +[profiles.p.deployments.receipts] +api_name = "receipt-parser" +org_id = "org_alias" +api_key = "alias-key" +""" + + +def resolved(overrides=None, profile=None): + return ResolvedConfig( + file=load_config(), profile_name=profile, overrides=overrides or {} + ) + + +def test_default_when_nothing_configured(): + assert resolved().get(LLMWHISPERER, "base_url") == DEFAULT_BASE_URLS[LLMWHISPERER] + assert resolved().get(LLMWHISPERER, "api_key") is None + + +def test_profile_beats_default(write_config): + write_config(PROFILE_TOML) + assert resolved().get(LLMWHISPERER, "base_url") == "https://profile.example/api/v2" + + +def test_env_beats_profile(write_config, monkeypatch): + write_config(PROFILE_TOML) + monkeypatch.setenv("LLMWHISPERER_BASE_URL", "https://env.example/api/v2") + assert resolved().get(LLMWHISPERER, "base_url") == "https://env.example/api/v2" + + +@pytest.mark.parametrize( + ("product", "key", "var", "value"), + [ + ( + LLMWHISPERER, + "base_url", + "LLMWHISPERER_BASE_URL_V2", + "https://staging.example/api/v2", + ), + (DOCSTUDIO, "api_key", "UNSTRACT_API_DEPLOYMENT_KEY", "deployment-key"), + ], +) +def test_the_env_names_the_clients_read_are_honoured( + monkeypatch, product, key, var, value +): + """An environment set up for the published client must not leave the CLI on + its built-in default, which points at production.""" + monkeypatch.setenv(var, value) + assert resolved().get(product, key) == value + + +def test_the_cli_s_own_env_name_wins_over_the_client_s(monkeypatch): + monkeypatch.setenv("LLMWHISPERER_BASE_URL", "https://first.example/api/v2") + monkeypatch.setenv("LLMWHISPERER_BASE_URL_V2", "https://second.example/api/v2") + assert resolved().get(LLMWHISPERER, "base_url") == "https://first.example/api/v2" + + +def test_override_beats_env(write_config, monkeypatch): + write_config(PROFILE_TOML) + monkeypatch.setenv("LLMWHISPERER_BASE_URL", "https://env.example/api/v2") + cfg = resolved(overrides={"llmwhisperer.base_url": "https://flag.example"}) + assert cfg.get(LLMWHISPERER, "base_url") == "https://flag.example" + + +def test_env_indirection_resolves_and_missing_var_reads_as_unset( + write_config, monkeypatch +): + write_config(PROFILE_TOML) + assert resolved().get(DOCSTUDIO, "api_key") is None + monkeypatch.setenv("UNSTRACT_DEPLOYMENT_KEY", "secret-value") + assert resolved().get(DOCSTUDIO, "api_key") == "secret-value" + + +def test_require_names_every_way_to_supply_the_setting(): + with pytest.raises(ConfigError) as excinfo: + resolved().require(DOCSTUDIO, "api_key") + message = str(excinfo.value) + assert "UNSTRACT_DEPLOYMENT_KEY" in message + assert "[profiles..docstudio]" in message + # Credentials get no flag, so none may be suggested. + assert "--api-key" not in message + + +def test_placeholder_is_not_a_value(write_config): + """`config init` writes `org_id = ""`, and that must not satisfy `require`.""" + write_config('default_profile = "p"\n\n[profiles.p.docstudio]\norg_id = ""\n') + assert resolved().get(DOCSTUDIO, "org_id") is None + assert resolved().resolution_source(DOCSTUDIO, "org_id")["resolved"] is False + with pytest.raises(ConfigError): + resolved().require(DOCSTUDIO, "org_id") + + +def test_starter_profile_org_id_does_not_satisfy_require(write_config): + path = write_config("") + save_config(ConfigFile(default_profile="cloud-us", profiles=starter_profiles()), path) + with pytest.raises(ConfigError): + resolved().require(DOCSTUDIO, "org_id") + + +def test_unknown_profile_is_an_error_not_a_silent_empty_block(write_config): + write_config(PROFILE_TOML) + with pytest.raises(ConfigError, match="not found"): + resolved(profile="nope").get(DOCSTUDIO, "org_id") + + +def test_profile_selected_by_env_var(write_config, monkeypatch): + write_config(PROFILE_TOML.replace('default_profile = "p"', "")) + monkeypatch.setenv("UNSTRACT_PROFILE", "p") + assert resolved().get(DOCSTUDIO, "org_id") == "org_from_profile" + + +def test_deployment_alias_falls_back_to_the_product_block(write_config, monkeypatch): + write_config(PROFILE_TOML) + monkeypatch.setenv("UNSTRACT_DEPLOYMENT_KEY", "secret-value") + alias = resolved().deployment("invoices") + assert alias == { + "api_name": "invoice-parser", + "org_id": "org_from_profile", + "api_key": "secret-value", + } + + +def test_deployment_alias_overrides_win(write_config): + write_config(PROFILE_TOML) + alias = resolved().deployment("receipts") + assert alias["org_id"] == "org_alias" + assert alias["api_key"] == "alias-key" + + +def test_unknown_deployment_alias_lists_the_known_ones(write_config): + write_config(PROFILE_TOML) + with pytest.raises(ConfigError, match="invoices, receipts"): + resolved().deployment("nope") + + +def test_resolution_source_reports_the_winner(write_config, monkeypatch): + write_config(PROFILE_TOML) + cfg = resolved() + assert ( + cfg.resolution_source(LLMWHISPERER, "base_url")["source"] == "profile (literal)" + ) + assert cfg.resolution_source(DOCSTUDIO, "base_url")["source"] == "built-in default" + assert cfg.resolution_source(DOCSTUDIO, "api_key") == { + "resolved": False, + "source": "profile -> env:UNSTRACT_DEPLOYMENT_KEY", + "detail": "$UNSTRACT_DEPLOYMENT_KEY is not set in this process's environment", + } + monkeypatch.setenv("LLMWHISPERER_API_KEY", "k") + assert resolved().resolution_source(LLMWHISPERER, "api_key") == { + "resolved": True, + "source": "env:LLMWHISPERER_API_KEY", + } + + +# --------------------------------------------------------------------------- # +# File discovery and writing +# --------------------------------------------------------------------------- # + + +def test_discovery_order(tmp_path, monkeypatch): + from unstract_cli import config as config_mod + + home_default = config_mod.HOME_CONFIG + assert config_path() == home_default + + project = tmp_path / "proj" / "nested" + project.mkdir(parents=True) + (tmp_path / "proj" / ".unstract.toml").touch() + monkeypatch.chdir(project) + assert config_path() == tmp_path / "proj" / ".unstract.toml" + + monkeypatch.setenv("UNSTRACT_CONFIG", str(tmp_path / "env.toml")) + assert config_path() == tmp_path / "env.toml" + + set_config_path(tmp_path / "flag.toml") + assert config_path() == tmp_path / "flag.toml" + + +def test_project_search_stops_at_home(tmp_path, monkeypatch): + home = tmp_path / "home" + work = home / "work" + work.mkdir(parents=True) + monkeypatch.setattr("pathlib.Path.home", lambda: home) + # Above $HOME, so it must not be picked up. + (tmp_path / ".unstract.toml").touch() + assert find_project_config(work) is None + + +def test_missing_file_is_not_an_error(): + cfg = load_config() + assert cfg.exists is False and cfg.profiles == {} + + +def test_saved_config_is_owner_only(tmp_path): + path = tmp_path / "nested" / "config.toml" + written = save_config( + ConfigFile(default_profile="cloud-us", profiles=starter_profiles()), path + ) + assert stat.S_IMODE(written.stat().st_mode) == 0o600 + assert load_config(written).default_profile == "cloud-us" + + +def test_an_existing_file_is_narrowed_before_the_secret_is_written(tmp_path, monkeypatch): + """The mode passed to `os.open` applies only on creation, so rewriting a + world-readable file would otherwise publish the new key while it is written.""" + path = tmp_path / "config.toml" + path.write_text("") + path.chmod(0o644) + + seen = [] + real = config_module.tomli_w.dump + monkeypatch.setattr( + config_module.tomli_w, + "dump", + lambda doc, fh: ( + seen.append(stat.S_IMODE(os.fstat(fh.fileno()).st_mode)), + real(doc, fh), + )[1], + ) + written = save_config(ConfigFile(profiles=starter_profiles()), path) + + assert seen == [0o600] + assert stat.S_IMODE(written.stat().st_mode) == 0o600 + + +def test_a_failed_write_leaves_the_previous_config_intact(tmp_path, monkeypatch): + """Truncating the real file first would trade a working config for an empty + one whenever anything after the truncate failed.""" + path = tmp_path / "config.toml" + path.write_text('default_profile = "keep"\n', encoding="utf-8") + + monkeypatch.setattr( + config_module.tomli_w, + "dump", + lambda doc, fh: (_ for _ in ()).throw(OSError("no space left on device")), + ) + with pytest.raises(OSError): + save_config(ConfigFile(profiles=starter_profiles()), path) + + assert path.read_text(encoding="utf-8") == 'default_profile = "keep"\n' + # And nothing half-written left behind next to it. + assert [p.name for p in tmp_path.iterdir()] == ["config.toml"] + + +def test_the_replacement_is_synced_before_it_is_renamed(tmp_path, monkeypatch): + """A rename that outruns its own bytes survives a crash while the content + does not, which turns a working config into an empty one.""" + path = tmp_path / "config.toml" + order: list[str] = [] + real_fsync, real_replace = os.fsync, os.replace + monkeypatch.setattr( + config_module.os, + "fsync", + lambda fd: (order.append("fsync"), real_fsync(fd))[1], + ) + monkeypatch.setattr( + config_module.os, + "replace", + lambda src, dst: (order.append("replace"), real_replace(src, dst))[1], + ) + + save_config(ConfigFile(profiles=starter_profiles()), path) + + assert order[: order.index("replace")] == ["fsync"] + # The last one is the directory, so the rename itself is on the disk too. + assert order[-1] == "fsync" + + +def test_an_unwritable_directory_is_reported_rather_than_raised(tmp_path): + """Replacing the file needs the directory, which overwriting it did not, so + the case says what is wrong instead of surfacing a bare PermissionError.""" + nested = tmp_path / "locked" + nested.mkdir() + path = nested / "config.toml" + path.write_text("", encoding="utf-8") + nested.chmod(0o500) + try: + with pytest.raises(ConfigError, match="not writable"): + save_config(ConfigFile(profiles=starter_profiles()), path) + finally: + nested.chmod(0o700) + + +def test_loose_permissions_warn_rather_than_fail(write_config): + path = write_config(PROFILE_TOML) + path.chmod(0o644) + assert any("readable by other users" in w for w in load_config().warnings) + + +#: What a repository could commit: a host of its own choosing, and a key. +PROJECT_TOML = """ +default_profile = "p" + +[profiles.p.llmwhisperer] +base_url = "https://elsewhere.example/api/v2" +api_key = "project-literal-key" + +[profiles.p.docstudio] +org_id = "org_from_project" + +[profiles.p.deployments.invoices] +api_name = "invoice-parser" +api_key = "alias-literal-key" +""" + + +def _plant_project_config(tmp_path, monkeypatch): + work = tmp_path / "checkout" + work.mkdir() + path = work / ".unstract.toml" + path.write_text(PROJECT_TOML, encoding="utf-8") + monkeypatch.chdir(work) + return path + + +def test_a_discovered_project_config_supplies_no_key_and_no_host(tmp_path, monkeypatch): + path = _plant_project_config(tmp_path, monkeypatch) + cfg = resolved() + + assert cfg.get(LLMWHISPERER, "base_url") == DEFAULT_BASE_URLS[LLMWHISPERER] + assert cfg.get(LLMWHISPERER, "api_key") is None + assert cfg.deployment("invoices")["api_key"] is None + # Everything the file is legitimately for still applies. + assert cfg.get(DOCSTUDIO, "org_id") == "org_from_project" + assert cfg.deployment("invoices")["api_name"] == "invoice-parser" + assert any(str(path) in w and "Ignoring" in w for w in cfg.file.warnings) + assert cfg.resolution_source(LLMWHISPERER, "api_key")["detail"] + + +def test_the_same_file_named_explicitly_is_honoured(tmp_path, monkeypatch): + path = _plant_project_config(tmp_path, monkeypatch) + monkeypatch.setenv("UNSTRACT_CONFIG", str(path)) + cfg = resolved() + + assert cfg.get(LLMWHISPERER, "base_url") == "https://elsewhere.example/api/v2" + assert cfg.get(LLMWHISPERER, "api_key") == "project-literal-key" + assert not any("Ignoring" in w for w in cfg.file.warnings) + + +def test_writing_back_a_project_config_keeps_the_keys_it_withheld(tmp_path, monkeypatch): + path = _plant_project_config(tmp_path, monkeypatch) + cfg = load_config() + cfg.profiles["p"]["docstudio"]["org_id"] = "org_edited" + save_config(cfg) + + monkeypatch.setenv("UNSTRACT_CONFIG", str(path)) + reloaded = load_config() + assert reloaded.profiles["p"]["docstudio"]["org_id"] == "org_edited" + assert reloaded.profiles["p"]["llmwhisperer"]["api_key"] == "project-literal-key" + assert reloaded.profiles["p"]["deployments"]["invoices"]["api_key"] == ( + "alias-literal-key" + ) + + +def test_withheld_keys_are_not_carried_into_a_file_the_user_names(tmp_path, monkeypatch): + _plant_project_config(tmp_path, monkeypatch) + elsewhere = tmp_path / "named.toml" + save_config(load_config(), elsewhere) + + monkeypatch.setenv("UNSTRACT_CONFIG", str(elsewhere)) + assert "api_key" not in load_config().profiles["p"]["llmwhisperer"] + + +def test_naming_the_discovered_file_does_not_make_it_trusted(tmp_path, monkeypatch): + path = _plant_project_config(tmp_path, monkeypatch) + work = path.parent + (work / "sub").mkdir() + (tmp_path / "link").symlink_to(work) + + # The outcome first: the flag is only the mechanism, withholding is the point. + cfg = ResolvedConfig(file=load_config(path)) + assert cfg.get(LLMWHISPERER, "api_key") is None + assert cfg.get(LLMWHISPERER, "base_url") == DEFAULT_BASE_URLS[LLMWHISPERER] + + # However the same file is spelled, it is the same file. + for spelling in ( + Path(PROJECT_CONFIG_NAME), + path, + work / "sub" / ".." / PROJECT_CONFIG_NAME, + tmp_path / "link" / PROJECT_CONFIG_NAME, + ): + assert load_config(spelling).is_project_local is True, spelling + + other = tmp_path / "elsewhere.toml" + other.write_text(PROJECT_TOML, encoding="utf-8") + assert load_config(other).is_project_local is False + + +def test_a_symlinked_project_candidate_is_not_discovered(tmp_path, monkeypatch): + work = tmp_path / "checkout" + work.mkdir() + victim = tmp_path / "victim.toml" + victim.write_text("keep = true\n", encoding="utf-8") + (work / ".unstract.toml").symlink_to(victim) + monkeypatch.chdir(work) + + assert find_project_config(work) is None + assert config_path() != work / ".unstract.toml" + + +def test_a_write_through_a_symlink_fails_without_touching_its_target(tmp_path): + victim = tmp_path / "victim.toml" + victim.write_text("keep = true\n", encoding="utf-8") + link = tmp_path / "config.toml" + link.symlink_to(victim) + + with pytest.raises(ConfigError, match="symlink"): + save_config(ConfigFile(profiles=starter_profiles()), link) + assert victim.read_text(encoding="utf-8") == "keep = true\n" + + +def test_a_withheld_alias_key_is_reported_against_the_alias(tmp_path, monkeypatch): + _plant_project_config(tmp_path, monkeypatch) + cfg = resolved() + assert cfg.withheld_detail("deployments", "invoices", "api_key") + assert cfg.withheld_detail("deployments", "invoices", "org_id") is None + + +def test_starter_profiles_hold_no_literal_secrets(): + for blocks in starter_profiles().values(): + for settings in blocks.values(): + key = settings.get("api_key") + assert key is None or key.startswith("env:") diff --git a/tests/test_contract.py b/tests/test_contract.py new file mode 100644 index 0000000..288bba3 --- /dev/null +++ b/tests/test_contract.py @@ -0,0 +1,145 @@ +"""What the CLI can reach of what the APIs offer. + +The vendored specs and the pinned clients move independently: a refreshed spec +can declare a parameter the published client has no argument for, and such a +parameter is dropped from the CLI rather than offered and then rejected at the +call. Dropping it silently is the failure mode this file exists to prevent -- +the gap is written down, so widening it is a decision someone makes on purpose. +""" + +from __future__ import annotations + +import inspect +import json +import os +from dataclasses import asdict +from pathlib import Path +from typing import Any + +import pytest +from unstract.api_deployments.client import APIDeploymentsClient +from unstract.llmwhisperer.client_v2 import LLMWhispererClientV2 + +from unstract_cli.core.params import derive_params, find_operation, operation_params + +#: (product, operationId, client method) per command that derives its flags, +#: with the spec parameters that method cannot accept -- parameters the client +#: owns rather than lacks. +COMMANDS = [ + ( + "llmwhisperer", + "extract", + LLMWhispererClientV2.whisper, + {"url_in_post"}, + ), + ("llmwhisperer", "highlights", LLMWhispererClientV2.get_highlight_data, set()), + ("docstudio", "execute", APIDeploymentsClient.structure_file, {"files"}), + ( + "docstudio", + "status", + APIDeploymentsClient.check_execution_status, + {"execution_id"}, + ), +] + + +@pytest.mark.parametrize( + ("product", "operation", "method", "unreachable"), + COMMANDS, + ids=[f"{p}:{o}" for p, o, _, _ in COMMANDS], +) +def test_the_parameters_no_command_can_reach_are_the_known_ones( + product, operation, method, unreachable +): + declared = {p.name for p in operation_params(product, operation)} + derived = {p.name for p in derive_params(product, operation, client_method=method)} + assert declared - derived == unreachable + assert derived <= declared + + +@pytest.mark.parametrize( + ("product", "operation", "method"), + [(p, o, m) for p, o, m, _ in COMMANDS], + ids=[f"{p}:{o}" for p, o, _, _ in COMMANDS], +) +def test_every_derived_flag_is_an_argument_the_client_accepts(product, operation, method): + """The check the CLI cannot make at runtime: a flag the client has no + parameter for raises TypeError at the call, after the document is read.""" + accepted = set(inspect.signature(method).parameters) + for param in derive_params(product, operation, client_method=method): + assert param.name in accepted + + +@pytest.mark.parametrize( + ("product", "operation", "method"), + [(p, o, m) for p, o, m, _ in COMMANDS], + ids=[f"{p}:{o}" for p, o, _, _ in COMMANDS], +) +def test_a_deprecated_spelling_does_not_become_a_second_flag(product, operation, method): + """Both spellings of a renamed parameter are declared and both are accepted + by the client, so nothing but the deprecation marks one of them wrong.""" + flags = [ + param.flag for param in derive_params(product, operation, client_method=method) + ] + assert len(flags) == len(set(flags)) + deprecated = { + p["name"] + for p in find_operation(product, operation).get("parameters", []) + if p.get("deprecated") + } + assert deprecated.isdisjoint( + param.name for param in derive_params(product, operation, client_method=method) + ) + + +#: The flags the specs derive today, written down rather than read from the +#: spec, so a parameter lost upstream fails here instead of vanishing quietly. +SNAPSHOT = Path(__file__).parent / "derived_flags.json" + +#: Refreshing the snapshot is a decision, not a side effect of running the suite. +REFRESH = "UNSTRACT_CLI_REFRESH_FLAG_SNAPSHOT" + + +def _derived_flags() -> dict[str, dict[str, Any]]: + """Every flag the specs derive, with the whole of what each one accepts. + + Names alone would let a spec narrow an enum, or change a type or a default, + without moving the snapshot -- and the CLI would start rejecting a value it + used to take, with nothing here to say so. + """ + return { + f"{product}:{operation}": { + # Choices as a list: JSON has no tuple, and the snapshot is compared + # against what a JSON reader gives back. + param.flag: {**asdict(param), "choices": list(param.choices)} + for param in sorted( + derive_params(product, operation, client_method=method), + key=lambda param: param.flag, + ) + } + for product, operation, method, _ in COMMANDS + } + + +def _changed(current: dict[str, Any], expected: dict[str, Any]) -> list[str]: + """The flags that moved, named. Comparing whole payloads reports neither.""" + return sorted( + f"{operation} {flag}" + for operation in current.keys() | expected.keys() + for flag in current.get(operation, {}).keys() | expected.get(operation, {}).keys() + if current.get(operation, {}).get(flag) != expected.get(operation, {}).get(flag) + ) + + +def test_the_derived_flags_are_the_ones_last_reviewed(): + current = _derived_flags() + if os.environ.get(REFRESH): + SNAPSHOT.write_text(json.dumps(current, indent=2) + "\n", encoding="utf-8") + expected = json.loads(SNAPSHOT.read_text(encoding="utf-8")) + assert current == expected, ( + "What the vendored specs derive has changed: " + f"{', '.join(_changed(current, expected))}. A flag that disappears here " + "disappears from the CLI, and a choice or a type that narrows here " + "rejects a value the CLI used to take. Review the difference, then " + f"refresh the snapshot with {REFRESH}=1." + ) diff --git a/tests/test_discover.py b/tests/test_discover.py new file mode 100644 index 0000000..67f4bfc --- /dev/null +++ b/tests/test_discover.py @@ -0,0 +1,227 @@ +"""`--discover`, and the live half of `config doctor`. + +Discovery is what an agent reads before it runs anything, so the tiers have to +stay cheap-then-detailed, and everything reported has to be read back from the +parser rather than described separately. +""" + +from __future__ import annotations + +import json + +import click +import pytest + +from unstract_cli.__main__ import main +from unstract_cli.commands import config_cmd +from unstract_cli.core.errors import CLIError, ExitCode +from unstract_cli.core.output import CONTRACT_VERSION + + +def run(capsys, *args): + code = main(["-o", "json", *args]) + out = capsys.readouterr().out + return code, json.loads(out)["data"] if out.strip() else None + + +def test_groups_names_the_products_and_stops_there(capsys): + """The cheap question stays cheap: no command list, no flags.""" + code, data = run(capsys, "--discover", "groups") + assert code == int(ExitCode.SUCCESS) + assert {g["name"] for g in data["groups"]} == {"config", "docstudio", "whisper"} + # A leaf listed among the groups is a group a consumer finds empty. + assert [c["name"] for c in data["commands"]] == ["clone"] + assert all(entry["help"] for entry in [*data["groups"], *data["commands"]]) + assert all("commands" not in entry for entry in data["groups"]) + + +def test_summary_lists_commands_without_their_flags(capsys): + _, data = run(capsys, "--discover", "summary") + whisper = data["commands"]["whisper"]["commands"] + assert "extract" in whisper + assert whisper["extract"]["help"] + assert "params" not in whisper["extract"] + + +def test_full_carries_enough_to_build_a_call(capsys): + _, data = run(capsys, "--discover", "full") + extract = data["commands"]["whisper"]["commands"]["extract"] + params = {p["name"]: p for p in extract["params"]} + + assert params["source"]["kind"] == "argument" and params["source"]["required"] + assert params["mode"]["choices"] == [ + "document_insights", + "excel", + "form", + "high_quality", + "low_cost", + "native_text", + "table", + ] + assert params["wait"]["flags"] == ["--wait", "--no-wait"] + assert params["interval"]["type"] == "float" + assert extract["raw_fields"] == ["result_text"] + + +def test_full_publishes_the_flags_that_are_not_on_the_command(capsys): + """The connection settings live on the group and the format on the root, so + a description of the leaves alone describes a call nobody can make.""" + _, data = run(capsys, "--discover", "full") + + root = {p["name"]: p for p in data["params"]} + assert "-o" in root["output"]["flags"] + assert "--profile" in root["profile"]["flags"] + + whisper = {p["name"]: p for p in data["commands"]["whisper"]["params"]} + assert "--api-key" in whisper["api_key"]["flags"] + assert "--base-url" in whisper["base_url"]["flags"] + assert "org_id" in {p["name"] for p in data["commands"]["docstudio"]["params"]} + + +def test_no_flag_publishes_a_default_it_does_not_have(capsys): + """Click marks "no default given" with a sentinel object, not None, and a + serialised sentinel reads as a value the caller could send back.""" + _, data = run(capsys, "--discover", "full") + + def defaults(node): + for param in node.get("params", []): + if "default" in param: + yield param["name"], param["default"] + for child in node.get("commands", {}).values(): + yield from defaults(child) + + published = list(defaults(data)) + assert published + for name, value in published: + assert not isinstance(value, str) or "Sentinel" not in value, name + assert not repr(value).startswith("<"), name + + +def test_an_on_off_flag_publishes_the_same_default_across_click_versions(): + """The one declaration this CLI uses most answers differently per version: + given no default, some report `False` and some their own sentinel. Neither + is what omitting the flag does, which is to send nothing.""" + from unstract_cli.core.discover import _param + + assert "default" not in _param(click.Option(["--x/--no-x"], default=None)) + # A default that was actually chosen still travels. + assert _param(click.Option(["--y/--no-y"], default=True))["default"] is True + # And a plain on-only flag keeps reporting the False it really defaults to. + assert _param(click.Option(["--z"], is_flag=True))["default"] is False + + +def test_a_bare_option_publishes_no_default(): + """The case the CLI's own flags do not cover: an option declared with no + default at all, which is what a derived required flag is.""" + from unstract_cli.core.discover import _param + + assert "default" not in _param(click.Option(["--bare"])) + + +@pytest.mark.parametrize("fmt", ["table", "raw", "json"]) +def test_discovery_answers_as_json_whatever_the_format_says(capsys, fmt): + """It is the machine-readable description; a wrapped table is not one.""" + assert main(["-o", fmt, "--discover", "summary"]) == int(ExitCode.SUCCESS) + assert json.loads(capsys.readouterr().out)["data"]["commands"] + + +def test_full_carries_the_exit_code_table(capsys): + """A caller branches on these; they are part of the contract, not prose.""" + _, data = run(capsys, "--discover", "full") + codes = {entry["name"]: entry["code"] for entry in data["exit_codes"]} + assert codes["already_consumed"] == int(ExitCode.ALREADY_CONSUMED) + assert codes["success"] == 0 + + +def test_full_publishes_how_to_consume_the_output(capsys): + """The compatibility bargain is only binding if the consumer can read it.""" + _, data = run(capsys, "--discover", "full") + contract = data["contract"] + assert contract["version"] == CONTRACT_VERSION + assert contract["envelope"] == ["ok", "data", "error", "meta"] + rules = " ".join(contract["rules"]).lower() + assert "-o json" in rules + assert "ignore fields you do not recognise" in rules + assert "contract_version" in rules + + +def test_discovery_needs_no_configuration(capsys, tmp_path, monkeypatch): + """It is how a caller finds out what to run, so it must work before anything + is set up.""" + monkeypatch.setenv("UNSTRACT_CONFIG", str(tmp_path / "nonexistent.toml")) + code, data = run(capsys, "--discover", "summary") + assert code == int(ExitCode.SUCCESS) and data["commands"] + + +def test_an_unknown_tier_is_a_usage_error(capsys): + code = main(["--discover", "sideways"]) + capsys.readouterr() + assert code == int(ExitCode.USAGE) + + +# --------------------------------------------------------------------------- # +# config doctor --probe +# --------------------------------------------------------------------------- # + + +@pytest.fixture +def probe_client(monkeypatch): + def install(reply=None): + class Fake: + def get_usage_info(self): + if isinstance(reply, Exception): + raise reply + return reply or {} + + monkeypatch.setattr(config_cmd, "llmwhisperer", lambda _config: Fake()) + + return install + + +def test_doctor_makes_no_call_without_probe(capsys, probe_client): + probe_client(CLIError("must not be called")) + code, data = run(capsys, "config", "doctor") + assert code == int(ExitCode.SUCCESS) + assert "probe" not in data + + +def test_probe_verifies_the_whisperer_key(capsys, probe_client): + probe_client({"quota": 1}) + _, data = run(capsys, "config", "doctor", "--probe") + assert data["probe"]["llmwhisperer"] == { + "checked": True, + "ok": True, + "detail": "The key was accepted by the usage endpoint.", + } + + +def test_a_rejected_key_reports_why(capsys, probe_client): + """A probe that failed exits non-zero: --probe is run from setup scripts, + and a script branches on the exit code, not on the payload.""" + probe_client(CLIError("bad key", ExitCode.AUTH)) + code = main(["-o", "json", "config", "doctor", "--probe"]) + report = json.loads(capsys.readouterr().out)["error"]["details"] + assert code == int(ExitCode.GENERIC) + entry = report["probe"]["llmwhisperer"] + assert entry == { + "checked": True, + "ok": False, + "detail": "bad key", + "exit_code": int(ExitCode.AUTH), + } + + +def test_the_deployment_probe_says_it_verified_nothing(capsys, probe_client, monkeypatch): + """The only deployment endpoint is an execution, so there is nothing + side-effect-free to call. Saying otherwise would be worse than not checking. + """ + probe_client({}) + monkeypatch.setenv("UNSTRACT_ORG_ID", "org_A") + monkeypatch.setenv("UNSTRACT_DEPLOYMENT_KEY", "key") + _, data = run(capsys, "config", "doctor", "--probe") + entry = data["probe"]["docstudio"] + # `ok` is null rather than true: a true beside `checked: false` is read as a + # live check that passed, which is the one thing this probe cannot claim. + assert entry["checked"] is False and entry["ok"] is None + assert entry["resolved"] is True + assert "NOT verified" in entry["detail"] diff --git a/tests/test_errors.py b/tests/test_errors.py new file mode 100644 index 0000000..98706ad --- /dev/null +++ b/tests/test_errors.py @@ -0,0 +1,134 @@ +"""The exit-code table, retry policy and redaction.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from unstract_cli.core.errors import ( + REDACTED, + ExitCode, + error_from_status, + exit_code_for_status, + hint_for, + is_retryable, + redact_headers, + redact_value, + scrub, + undeclared_status_error, +) + + +@pytest.mark.parametrize( + ("status", "expected"), + [ + # Only a failure ever reaches this map: a 2xx or an unfollowed 3xx here + # means something answered outside the contract, which is not success. + (200, ExitCode.GENERIC), + (302, ExitCode.GENERIC), + (400, ExitCode.VALIDATION), + (401, ExitCode.AUTH), + (403, ExitCode.AUTH), + (404, ExitCode.NOT_FOUND), + (406, ExitCode.ALREADY_CONSUMED), + (408, ExitCode.TIMEOUT), + (409, ExitCode.VALIDATION), + (418, ExitCode.GENERIC), + (422, ExitCode.VALIDATION), + (429, ExitCode.RATE_LIMITED), + (500, ExitCode.SERVER_ERROR), + (503, ExitCode.SERVER_ERROR), + ], +) +def test_status_to_exit_code(status, expected): + assert exit_code_for_status(status) is expected + + +def test_exit_codes_are_stable_integers(): + # A caller branches on these numbers, so they are an API, not an enum detail. + assert [int(c) for c in ExitCode] == [*range(11), 130] + assert int(ExitCode.ALREADY_CONSUMED) == 9 + assert int(ExitCode.SAVE_FAILED) == 10 + # 128 + SIGINT, which every shell and job runner already reads as + # "stopped", rather than the next number in this CLI's own sequence. + assert int(ExitCode.INTERRUPTED) == 130 + + +@pytest.mark.parametrize("status", [429, 500, 502, 503]) +def test_retryable(status): + assert is_retryable(status) + + +@pytest.mark.parametrize("status", [400, 401, 403, 404, 406, 409, 422]) +def test_not_retryable(status): + # A 4xx retry re-sends what the server already rejected, and for a one-shot + # read it can consume a result the first attempt already delivered. + assert not is_retryable(status) + + +def test_one_shot_status_carries_its_own_hint(): + assert "already retrieved" in hint_for(406) + assert "--save" in hint_for(406) + + +def test_error_from_status_fills_code_hint_and_retryability(): + err = error_from_status(429, "slow down", endpoint="POST /whisper") + assert err.exit_code is ExitCode.RATE_LIMITED + assert err.retryable is True + assert err.to_dict()["endpoint"] == "POST /whisper" + + +def test_undeclared_status_is_reported_verbatim_never_guessed(): + err = undeclared_status_error(418, {"detail": "teapot"}) + assert "Undeclared status 418" in err.message + assert "teapot" in err.message + assert err.to_dict()["details"] == {"detail": "teapot"} + + +def test_redact_headers(): + out = redact_headers( + { + "unstract-key": "abc", + "Authorization": "Bearer x", + "X-Api-Key": "y", + "Content-Type": "application/json", + } + ) + assert out["unstract-key"] == out["Authorization"] == out["X-Api-Key"] == REDACTED + assert out["Content-Type"] == "application/json" + + +def test_redact_value_walks_nested_payloads(): + out = redact_value({"a": {"api_key": "secret", "n": 1}, "b": [{"token": "t"}]}) + assert out == {"a": {"api_key": REDACTED, "n": 1}, "b": [{"token": REDACTED}]} + + +def test_scrub_ignores_short_values(): + # Redacting a 3-character "key" would mangle unrelated text. + assert scrub("the key is abc", ["abc"]) == "the key is abc" + assert scrub("the key is abcdefghij", ["abcdefghij"]) == f"the key is {REDACTED}" + + +def test_the_readme_table_lists_every_exit_code(): + """The README table is a copy of the enum, and the only one users read.""" + readme = (Path(__file__).resolve().parents[1] / "README.md").read_text() + documented = { + int(row.split("|")[1]) for row in readme.splitlines() if _is_code_row(row) + } + + assert documented == {int(code) for code in ExitCode} + + +def _is_code_row(row: str) -> bool: + cells = row.split("|") + return len(cells) > 2 and cells[1].strip().isdigit() + + +def test_the_rejected_key_hint_does_not_blame_the_organisation(): + """A key from another organisation cannot produce this: the resource is + resolved within its own organisation first, so that answers 404.""" + hint = hint_for(401) + assert "organisation" not in hint + assert "does not cover" in hint + assert "organisation" in hint_for(404) diff --git a/tests/test_output.py b/tests/test_output.py new file mode 100644 index 0000000..02f3c62 --- /dev/null +++ b/tests/test_output.py @@ -0,0 +1,140 @@ +"""The stdout envelope and its renderings.""" + +from __future__ import annotations + +import json + +from unstract_cli.core.errors import CLIError, ExitCode +from unstract_cli.core.output import ( + CONTRACT_VERSION, + AgentMode, + OutputFormat, + emit_error, + emit_result, + envelope, + render, + render_table, + resolve_format, +) + +ENVELOPE_KEYS = {"ok", "data", "error", "meta"} + + +def test_success_envelope_shape(): + env = envelope(data={"a": 1}, meta={"took": 2}) + assert set(env) == ENVELOPE_KEYS + assert env == { + "ok": True, + "data": {"a": 1}, + "error": None, + "meta": {"took": 2, "contract_version": CONTRACT_VERSION}, + } + + +def test_error_envelope_shape(): + err = CLIError("boom", ExitCode.AUTH, http_status=401, hint="check the key") + env = envelope(error=err.to_dict()) + assert set(env) == ENVELOPE_KEYS + assert env["ok"] is False and env["data"] is None + assert env["error"] == { + "code": "auth_error", + "message": "boom", + "exit_code": 3, + "retryable": False, + "http_status": 401, + "hint": "check the key", + } + + +def test_meta_defaults_to_an_object_not_null(): + # A caller reading meta. should not have to null-check the container. + assert envelope(data=1)["meta"] == {"contract_version": CONTRACT_VERSION} + + +def test_every_envelope_is_versioned(): + """A consumer cannot refuse a shape it was not written for without this.""" + for env in (envelope(data=1, meta={"job": "x"}), envelope(error={"code": "x"})): + assert env["meta"]["contract_version"] == CONTRACT_VERSION + + +def test_stdout_carries_the_envelope_on_success(capsys): + emit_result({"text": "hello"}, OutputFormat.JSON) + out = capsys.readouterr() + assert json.loads(out.out) == { + "ok": True, + "data": {"text": "hello"}, + "error": None, + "meta": {"contract_version": CONTRACT_VERSION}, + } + assert out.err == "" + + +def test_stdout_carries_the_envelope_on_failure_and_stderr_gets_a_summary(capsys): + code = emit_error(CLIError("nope", ExitCode.NOT_FOUND)) + out = capsys.readouterr() + parsed = json.loads(out.out) + assert parsed["ok"] is False and parsed["error"]["code"] == "not_found" + assert out.err.strip() == "error: nope" + assert code == ExitCode.NOT_FOUND + + +def test_secrets_are_scrubbed_from_both_streams(capsys): + secret = "sk-supersecret-value" + emit_error(CLIError(f"rejected token {secret}"), secrets=[secret]) + out = capsys.readouterr() + assert secret not in out.out and secret not in out.err + assert "***REDACTED***" in out.out + + +def test_table_and_raw_render_the_payload_not_the_envelope(): + env = envelope(data={"text": "hello"}) + assert "hello" in render(env, OutputFormat.TABLE) + assert "ok" not in render(env, OutputFormat.TABLE) + assert render(env, OutputFormat.RAW, raw_fields=("text",)) == "hello" + + +def test_raw_renders_the_error_when_the_run_failed(): + env = envelope(error=CLIError("boom").to_dict()) + assert "boom" in render(env, OutputFormat.RAW) + + +def test_table_wraps_long_cells_rather_than_truncating(): + long = "word " * 40 + rendered = render_table([{"text": long.strip()}], max_width=40) + assert rendered.count("\n") > 2 + assert "".join(rendered.split()).count("word") == 40 + + +def test_table_of_an_empty_list_says_so(): + assert render_table([]) == "(no results)" + + +class TestFormatSelection: + """Which rendering a run gets, and what is allowed to influence it.""" + + AGENT = {"CLAUDECODE": "1"} + + def test_the_default_is_a_table(self): + assert resolve_format(None, env={}) is OutputFormat.TABLE + + def test_an_agent_environment_moves_the_default_to_json(self): + for var in ("CLAUDECODE", "CURSOR_AGENT", "CODEX_SANDBOX", "AI_AGENT"): + assert resolve_format(None, env={var: "1"}) is OutputFormat.JSON + + def test_an_unset_marker_is_not_an_agent(self): + """An exported-but-empty variable is how a shell spells 'no'.""" + assert resolve_format(None, env={"CLAUDECODE": ""}) is OutputFormat.TABLE + + def test_an_explicit_format_beats_detection_in_both_directions(self): + assert resolve_format("table", env=self.AGENT) is OutputFormat.TABLE + assert resolve_format("json", env={}) is OutputFormat.JSON + + def test_the_agent_flag_overrides_what_the_environment_says(self): + assert resolve_format(None, AgentMode.NO, self.AGENT) is OutputFormat.TABLE + assert resolve_format(None, AgentMode.YES, {}) is OutputFormat.JSON + + def test_json_renders_the_same_bytes_wherever_it_is_asked_for(self): + env = envelope(data={"text": "hello"}) + one = render(env, resolve_format("json", AgentMode.NO, {})) + two = render(env, resolve_format("json", AgentMode.YES, self.AGENT)) + assert one == two diff --git a/tests/test_params.py b/tests/test_params.py new file mode 100644 index 0000000..ab540ca --- /dev/null +++ b/tests/test_params.py @@ -0,0 +1,279 @@ +"""Flag derivation: what the spec says, what the client accepts, what is sent. + +Each test here corresponds to a way derived flags can be wrong while still +looking right: a value silently dropped, a default silently pinned, a flag +offered that the client cannot accept. +""" + +from __future__ import annotations + +import click +import pytest +from unstract.api_deployments.client import APIDeploymentsClient +from unstract.llmwhisperer.client_v2 import LLMWhispererClientV2 + +from unstract_cli.core import params as params_module +from unstract_cli.core.params import ( + Param, + click_option, + derive_params, + docstring_params, + find_operation, + operation_params, + requested, +) + + +def _by_name(params: list[Param]) -> dict[str, Param]: + return {p.name: p for p in params} + + +# --------------------------------------------------------------------------- # +# Reading the spec +# --------------------------------------------------------------------------- # + + +def test_query_parameters_carry_type_and_default(): + params = _by_name(operation_params("llmwhisperer", "extract")) + assert params["mode"].type == "string" + assert params["add_line_nos"].type == "boolean" + assert params["median_filter_size"].type == "integer" + assert params["horizontal_stretch_factor"].default == 1.0 + + +def test_body_parameters_are_derived_too(): + """The deployment declares its parameters in a multipart body, not a query.""" + params = _by_name(operation_params("docstudio", "execute")) + assert params["tags"].type == "string" + assert params["timeout"].type == "integer" + assert params["presigned_urls"].array is True + # `null | string` in the spec: the null branch carries nothing for a flag. + assert params["llm_profile_id"].type == "string" + assert params["llm_profile_id"].nullable is True + + +def test_a_required_body_parameter_stays_required(): + params = _by_name(operation_params("llmwhisperer", "webhook_post")) + assert {p.name for p in params.values() if p.required} == { + "url", + "auth_token", + "webhook_name", + } + + +def test_the_uploaded_document_is_not_a_flag(): + """The binary body is the document itself, which the command takes as an + argument.""" + assert "body" not in _by_name(operation_params("llmwhisperer", "extract")) + assert find_operation("llmwhisperer", "extract")["method"] == "post" + + +def test_an_unknown_operation_names_itself(): + with pytest.raises(KeyError, match="whisper_sideways"): + find_operation("llmwhisperer", "whisper_sideways") + + +# --------------------------------------------------------------------------- # +# Intersecting the spec with the published client +# --------------------------------------------------------------------------- # + + +def test_only_parameters_the_client_accepts_become_flags(): + """A flag the client cannot accept raises TypeError at the call instead of + reaching the API, so it is not offered at all.""" + spec = set(_by_name(operation_params("llmwhisperer", "extract"))) + derived = set( + _by_name( + derive_params( + "llmwhisperer", "extract", client_method=LLMWhispererClientV2.whisper + ) + ) + ) + assert derived < spec + # In URL mode the URL travels in the body, and saying so is the client's + # decision, not a caller's. + assert spec - derived == {"url_in_post"} + + +def test_the_clients_default_wins_over_the_specs(): + """What a caller gets by omitting a flag is the client's default, since the + client sends its own value regardless of the spec's.""" + spec = _by_name(operation_params("llmwhisperer", "extract")) + derived = _by_name( + derive_params( + "llmwhisperer", "extract", client_method=LLMWhispererClientV2.whisper + ) + ) + assert spec["line_splitter_tolerance"].default == 0.75 + assert derived["line_splitter_tolerance"].default == 0.4 + + +def test_every_deployment_parameter_survives_the_intersection(): + derived = _by_name( + derive_params( + "docstudio", "execute", client_method=APIDeploymentsClient.structure_file + ) + ) + assert "tags" in derived and "hitl_queue_name" in derived + + +def test_excluded_parameters_do_not_become_flags(): + derived = _by_name( + derive_params( + "llmwhisperer", + "extract", + client_method=LLMWhispererClientV2.whisper, + exclude=("use_webhook",), + ) + ) + assert "use_webhook" not in derived + + +# --------------------------------------------------------------------------- # +# Help text +# --------------------------------------------------------------------------- # + + +def test_help_comes_from_the_clients_docstring(): + """The specs carry no parameter descriptions; the clients document every + parameter, so that is where the text comes from.""" + derived = _by_name( + derive_params( + "llmwhisperer", "extract", client_method=LLMWhispererClientV2.whisper + ) + ) + assert "language" in derived["lang"].description.lower() + + +def test_the_docstrings_own_restated_sentences_are_dropped(): + """The default and the allowed values are rendered from the signature and the + spec; printing the docstring's copies too shows each twice and disagrees the + moment either drifts.""" + described = docstring_params(LLMWhispererClientV2.whisper) + assert not described["lang"].endswith('Defaults to "eng".') + # A default that itself contains a period, which is where a sentence-shaped + # match stops early and leaves half of it behind. + assert not described["checkbox_confidence_threshold"].endswith("Defaults to 0.3.") + assert described["mode"] == "The processing mode." + assert described["tag"] == "The tag for the document." + + +def test_the_spec_wins_over_the_docstring_and_the_overlay_wins_over_both(monkeypatch): + """Three sources can describe one flag, and only the most specific should + show. Today no spec parameter carries a description, so the precedence is + unexercised until one does -- which is when it would silently invert.""" + described = Param("lang", "string", description="From the spec.") + monkeypatch.setattr(params_module, "operation_params", lambda *_: [described]) + + derived = derive_params( + "llmwhisperer", "extract", client_method=LLMWhispererClientV2.whisper + ) + assert derived[0].description == "From the spec." + assert click_option(derived[0], {}).help.startswith("From the spec.") + assert click_option(derived[0], {"lang": {"help": "From the overlay."}}).help == ( + "From the overlay." + ) + + +def test_a_multi_line_description_is_joined(): + text = docstring_params(LLMWhispererClientV2.whisper)["word_confidence_threshold"] + assert "\n" not in text and "confidence" in text + + +def test_the_default_is_reported_in_help(): + param = Param("mode", "string", default="form", description="The mode.") + assert click_option(param, {}).help == "The mode. [default: form]" + + +# --------------------------------------------------------------------------- # +# Building Click options +# --------------------------------------------------------------------------- # + + +def test_a_boolean_gets_a_paired_flag_defaulting_to_neither(): + """`is_flag` cannot turn off a parameter that defaults to on, and cannot + distinguish "not passed" from "passed false".""" + option = click_option(Param("allow_rotated_text", "boolean", default=True), {}) + assert option.secondary_opts == ["--no-allow-rotated-text"] + assert option.default is None + + +def test_no_option_carries_a_value_by_default(): + """A default written into the option would be sent on every call, pinning a + value the client or server would otherwise choose.""" + for param in derive_params( + "llmwhisperer", "extract", client_method=LLMWhispererClientV2.whisper + ): + assert click_option(param, {}).default is None + + +def test_choices_come_from_the_spec_unless_the_overlay_narrows_them(): + """A wrong value must fail before the request, not after -- and the list it + is checked against is the service's own, not a copy that can fall behind.""" + spec_declared = _by_name(operation_params("llmwhisperer", "extract"))["mode"] + assert "excel" in spec_declared.choices + assert click_option(spec_declared, {}).type.choices == spec_declared.choices + + option = click_option(spec_declared, {"mode": {"choices": ["form", "table"]}}) + assert isinstance(option.type, click.Choice) + assert option.type.choices == ("form", "table") + + +def test_an_array_becomes_a_repeatable_option(): + option = click_option(Param("presigned_urls", "string", array=True), {}) + assert option.multiple is True + + +def test_types_map_onto_click_types(): + assert click_option(Param("n", "integer"), {}).type is click.INT + assert click_option(Param("x", "number"), {}).type is click.FLOAT + assert click_option(Param("s", "string"), {}).type is click.STRING + + +def test_a_required_parameter_stays_required(): + assert click_option(Param("url", "string", required=True), {}).required is True + + +@pytest.mark.parametrize("type_name", ["string", "boolean"]) +def test_a_required_flag_carries_no_default_at_all(type_name): + """No command mounts a required derived flag today, so the parser check + below has nothing live to protect. This pins the property itself: Click + treats any default as a value the caller supplied.""" + option = click_option(Param("lines", type_name, required=True), {}) + bare = click.Option(["--bare"]) + assert option.default is bare.default + assert click_option(Param("lines", type_name), {}).default is None + + +@pytest.mark.parametrize("type_name", ["string", "boolean"]) +def test_a_required_flag_is_enforced_by_the_parser(type_name): + """From Click 8.2 a default counts as a value the caller supplied, so a + required option given one is never actually required.""" + option = click_option(Param("lines", type_name, required=True), {}) + command = click.Command("c", params=[option], callback=lambda **_: None) + with pytest.raises(click.MissingParameter): + command.make_context("c", []) + + +# --------------------------------------------------------------------------- # +# Choosing what to send +# --------------------------------------------------------------------------- # + + +def test_falsy_values_are_sent(): + """0, false and "" are choices. A truthiness filter eats them and hands the + decision back to the server without telling anyone.""" + assert requested({"a": 0, "b": False, "c": "", "d": 0.0}) == { + "a": 0, + "b": False, + "c": "", + "d": 0.0, + } + + +def test_unpassed_values_are_not_sent(): + assert requested({"a": None, "b": (), "c": 1}) == {"c": 1} + + +def test_dropped_names_are_not_sent(): + assert requested({"a": 1, "b": 2}, drop=("b",)) == {"a": 1} diff --git a/tests/test_poll.py b/tests/test_poll.py new file mode 100644 index 0000000..3e0e433 --- /dev/null +++ b/tests/test_poll.py @@ -0,0 +1,300 @@ +"""The `--wait` engine, driven by a fake clock and fake responses. No network.""" + +from __future__ import annotations + +import json + +import pytest + +from unstract_cli.core.errors import ExitCode +from unstract_cli.core.poll import ( + CLIError, + PollSpec, + extract_handle, + extract_status, + persist, + preflight, + wait_for_completion, +) + +SPEC = PollSpec( + handle_field="whisper_hash", + terminal_success=("processed",), + terminal_failure=("error",), + status_field=("status", "execution_status"), +) + + +class Clock: + """Monotonic clock that only advances when the engine sleeps.""" + + def __init__(self) -> None: + self.t = 0.0 + self.slept: list[float] = [] + + def now(self) -> float: + return self.t + + def sleep(self, seconds: float) -> None: + self.slept.append(seconds) + self.t += seconds + + +def responses(*payloads): + """A poll callable returning each payload in turn, then repeating the last.""" + queue = list(payloads) + calls: list[str] = [] + + def poll(handle: str): + calls.append(handle) + return queue.pop(0) if len(queue) > 1 else queue[0] + + poll.calls = calls + return poll + + +def test_polls_until_terminal_success(): + clock = Clock() + poll = responses( + {"status": "processing"}, + {"status": "processing"}, + {"status": "processed", "n": 1}, + ) + out = wait_for_completion( + initial={"whisper_hash": "h1"}, + spec=SPEC, + poll=poll, + interval=3, + sleep=clock.sleep, + now=clock.now, + ) + assert out == {"status": "processed", "n": 1} + assert poll.calls == ["h1", "h1", "h1"] + assert clock.slept == [3, 3] + + +def test_terminal_state_comes_from_the_body_not_the_http_status(): + # The deployment API returns HTTP 422 while still executing; only the body's + # status decides, so this reaches COMPLETED without any status-code input. + spec = PollSpec( + handle_field="execution_id", + terminal_success=("COMPLETED",), + terminal_failure=("ERROR",), + status_field=("status", "execution_status"), + ) + clock = Clock() + out = wait_for_completion( + initial={"message": {"execution_id": "e1", "execution_status": "PENDING"}}, + spec=spec, + poll=responses({"status": "EXECUTING"}, {"status": "COMPLETED"}), + sleep=clock.sleep, + now=clock.now, + ) + assert out == {"status": "COMPLETED"} + + +def test_terminal_failure_raises_with_the_handle_attached(): + clock = Clock() + with pytest.raises(CLIError) as excinfo: + wait_for_completion( + initial={"whisper_hash": "h1"}, + spec=SPEC, + poll=responses({"status": "error", "detail": "bad page"}), + sleep=clock.sleep, + now=clock.now, + ) + err = excinfo.value + assert err.exit_code is ExitCode.VALIDATION + assert err.to_dict()["whisper_hash"] == "h1" + assert err.to_dict()["details"]["detail"] == "bad page" + + +def test_timeout_carries_the_handle_so_work_is_resumable(): + clock = Clock() + with pytest.raises(CLIError) as excinfo: + wait_for_completion( + initial={"whisper_hash": "h1"}, + spec=SPEC, + poll=responses({"status": "processing"}), + interval=5, + timeout=12, + sleep=clock.sleep, + now=clock.now, + ) + err = excinfo.value + assert err.exit_code is ExitCode.TIMEOUT + payload = err.to_dict() + assert payload["whisper_hash"] == "h1" + assert payload["last_status"] == "processing" + assert "Resume" in payload["hint"] + # The job is still running, so this is the failure a caller is meant to + # come back from rather than the one that ends the attempt. + assert payload["retryable"] is True + # The last sleep is clipped so the wait lasts exactly as long as asked. + assert clock.slept == [5, 5, 2] + assert clock.now() == 12 + + +def test_missing_handle_returns_the_initial_response_unpolled(): + poll = responses({"status": "processed"}) + out = wait_for_completion( + initial={"no_handle_here": True}, spec=SPEC, poll=poll, sleep=Clock().sleep + ) + assert out == {"no_handle_here": True} + assert poll.calls == [] + + +def test_status_changes_are_reported_once_each(): + clock = Clock() + seen: list[str | None] = [] + wait_for_completion( + initial={"whisper_hash": "h1"}, + spec=SPEC, + poll=responses( + {"status": "accepted"}, + {"status": "processing"}, + {"status": "processing"}, + {"status": "processed"}, + ), + on_status=seen.append, + sleep=clock.sleep, + now=clock.now, + ) + assert seen == ["accepted", "processing", "processed"] + + +def test_retrieve_step_runs_after_terminal_success(): + clock = Clock() + out = wait_for_completion( + initial={"whisper_hash": "h1"}, + spec=SPEC, + poll=responses({"status": "processed"}), + retrieve=lambda handle: {"result_for": handle}, + sleep=clock.sleep, + now=clock.now, + ) + assert out == {"result_for": "h1"} + + +def test_save_persists_the_retrieved_result_before_returning(tmp_path): + target = tmp_path / "out" / "result.json" + on_disk: list[bool] = [] + + def retrieve(handle): + return {"text": "extracted"} + + out = wait_for_completion( + initial={"whisper_hash": "h1"}, + spec=SPEC, + poll=responses({"status": "processed"}), + retrieve=retrieve, + save=target, + # Observed from inside the engine, before the caller is handed anything: + # asserting after the return passes for either ordering. + on_saved=lambda path: on_disk.append(path.exists()), + sleep=Clock().sleep, + ) + assert on_disk == [True] + assert json.loads(target.read_text()) == out + + +def test_an_unwritable_save_target_is_refused_before_anything_is_read(tmp_path): + blocker = tmp_path / "not-a-dir" + blocker.write_text("") + + with pytest.raises(CLIError) as caught: + preflight(blocker / "result.json") + + assert caught.value.exit_code is ExitCode.USAGE + assert "nothing is lost" in (caught.value.hint or "") + + +def test_a_failed_save_carries_the_result_it_could_not_write(tmp_path): + """By this point the service has served the result and will not again, so + the payload has to leave through the error.""" + blocker = tmp_path / "not-a-dir" + blocker.write_text("") + + with pytest.raises(CLIError) as caught: + persist(blocker / "result.json", {"result_text": "IRREPLACEABLE"}) + + assert caught.value.exit_code is ExitCode.SAVE_FAILED + assert caught.value.details == {"result_text": "IRREPLACEABLE"} + + +def test_a_save_leaves_no_temporary_file_behind(tmp_path): + target = persist(tmp_path / "out.json", {"a": 1}) + assert [p.name for p in tmp_path.iterdir()] == [target.name] + + +def test_a_planted_temporary_file_is_not_written_through(tmp_path): + """A save directory another user can write is a directory they can plant a + symlink in, and following it would truncate whatever it points at.""" + victim = tmp_path / "victim" + victim.write_text("do not touch") + (tmp_path / "out.json.tmp").symlink_to(victim) + + persist(tmp_path / "out.json", {"a": 1}) + + assert victim.read_text() == "do not touch" + + +def test_a_symlinked_save_target_is_refused_before_anything_is_read(tmp_path): + """Saving over the link would turn it into a regular file and leave what it + stood for behind, so it is rejected while the result can still be re-read.""" + real = tmp_path / "results.json" + real.write_text("previous") + link = tmp_path / "latest.json" + link.symlink_to(real) + + with pytest.raises(CLIError) as caught: + preflight(link) + + assert caught.value.exit_code is ExitCode.USAGE + assert "symlink" in str(caught.value) + assert link.is_symlink() + assert real.read_text() == "previous" + + +def test_a_target_that_became_a_symlink_is_not_replaced(tmp_path): + """The preflight cannot hold the path for the length of the request, so the + rename checks again instead of destroying a link planted in between.""" + real = tmp_path / "results.json" + real.write_text("previous") + link = tmp_path / "latest.json" + link.symlink_to(real) + + with pytest.raises(CLIError) as caught: + persist(link, {"result_text": "IRREPLACEABLE"}) + + assert caught.value.exit_code is ExitCode.SAVE_FAILED + assert caught.value.details == {"result_text": "IRREPLACEABLE"} + assert link.is_symlink() + assert real.read_text() == "previous" + assert sorted(p.name for p in tmp_path.iterdir()) == [ + "latest.json", + "results.json", + ] + + +def test_persist_writes_text_payloads_unwrapped(tmp_path): + target = persist(tmp_path / "a.txt", "plain extracted text") + assert target.read_text() == "plain extracted text" + + +@pytest.mark.parametrize( + "payload", + [ + {"status": "processed"}, + {"message": {"status": "processed"}}, + {"data": {"status": "processed"}}, + {"result": {"status": "processed"}}, + ], +) +def test_status_is_found_one_level_into_the_common_envelopes(payload): + assert extract_status(payload) == "processed" + + +def test_handle_is_found_one_level_in_too(): + assert extract_handle({"message": {"execution_id": "e1"}}, "execution_id") == "e1" + assert extract_handle({"nothing": 1}, "execution_id") is None diff --git a/tests/test_specs.py b/tests/test_specs.py new file mode 100644 index 0000000..1537b42 --- /dev/null +++ b/tests/test_specs.py @@ -0,0 +1,29 @@ +"""The vendored specs are the ones the pinned clients were generated from. + +A spec copied from anywhere else derives flags the released client cannot +carry, and the failure surfaces at the call rather than here. +""" + +from __future__ import annotations + +import hashlib +import json +from importlib import resources + +import pytest + +from unstract_cli.core.params import SPEC_FILES + +PROVENANCE = json.loads( + (resources.files("unstract_cli.specs") / "provenance.json").read_text("utf-8") +) + + +@pytest.mark.parametrize("filename", sorted(SPEC_FILES.values())) +def test_each_vendored_spec_is_the_pinned_one(filename): + blob = (resources.files("unstract_cli.specs") / filename).read_bytes() + assert hashlib.sha256(blob).hexdigest() == PROVENANCE[filename]["sha256"] + + +def test_every_vendored_spec_has_a_provenance_entry(): + assert set(PROVENANCE) == set(SPEC_FILES.values()) diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..a1ea6a2 --- /dev/null +++ b/uv.lock @@ -0,0 +1,383 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" + +[[package]] +name = "anyio" +version = "4.14.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + +[[package]] +name = "certifi" +version = "2026.7.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/2a/23f34ec9d04624958e137efdc394888716353190e75f25dd22c7a2c7a8aa/charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b", size = 152439, upload-time = "2026-07-07T14:34:58.454Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/70/4a/ecbd131485c07fcdfad54e28946d513e3da22ef3b4bd854dcafae54ec739/charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0", size = 319300, upload-time = "2026-07-07T14:33:15.666Z" }, + { url = "https://files.pythonhosted.org/packages/ec/96/5d9364e3342d69f3a045e1777bc47c85c383e6e9466d561b33fdb419d1f9/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9", size = 215802, upload-time = "2026-07-07T14:33:17.031Z" }, + { url = "https://files.pythonhosted.org/packages/4b/4c/5361f9aa7f2cb58d94f2ab831b3d493f69efb1d239654b4744e3c09527cb/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44", size = 237171, upload-time = "2026-07-07T14:33:18.576Z" }, + { url = "https://files.pythonhosted.org/packages/50/78/ce342ca4ff30b2eb49fe6d9578df85974f90c67d294113e94efdd9664cbd/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9", size = 233075, upload-time = "2026-07-07T14:33:20.084Z" }, + { url = "https://files.pythonhosted.org/packages/01/c4/4fa4c8b3097a11f3c5f09a35b72ed6855fb1d332469504962ab7bafcc702/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd", size = 224256, upload-time = "2026-07-07T14:33:21.747Z" }, + { url = "https://files.pythonhosted.org/packages/87/3a/ad914516df7e358a81aae018caa5e0470ba827fa6d763b1d2e87d920a5f6/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84", size = 208784, upload-time = "2026-07-07T14:33:23.313Z" }, + { url = "https://files.pythonhosted.org/packages/d7/74/3c12f9755717dfe5c5c87da63f35d765fa0c00382ec26bf23f7fae34f2ba/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b", size = 219928, upload-time = "2026-07-07T14:33:24.814Z" }, + { url = "https://files.pythonhosted.org/packages/33/9a/895095b83e7907abd6d3d99aad3a38ad0d9686cc186cb0c94c24320fe63e/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde", size = 218489, upload-time = "2026-07-07T14:33:26.42Z" }, + { url = "https://files.pythonhosted.org/packages/a1/34/ef5c05f412f42520d7709b7d3784d19640839eb7366ded1755511585429f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39", size = 210267, upload-time = "2026-07-07T14:33:27.952Z" }, + { url = "https://files.pythonhosted.org/packages/83/dc/9b29fa4412b318bf3bfea985c35d67eb55e04b59a7c3f2237168b0e0be6f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62", size = 226030, upload-time = "2026-07-07T14:33:29.397Z" }, + { url = "https://files.pythonhosted.org/packages/0e/42/6dbc00b8cd16011691203e33570fa42ed5746599a2e878112d16eab403a3/charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642", size = 151185, upload-time = "2026-07-07T14:33:30.781Z" }, + { url = "https://files.pythonhosted.org/packages/80/cc/f920afd1a23c58ccd53c1d36085a71893a4737ff5e66e0371efab6809850/charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0", size = 162557, upload-time = "2026-07-07T14:33:32.176Z" }, + { url = "https://files.pythonhosted.org/packages/f0/e6/0386d43a261ff4e4b30c5857af7df877254b46bec7b9d1b74b6bf969a90b/charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2", size = 152665, upload-time = "2026-07-07T14:33:33.711Z" }, + { url = "https://files.pythonhosted.org/packages/b2/06/97ec2aeae780b31d742b6352218b43841a6871e2564578ca522dce4a45c3/charset_normalizer-3.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614", size = 317688, upload-time = "2026-07-07T14:33:35.408Z" }, + { url = "https://files.pythonhosted.org/packages/d0/39/8ff066c672434225f8d25f8b739f992af250944392173dcc88362681c9bf/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698", size = 214982, upload-time = "2026-07-07T14:33:36.996Z" }, + { url = "https://files.pythonhosted.org/packages/92/8f/3a47a3667c83c2df9483d91644c6c107de3bf8874aa1793da9d3012eb986/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b", size = 236460, upload-time = "2026-07-07T14:33:38.536Z" }, + { url = "https://files.pythonhosted.org/packages/f1/60/b22cdbee7e4013dab8b0d7647fc6181120fbbbc8f7025c226d15bd5a47fc/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9", size = 232003, upload-time = "2026-07-07T14:33:40.059Z" }, + { url = "https://files.pythonhosted.org/packages/ea/f8/72eb13dcabe7257035cea8aefd922caad2f110d252bf9f67c4c2ca763aee/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33", size = 223149, upload-time = "2026-07-07T14:33:41.631Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3e/faee8f9de92b14ee1198e9163252bb15efee7301b31256a3b6d9ebfdd0dd/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63", size = 207901, upload-time = "2026-07-07T14:33:43.209Z" }, + { url = "https://files.pythonhosted.org/packages/3a/25/45f30093ae27dd7b92a793b61882a38685f993700113ca36e0c9c14965e1/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0", size = 219176, upload-time = "2026-07-07T14:33:44.725Z" }, + { url = "https://files.pythonhosted.org/packages/48/18/c8f397329c35e32f6a837e488986f4ae03bd2abebc453b48714991630c2f/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe", size = 217356, upload-time = "2026-07-07T14:33:46.192Z" }, + { url = "https://files.pythonhosted.org/packages/86/7e/5ce0bba863470fd1902d5e5843968951bddf38abe4742fc97116ef4598b3/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35", size = 209614, upload-time = "2026-07-07T14:33:47.705Z" }, + { url = "https://files.pythonhosted.org/packages/6c/ef/2473d3c4d869155be4af1191111d59c4d5c4e0173026f7e85b176e23bf65/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8", size = 224991, upload-time = "2026-07-07T14:33:49.238Z" }, + { url = "https://files.pythonhosted.org/packages/d0/a3/53ddae3db108a088156aa8ddfafd411ebbc1340f48c5573f697b27f69a39/charset_normalizer-3.4.9-cp313-cp313-win32.whl", hash = "sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9", size = 150622, upload-time = "2026-07-07T14:33:50.711Z" }, + { url = "https://files.pythonhosted.org/packages/e8/ef/6953a77c7cf2c2ff9998e6f575ab3e380119f100223381565a4f94c1f836/charset_normalizer-3.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115", size = 161947, upload-time = "2026-07-07T14:33:52.197Z" }, + { url = "https://files.pythonhosted.org/packages/6e/fb/d560d1d1555debbfe7849d9cac6145c1b537709d79576bf22557ed803b82/charset_normalizer-3.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012", size = 152594, upload-time = "2026-07-07T14:33:53.486Z" }, + { url = "https://files.pythonhosted.org/packages/7e/8d/496817fa0944239ecae662dd57ea765cfeaec6a735f9f025d4b7b72e7143/charset_normalizer-3.4.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380", size = 317253, upload-time = "2026-07-07T14:33:54.994Z" }, + { url = "https://files.pythonhosted.org/packages/2b/f9/ef4a69ea338ad3c0deceea0f5f7d2380ae8b52132b06d652cb0d2cd86706/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9", size = 215898, upload-time = "2026-07-07T14:33:56.334Z" }, + { url = "https://files.pythonhosted.org/packages/8c/e7/5ddfd76fc061eb52de219658a4aa431cbacadf0a0219c8854f00da50d289/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4", size = 236718, upload-time = "2026-07-07T14:33:57.9Z" }, + { url = "https://files.pythonhosted.org/packages/49/ba/768fa3f36048d81c477a0ce61f813bc1454d80917ccfe550abd9f44f5e24/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a", size = 232519, upload-time = "2026-07-07T14:33:59.811Z" }, + { url = "https://files.pythonhosted.org/packages/f4/c4/b3e049d2aa3766180c78507110543d9d50894cc97f57de543f1be521dcdc/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046", size = 223143, upload-time = "2026-07-07T14:34:01.517Z" }, + { url = "https://files.pythonhosted.org/packages/19/79/55c32d06d76ae4feafe053f061f3e3ab70bcf19f4007797ce8c3efda7830/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81", size = 206742, upload-time = "2026-07-07T14:34:03.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/e0/47c079dd82d217c807479cd59ffd30af56307ea31c108b75758970459ad3/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917", size = 219191, upload-time = "2026-07-07T14:34:04.657Z" }, + { url = "https://files.pythonhosted.org/packages/42/ab/b9bc2e77d6b44a7e46ef62ec5cac1c9a6ba7b9135a5d560f002696ec9995/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41", size = 218328, upload-time = "2026-07-07T14:34:06.115Z" }, + { url = "https://files.pythonhosted.org/packages/f1/78/c9c71d599f5aa2d42bcdd35cbbd46d7f535351a57e40ff7d8e5a7e219401/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1", size = 207406, upload-time = "2026-07-07T14:34:07.554Z" }, + { url = "https://files.pythonhosted.org/packages/f6/39/c914445c321a845097ce4f6ac7de9a18228a77b766272125a1ce00d851eb/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf", size = 225157, upload-time = "2026-07-07T14:34:09.061Z" }, + { url = "https://files.pythonhosted.org/packages/9b/f2/c0d4b8508565a36bc5c624e88ed297f5b0b1095011034d7f5b83a69908b5/charset_normalizer-3.4.9-cp314-cp314-win32.whl", hash = "sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48", size = 151095, upload-time = "2026-07-07T14:34:10.901Z" }, + { url = "https://files.pythonhosted.org/packages/49/fd/a1d26144398c67486422a72bf5812cda22cb4ccfcd95a290fb41ceb4b8e2/charset_normalizer-3.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b", size = 162796, upload-time = "2026-07-07T14:34:12.47Z" }, + { url = "https://files.pythonhosted.org/packages/20/95/d75e82f8ce9fd323ebf059c16c9aadefb22a1ecde13b7840b35835e4886c/charset_normalizer-3.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519", size = 153334, upload-time = "2026-07-07T14:34:14.044Z" }, + { url = "https://files.pythonhosted.org/packages/00/5e/17398df3a139985ba9d11ed072531986f408c8fca952835ef1ab1820c02b/charset_normalizer-3.4.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198", size = 338848, upload-time = "2026-07-07T14:34:15.688Z" }, + { url = "https://files.pythonhosted.org/packages/cd/91/7253a32e86b7e1d1239b1b36ba6dd0f021a21107ab33054b53119cc083b9/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32", size = 223022, upload-time = "2026-07-07T14:34:17.248Z" }, + { url = "https://files.pythonhosted.org/packages/cb/32/2e64bd2be10e89c61e57ebe6a93fd98ae88eb7ebe414b5121f22c96c69eb/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632", size = 241590, upload-time = "2026-07-07T14:34:18.813Z" }, + { url = "https://files.pythonhosted.org/packages/3d/ef/d96ec496cfea0c21db43b0ad03891308b02388d054cc902cf0e5a1ad6a88/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf", size = 239584, upload-time = "2026-07-07T14:34:20.52Z" }, + { url = "https://files.pythonhosted.org/packages/d4/ce/9af95f7876194bd7a14e3dfe4a4de2e0bff02666a3910d72beafd06cc297/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990", size = 230224, upload-time = "2026-07-07T14:34:22.189Z" }, + { url = "https://files.pythonhosted.org/packages/52/94/af74dde74a3996bd959c350709bfe50e297823d70a8c1cbd54b838880863/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d", size = 212667, upload-time = "2026-07-07T14:34:23.857Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f0/f1c4fe746c395922961b5916ed1d7d6e7d4c84851d19ed43cc89980ec953/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e", size = 227179, upload-time = "2026-07-07T14:34:25.586Z" }, + { url = "https://files.pythonhosted.org/packages/e4/56/6c745619ac397e8871e2bcd3cea1eec86b877488f33888b3aef5c3ed506e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c", size = 225372, upload-time = "2026-07-07T14:34:27.212Z" }, + { url = "https://files.pythonhosted.org/packages/78/ad/98aae8630ac71f16711968e38a5acfecce41b778bf2f0312851020f565a8/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2", size = 215222, upload-time = "2026-07-07T14:34:28.774Z" }, + { url = "https://files.pythonhosted.org/packages/f7/40/9593d54209765207a7f11073c06494c1721e4ca4a0a426c597679bf7f91e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534", size = 231958, upload-time = "2026-07-07T14:34:30.345Z" }, + { url = "https://files.pythonhosted.org/packages/b1/27/693ee5e8a18191eb38647360c51cd505013e2bd3b366aa43fd5344c21e3c/charset_normalizer-3.4.9-cp314-cp314t-win32.whl", hash = "sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226", size = 155580, upload-time = "2026-07-07T14:34:31.884Z" }, + { url = "https://files.pythonhosted.org/packages/80/3f/bd97d3d9c613013d07cb7733d299385b41df37f0471310f5a73dc359f0b8/charset_normalizer-3.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177", size = 167620, upload-time = "2026-07-07T14:34:33.438Z" }, + { url = "https://files.pythonhosted.org/packages/3d/c6/eee9dca4439b1061f76373f06ea855678cc4a64c1c3c90b50e479edbb8eb/charset_normalizer-3.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501", size = 158037, upload-time = "2026-07-07T14:34:35.018Z" }, + { url = "https://files.pythonhosted.org/packages/98/2b/f97f1c193fb855c345d678f5077d6926034db0722df74c8f057020e05a25/charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5", size = 64538, upload-time = "2026-07-07T14:34:56.993Z" }, +] + +[[package]] +name = "click" +version = "8.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "idna" +version = "3.18" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "llmwhisperer-client" +version = "2.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "httpx" }, + { name = "requests" }, + { name = "tenacity" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/25/c7/d8c822ac12837073d07b9c4c3b4e3967ac6b6f8c91737899ca192e1f24fa/llmwhisperer_client-2.9.0.tar.gz", hash = "sha256:8db10ba2c3a9a8351f22bce809535489154bdc7531e7a54f7c04c7601b0cd784", size = 3317195, upload-time = "2026-09-01T10:15:39.042Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/7b/b192239d0b31e6979ba4d47336585de0fe437847d6839ff936bb9cb0310d/llmwhisperer_client-2.9.0-py3-none-any.whl", hash = "sha256:a2895053b21819fed2a6c85bc15ba05c74b883071b980360e32be537a285836a", size = 69257, upload-time = "2026-09-01T10:15:37.751Z" }, +] + +[[package]] +name = "markdown-it-py" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] +name = "rich" +version = "15.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, +] + +[[package]] +name = "ruff" +version = "0.16.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/73/e1/4508a569211b35599016e84ba65c1a992b7a4004b4b6c4bea02a851cba1b/ruff-0.16.2.tar.gz", hash = "sha256:c3d7828d12e8927a6fc65fe38e2c2541b9e762d360a1786d752cb1b8883b3c9c", size = 4885811, upload-time = "2026-08-07T13:31:01.432Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/57/db19951540f98859c956b50bdb4d31089b4d91e9f15e2968e7d5193806d5/ruff-0.16.2-py3-none-linux_armv6l.whl", hash = "sha256:3c8de4cf2181f01d57946d87d777aa52916976fc09942aed89938fab5e013318", size = 10847925, upload-time = "2026-08-07T13:30:14.468Z" }, + { url = "https://files.pythonhosted.org/packages/13/5a/995fe85a8470d3e391ac0f7fa8054bb454eaf33ee138196d6172ed1079c0/ruff-0.16.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9a48cc05c6fbc811ca81b5d7ba95375affea6582d1b8024e455e41afbbf55344", size = 11072662, upload-time = "2026-08-07T13:30:18.143Z" }, + { url = "https://files.pythonhosted.org/packages/32/53/370d767c61c71a971a4ace36703a7ecd8c393956349a7325d7fab2b56827/ruff-0.16.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a2c0d14fcbb26c91f0f867a6dc9bd71bbc30b1b6151829c884f23faeab2e5700", size = 10566771, upload-time = "2026-08-07T13:30:20.899Z" }, + { url = "https://files.pythonhosted.org/packages/85/d6/9d96948caf5a632be62d62202d5ec914d6856f204fd79eb036e5915e79ea/ruff-0.16.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:335c621622c4650330be50842561c6586ac6971bb8ab5407fe34dcc9efb16bbe", size = 10975825, upload-time = "2026-08-07T13:30:23.517Z" }, + { url = "https://files.pythonhosted.org/packages/3b/92/ea87129b3414acb0b5770563779c51804d37ac67675c7ba35447ddb14773/ruff-0.16.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:20e66910f2c37cc753f9ef6580c914a621b80c4fa3549d3e3521e29d0f5bfc3f", size = 10649437, upload-time = "2026-08-07T13:30:26.097Z" }, + { url = "https://files.pythonhosted.org/packages/ac/43/f8f291dcd4af5bb7872b74fdfa41a7cd7c856ca1d4069670971cf1b9f5cb/ruff-0.16.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7e36fbfba65510548156902bcf1350a979a958ce0347ce0f90d73894036b39f", size = 11446761, upload-time = "2026-08-07T13:30:28.752Z" }, + { url = "https://files.pythonhosted.org/packages/71/4a/ef991fb2fcf516ab71f0808adcdd8da5e18c8cde447f4ceaf5f47a5132a5/ruff-0.16.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f0eab35f80df8f134aae5d1630e751901321d317cc8e50dc39e36fa3ed34cd12", size = 12336364, upload-time = "2026-08-07T13:30:31.468Z" }, + { url = "https://files.pythonhosted.org/packages/f3/24/f615e74f307e6ca0e56a482872477b856c70d530aa356abfb6dfe5ca8a80/ruff-0.16.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:40ea8c0594feb894e89c8c61ab9c103d38b0ea72dfde6c594107147ca31b1140", size = 11630720, upload-time = "2026-08-07T13:30:34.426Z" }, + { url = "https://files.pythonhosted.org/packages/c5/d3/8ef50149e8412a77f7ab409efdef0e2b23803707a3863da4fc64cb23d459/ruff-0.16.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab3d62dde0b19facdd632008cc4827fc28ada7736c6bd35ab6f1050f0bfed53f", size = 11466130, upload-time = "2026-08-07T13:30:36.958Z" }, + { url = "https://files.pythonhosted.org/packages/dd/a7/a19334985c4dea8c381981fa252cd854c7ee52dc4b1686dc16f4a911c702/ruff-0.16.2-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:e43e1f5b8388da9eca1b9e88328d47a5cec794633ccf6f7484ac2dd15eee92c0", size = 11523634, upload-time = "2026-08-07T13:30:39.822Z" }, + { url = "https://files.pythonhosted.org/packages/6e/6c/96d192b0e742412ceda08c0a50f9669b253dde9fd6a60ea1a10c9fa79a63/ruff-0.16.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c24788a980581e1d7ea3a0cbe4344c4fbeb0a6a9b1f4713aa46bb104f8294690", size = 10949807, upload-time = "2026-08-07T13:30:42.745Z" }, + { url = "https://files.pythonhosted.org/packages/fa/51/e26599ceca11e79ee255c7df515995561edf87e9ca1893284e44d98f5a86/ruff-0.16.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:81806b08329130005dd4a8a8394a0c9da8c6f4cafb16ba438d2a2ee6a18bedf1", size = 10646891, upload-time = "2026-08-07T13:30:45.522Z" }, + { url = "https://files.pythonhosted.org/packages/68/01/800c4b1f97bc8d7c6029e06b1f20473a3cf1e13c4933d8f3342add83fc55/ruff-0.16.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:4ce4e02bad779bef557f541a1b31f20d6abeae1cc05ed1b1ac019d4ffd1044c8", size = 11162063, upload-time = "2026-08-07T13:30:48.131Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d0/1477ea50fc5a0d4b0b71d1d63d50770bdd794d90b43e37a7618e63ec9894/ruff-0.16.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e0422abdf70070255fc4073ce9dfc814cc03db577013761ddd09bc1e4a9a4fbd", size = 11556038, upload-time = "2026-08-07T13:30:50.686Z" }, + { url = "https://files.pythonhosted.org/packages/b8/76/a7776f32048d991e16d4fa8ff91790b877342d3596cc3ed04acdbf1aaedc/ruff-0.16.2-py3-none-win32.whl", hash = "sha256:bf3a63d78fb39f4bf5ac8ae52051c5520505301abe19ba4e204c453b3f09bb0b", size = 10872850, upload-time = "2026-08-07T13:30:53.471Z" }, + { url = "https://files.pythonhosted.org/packages/00/0d/929c800d920e61397d82a01b60bffc68da3052c17d31de59efaad2e4ed75/ruff-0.16.2-py3-none-win_amd64.whl", hash = "sha256:bcabe2f6d0fc7819f1431793005af4e4de7371927d037345bf941252b195b9fa", size = 12023338, upload-time = "2026-08-07T13:30:56.193Z" }, + { url = "https://files.pythonhosted.org/packages/5b/6c/93e26c22c5f78ff87363e07da49c84955affbeb1098bd1936bf3b3f293bf/ruff-0.16.2-py3-none-win_arm64.whl", hash = "sha256:d614e95cedf38a2053fd351c55b103ba30d017d61688fdbfd40ee0412852a99f", size = 11374065, upload-time = "2026-08-07T13:30:58.775Z" }, +] + +[[package]] +name = "tenacity" +version = "9.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/c6/ee486fd809e357697ee8a44d3d69222b344920433d3b6666ccd9b374630c/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a", size = 49413, upload-time = "2026-02-07T10:45:33.841Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, +] + +[[package]] +name = "tomli-w" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/19/75/241269d1da26b624c0d5e110e8149093c759b7a286138f4efd61a60e75fe/tomli_w-1.2.0.tar.gz", hash = "sha256:2dd14fac5a47c27be9cd4c976af5a12d87fb1f0b4512f81d69cce3b35ae25021", size = 7184, upload-time = "2025-01-15T12:07:24.262Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/18/c86eb8e0202e32dd3df50d43d7ff9854f8e0603945ff398974c1d91ac1ef/tomli_w-1.2.0-py3-none-any.whl", hash = "sha256:188306098d013b691fcadc011abd66727d3c414c571bb01b1a174ba8c983cf90", size = 6675, upload-time = "2025-01-15T12:07:22.074Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "unstract-cli" +source = { editable = "." } +dependencies = [ + { name = "click" }, + { name = "llmwhisperer-client" }, + { name = "tomli-w" }, + { name = "unstract-client" }, +] + +[package.optional-dependencies] +dev = [ + { name = "pytest" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "click", specifier = ">=8.1,<9" }, + { name = "llmwhisperer-client", specifier = "==2.9.0" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, + { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.6" }, + { name = "tomli-w", specifier = ">=1.0" }, + { name = "unstract-client", specifier = "==1.6.0" }, +] +provides-extras = ["dev"] + +[[package]] +name = "unstract-client" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "click" }, + { name = "httpx" }, + { name = "requests" }, + { name = "rich" }, + { name = "tenacity" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ad/8b/7c4eea378ec143a6c862ad03d96bc1cf0b8fab5f23078d51569b78a704a0/unstract_client-1.6.0.tar.gz", hash = "sha256:9fabdcf7c6752910d986bee5c66fb0e0f32b17f73cb1c27e51402eb372dfe81d", size = 203664, upload-time = "2026-09-01T10:13:54.391Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/18/9492ecbc9b0e83128147f688d815ace4cfee4d332ad1e9543cb8b1550344/unstract_client-1.6.0-py3-none-any.whl", hash = "sha256:8e4642d1fd0d2afd9983117ac6c78436f6476724b53c15f3266c725fde901757", size = 114425, upload-time = "2026-09-01T10:13:53.25Z" }, +] + +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, +]