UN-4008 [FEAT] Add the unstract CLI to run extractions and API deployments from the terminal

.github/workflows/ci.yml

@@ -0,0 +1,31 @@
+name: ci
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  # Offline by design: no network, no credentials, sub-second. Live round trips
+  # are a manual pre-release step, not a per-PR gate.
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+      - run: uv venv --python 3.12
+      - run: uv pip install -e '.[dev]'
+      - run: uv run ruff check .
+      - run: uv run ruff format --check .
+      - run: uv run pytest -q
+      # `uv run` resolves from uv.lock, whose click sits in the middle of the
+      # range the pin allows. Both ends are what an install in the wild gets,
+      # and discovery is a published contract read out of Click's own objects,
+      # so both ends are run and their answers compared.
+      - run: uv pip install 'click~=8.1.0'
+      - run: .venv/bin/python -m pytest -q
+      - run: .venv/bin/python -m unstract_cli -o json --discover full > floor.json
+      - run: uv pip install -U click
+      - run: .venv/bin/python -m pytest -q
+      - run: .venv/bin/python -m unstract_cli -o json --discover full > latest.json
+      - run: diff floor.json latest.json

.github/workflows/release.yml

@@ -0,0 +1,172 @@
+name: Release Tag and Publish Package
+
+on:
+  workflow_dispatch:
+    inputs:
+      version_bump:
+        description: "Version bump type. `none` publishes the version already in the repo, which is what the first release needs."
+        required: true
+        default: "patch"
+        type: choice
+        options:
+          - patch
+          - minor
+          - major
+          - none
+      pre_release:
+        description: "Publish a release candidate (`<version>rcN`) instead of the version itself. Dispatch again with this off to promote the same version to stable."
+        required: false
+        default: false
+        type: boolean
+      release_notes:
+        description: "Release notes (optional)"
+        required: false
+        type: string
+
+jobs:
+  release-and-publish:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      # Publishing is by PyPI Trusted Publisher, so there is no API token.
+      id-token: write
+    steps:
+      - name: Generate GitHub App Token
+        id: generate-token
+        uses: actions/create-github-app-token@v2
+        with:
+          app-id: ${{ vars.PUSH_TO_MAIN_APP_ID }}
+          private-key: ${{ secrets.PUSH_TO_MAIN_APP_PRIVATE_KEY }}
+          owner: Zipstack
+          repositories: |
+            unstract-cli
+
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ steps.generate-token.outputs.token }}
+          fetch-depth: 0
+
+      - name: Configure Git
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - uses: astral-sh/setup-uv@v5
+
+      # The same install as ci.yml, so what the release run lints and tests is
+      # what the PR gate lints and tests.
+      - run: uv venv --python 3.12
+      - run: uv pip install -e '.[dev]'
+
+      # Staged locally only: nothing is committed, tagged or released until the
+      # checks, the build and the publish have all passed, so a failure leaves
+      # main untouched.
+      - name: Compute new version
+        id: version
+        run: |
+          VERSION_FILE=src/unstract_cli/__init__.py
+          CURRENT_VERSION=$(sed -nE 's/^__version__ = "(.*)"/\1/p' "$VERSION_FILE")
+          echo "Current version: $CURRENT_VERSION"
+
+          IFS='.' read -r MAJOR MINOR PATCH <<< "$CURRENT_VERSION"
+          case "${{ github.event.inputs.version_bump }}" in
+            major) MAJOR=$((MAJOR + 1)); MINOR=0; PATCH=0 ;;
+            minor) MINOR=$((MINOR + 1)); PATCH=0 ;;
+            patch) PATCH=$((PATCH + 1)) ;;
+          esac
+          NEXT_VERSION="$MAJOR.$MINOR.$PATCH"
+
+          # A pre-release is a candidate for NEXT_VERSION, not a version of its
+          # own, so it never moves the committed one: the file keeps naming the
+          # last stable release, and repeat dispatches count up from the rc tags
+          # already published for that target.
+          if [ "${{ github.event.inputs.pre_release }}" = "true" ]; then
+            HIGHEST_RC=$(git tag -l "v${NEXT_VERSION}rc*" \
+              | sed -nE "s/^v${NEXT_VERSION}rc([0-9]+)$/\1/p" | sort -n | tail -1)
+            NEW_VERSION="${NEXT_VERSION}rc$(( ${HIGHEST_RC:-0} + 1 ))"
+          else
+            NEW_VERSION="$NEXT_VERSION"
+          fi
+
+          echo "New version: $NEW_VERSION"
+          echo "version=$NEW_VERSION" >> "$GITHUB_OUTPUT"
+
+          sed -i "s/^__version__ = \".*\"/__version__ = \"$NEW_VERSION\"/" "$VERSION_FILE"
+
+          if git rev-parse -q --verify "refs/tags/v$NEW_VERSION" >/dev/null; then
+            echo "Tag v$NEW_VERSION already exists. Exiting..."
+            exit 1
+          fi
+
+      - name: Verify version update
+        run: |
+          BUILT_VERSION=$(uv run python -c "import unstract_cli; print(unstract_cli.__version__)")
+          echo "Package version: $BUILT_VERSION"
+          echo "Target version: ${{ steps.version.outputs.version }}"
+          if [ "$BUILT_VERSION" != "${{ steps.version.outputs.version }}" ]; then
+            echo "Version mismatch! Exiting..."
+            exit 1
+          fi
+
+      - name: Run linting
+        run: |
+          uv run ruff check .
+          uv run ruff format --check .
+
+      - name: Run tests
+        run: uv run pytest -q
+
+      - name: Build package
+        run: uv build
+
+      # Publishing is the only step that cannot be undone, so the git metadata
+      # is written after it: a failure before this point leaves nothing to
+      # unpublish, and one after it is retried by hand against a live artifact.
+      - name: Publish to PyPI
+        run: uv publish
+
+      - name: Commit version bump and create release
+        run: |
+          NEW_VERSION="${{ steps.version.outputs.version }}"
+
+          # A pre-release leaves the committed version alone, and `none`
+          # publishes the version already in the file, so both reach here with
+          # nothing to commit.
+          if [ "${{ github.event.inputs.pre_release }}" = "true" ]; then
+            git checkout -- src/unstract_cli/__init__.py
+          elif ! git diff --quiet; then
+            git add src/unstract_cli/__init__.py
+            git commit -m "chore: bump version to $NEW_VERSION [skip ci]"
+            git push origin main
+          fi
+
+          git tag "v$NEW_VERSION"
+          git push origin "v$NEW_VERSION"
+
+          RELEASE_NOTES="${{ github.event.inputs.release_notes }}"
+          if [ -z "$RELEASE_NOTES" ]; then
+            gh release create "v$NEW_VERSION" \
+              --title "Release v$NEW_VERSION" \
+              --generate-notes \
+              ${{ github.event.inputs.pre_release == 'true' && '--prerelease' || '' }}
+          else
+            gh release create "v$NEW_VERSION" \
+              --title "Release v$NEW_VERSION" \
+              --notes "$RELEASE_NOTES" \
+              --generate-notes \
+              ${{ github.event.inputs.pre_release == 'true' && '--prerelease' || '' }}
+          fi
+
+          echo "Created release v$NEW_VERSION"
+        env:
+          GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }}
+
+      - name: Success message
+        run: |
+          echo "Published ${{ steps.version.outputs.version }} to PyPI with uv publish using Trusted Publishers"
+          echo "Release: https://github.com/${{ github.repository }}/releases/tag/v${{ steps.version.outputs.version }}"
+          echo "PyPI: https://pypi.org/project/unstract-cli/${{ steps.version.outputs.version }}/"

.gitignore

@@ -0,0 +1,7 @@
+.venv/
+__pycache__/
+*.egg-info/
+.pytest_cache/
+.ruff_cache/
+dist/
+build/

README.md

@@ -0,0 +1,141 @@
+# unstract-cli
+
+`unstract` — one CLI for the Unstract suite: extract a document with
+LLMWhisperer, run it through a Document Studio API deployment, get structured
+JSON back. It also clones one organization's resources into another.
+
+```bash
+curl -LsSf https://raw.githubusercontent.com/Zipstack/unstract-cli/main/install.sh | sh
+unstract config init
+unstract config doctor
+```
+
+The installer fetches `uv` if it is missing and installs the CLI with it; `uv`
+brings its own Python, so nothing on the machine has to match. Already have
+`uv`? `uv tool install git+https://github.com/Zipstack/unstract-cli` is the same
+thing. Set `UNSTRACT_CLI_SOURCE` to install a branch or a local checkout
+instead.
+
+Or run it without installing: `uvx --from git+https://github.com/Zipstack/unstract-cli unstract --discover groups`.
+
+## Output
+
+`unstract` prints a table by default — in a terminal and in a pipe alike, so
+what you see while trying something is what a script sees running it.
+
+**Parsing anything? Pass `-o json`.** stdout then carries exactly one envelope,
+on success and on failure alike:
+
+```json
+{"ok": true, "data": {...}, "error": null, "meta": {"contract_version": 1}}
+```
+
+`-o json` output depends on nothing but the command and its arguments — not the
+terminal, not the config, not the environment. `-o raw` prints one field
+unwrapped, for piping a document's text somewhere else. Diagnostics, warnings
+and progress always go to stderr.
+
+Consuming the JSON: ignore fields you do not recognise, and refuse a
+`meta.contract_version` above the one you were written against. `unstract
+--discover full` publishes the whole contract alongside every command and flag.
+
+If a coding agent is driving (detected from the environment it sets), the
+*default* becomes json. `--agent yes|no` forces that either way, and an explicit
+`-o` always wins over both.
+
+Failures exit non-zero with a stable code. The codes are this CLI's own
+convention, not a service's — they are the `ExitCode` enum in
+`core/errors.py`, and `--discover full` publishes the table so a caller does not
+have to copy it:
+
+| Code | Meaning |
+|------|---------|
+| 0 | success |
+| 1 | generic failure |
+| 2 | usage error |
+| 3 | authentication failed |
+| 4 | not found |
+| 5 | validation failed |
+| 6 | rate limited |
+| 7 | timed out (the job handle is in the error payload — resume, do not resubmit) |
+| 8 | server error |
+| 9 | result already consumed (one-shot read; use `--save` next time) |
+| 10 | the result was read but could not be saved — it is in `error.details` |
+| 130 | interrupted (128 + SIGINT) — the user stopped it, not a failure |
+
+## Configuration
+
+`~/.unstract/config.toml`, or a project-local `.unstract.toml` found by upward
+search, or `$UNSTRACT_CONFIG`, or `--config`. Every setting resolves
+**flag > env > profile > built-in default**, and the CLI is fully usable with no
+config file at all. The flag tier is the connection options on each product
+group — `unstract docstudio --base-url … --org-id … deployment run …`, and
+`--base-url`/`--api-key` on `whisper` — which override the profile for that one
+invocation without writing anything.
+
+```toml
+default_profile = "cloud-us"
+
+[profiles.cloud-us.llmwhisperer]
+base_url = "https://llmwhisperer-api.us-central.unstract.com/api/v2"
+api_key = "env:LLMWHISPERER_API_KEY"
+
+[profiles.cloud-us.docstudio]
+base_url = "https://us-central.unstract.com"
+org_id = "org_ABC123"
+api_key = "env:UNSTRACT_DEPLOYMENT_KEY"
+
+[profiles.cloud-us.deployments.invoices]
+api_name = "invoice-parser"
+```
+
+One `api_key` on the `docstudio` block covers every alias under it: a key minted
+under **Settings → API Key Manager** authenticates every API deployment in the
+organisation, so an alias normally carries only its `api_name`. Give an alias its
+own `api_key` when its deployment has a separate key of its own.
+
+Get an LLMWhisperer key from the LLMWhisperer console; a deployment key is shown
+on the API deployment's own page in the Unstract UI, and an organisation-wide one
+under Settings → API Key Manager. `config init` also writes an
+`onprem-example` profile as a shape to copy for a self-hosted install — its host
+is a placeholder, and only the *active* profile is ever resolved.
+
+A credential can be written into the file literally, but `env:VAR_NAME`
+indirection is what `config init` writes and what the examples use: the file
+then records where a secret lives rather than the secret itself, and stays safe
+to copy or commit. Either way the file is created `0600`, and `config doctor`
+warns when its mode is wider than that.
+
+`unstract config doctor` reports where each setting resolved from — including
+whether an `env:` reference is actually set in the current process — without
+echoing any value. It exits non-zero when one of its own checks failed, so a
+setup script can branch on it.
+
+A project-local `.unstract.toml` **found by upward search** may not supply
+`api_key` or `base_url`. Those are ignored, with a warning; everything else in it
+— profile selection, `org_id`, deployment aliases — applies as usual. A checkout
+you did not write is not trusted to name the host your key is sent to. Name the
+file explicitly (`--config` or `$UNSTRACT_CONFIG`) and it is honoured in full.
+
+What that protects is the key and the host, not the routing: `org_id`,
+`api_name` and profile selection stay repo-controllable by design, so a
+project file can still decide *which* deployment a command runs against on a
+host you trust. Read one before you run inside a checkout you did not write.
+
+`clone` is the exception, and it is an operator command: a human moving one
+organisation's resources into another, holding two admin Platform keys. It is
+not part of the document-processing path the rest of this CLI wraps, so an agent
+serving a user request should not reach for it unasked. It talks to two
+deployments at once, which no single profile describes, so it takes both
+endpoints as flags and both keys from `UNSTRACT_SRC_PLATFORM_KEY` /
+`UNSTRACT_TGT_PLATFORM_KEY`. It exits 0 when nothing failed, which is not the
+same as everything having moved: oversize and unsupported documents are skipped
+by design, and `data.skipped` counts them.
+
+## Development
+
+```bash
+uv venv && uv pip install -e '.[dev]'
+uv run pytest   # offline; no network, no credentials
+uv run ruff check .
+```

install.sh

@@ -0,0 +1,41 @@
+#!/bin/sh
+# Installs the `unstract` CLI. Override the source to install a branch or a
+# local checkout:
+#   UNSTRACT_CLI_SOURCE=/path/to/checkout sh install.sh
+set -eu
+
+# Flips to the bare PyPI name once the CLI is published there.
+SOURCE="${UNSTRACT_CLI_SOURCE:-git+https://github.com/Zipstack/unstract-cli@main}"
+
+if ! command -v uv >/dev/null 2>&1; then
+    echo "Installing uv..." >&2
+    curl -LsSf https://astral.sh/uv/install.sh | sh
+    # The installer only edits shell rc files, which this shell has already read.
+    PATH="${XDG_BIN_HOME:-${HOME}/.local/bin}:${HOME}/.cargo/bin:${PATH}"
+    export PATH
+fi
+
+if ! command -v uv >/dev/null 2>&1; then
+    echo "uv is installed but not on PATH; open a new shell and re-run." >&2
+    exit 1
+fi
+
+# uv fetches its own interpreter, so the CLI's Python floor is not the user's problem.
+uv tool install --force "$SOURCE"
+
+if command -v unstract >/dev/null 2>&1; then
+    echo
+    unstract --version 2>/dev/null || true
+    echo "Run 'unstract config init' to get started." >&2
+    exit 0
+fi
+
+cat >&2 <<MSG
+
+Installed, but 'unstract' is not on your PATH. Add uv's tool directory:
+
+    export PATH="$(uv tool dir --bin 2>/dev/null || echo "${HOME}/.local/bin"):\$PATH"
+
+Then re-open your shell, or run 'uv tool update-shell'.
+MSG
+exit 1