UN-4024 [FEAT] Ship the CLI as a standalone binary for Linux and Apple Silicon

.github/workflows/ci.yml

@@ -23,3 +23,19 @@ jobs:
       # what discovery reads the internals of.
       - run: uv pip install -U click
       - run: .venv/bin/python -m pytest -q
+
+  # The binaries are built and attached only after a release has published, so
+  # without this a broken spec -- a PyInstaller bump, a client re-pin, a data
+  # file added without a `datas` entry -- would surface as a live release whose
+  # advertised download 404s. One platform is enough to catch that; the release
+  # matrix covers the other two.
+  binary:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: python -m pip install . 'pyinstaller==6.22.2'
+      - run: pyinstaller --clean --noconfirm unstract.spec
+      - run: scripts/smoke-binary.sh dist/unstract

.github/workflows/release.yml

@@ -26,6 +26,10 @@ on:
 jobs:
   release-and-publish:
     runs-on: ubuntu-latest
+    outputs:
+      # Consumed by `build-binaries`, which checks out the tag this run created
+      # rather than whatever the branch has moved on to.
+      version: ${{ steps.version.outputs.version }}
     permissions:
       contents: write
       # Publishing is by PyPI Trusted Publisher, so there is no API token.
@@ -170,3 +174,81 @@ jobs:
           echo "Published ${{ steps.version.outputs.version }} to PyPI with uv publish using Trusted Publishers"
           echo "Release: https://github.com/${{ github.repository }}/releases/tag/v${{ steps.version.outputs.version }}"
           echo "PyPI: https://pypi.org/project/unstract-cli/${{ steps.version.outputs.version }}/"
+
+  # `needs`, not a tag trigger: the release has to exist before anything attaches
+  # an asset to it. A tag-push workflow would race the `gh release create` above,
+  # and the loser is whichever calls POST /releases second -- which for `gh`
+  # means failing after PyPI has already published.
+  #
+  # PyInstaller does not cross-compile, so one runner per OS and architecture.
+  build-binaries:
+    needs: [release-and-publish]
+    permissions:
+      contents: write
+    strategy:
+      # One platform's toolchain breaking is not a reason to lose the other two.
+      fail-fast: false
+      matrix:
+        include:
+          # 22.04 rather than 24.04: a binary's glibc floor is its builder's, and
+          # 22.04's 2.35 still covers Ubuntu 22.04 and Debian 12, where 24.04's
+          # 2.39 would drop both.
+          - runner: ubuntu-22.04
+            asset: unstract-linux-x86_64
+          - runner: ubuntu-22.04-arm
+            asset: unstract-linux-arm64
+          - runner: macos-latest # arm64
+            asset: unstract-macos-arm64
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: v${{ needs.release-and-publish.outputs.version }}
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      # A pre-release reverts `__init__.py` before tagging, so `v0.2.0rc1` is a
+      # tag whose tree still names the last stable version. `--version` reads
+      # that file, so without this the binary attached to an rc would report a
+      # version the release does not have. `-i.bak` because BSD sed on the macOS
+      # runner has no bare `-i`.
+      - name: Stamp the version being released
+        run: |
+          sed -i.bak 's/^__version__ = ".*"/__version__ = "${{ needs.release-and-publish.outputs.version }}"/' \
+            src/unstract_cli/__init__.py
+          rm -f src/unstract_cli/__init__.py.bak
+
+      # The install is for the dependencies: PyInstaller freezes `unstract_cli`
+      # itself from `src/`, which is why the stamp above lands in the binary.
+      - name: Install the CLI and PyInstaller
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install .
+          python -m pip install 'pyinstaller==6.22.2'
+
+      - run: pyinstaller --clean --noconfirm unstract.spec
+
+      # The same checks ci.yml runs on every pull request, plus the version,
+      # which only a release knows.
+      - name: Check the binary
+        run: |
+          scripts/smoke-binary.sh dist/unstract \
+            "${{ needs.release-and-publish.outputs.version }}"
+
+      - name: Name and checksum
+        run: |
+          mv dist/unstract "${{ matrix.asset }}"
+          chmod +x "${{ matrix.asset }}"
+          shasum -a 256 "${{ matrix.asset }}" > "${{ matrix.asset }}.sha256"
+
+      # The release already exists, so the default token is enough and the App
+      # credential never reaches a job that runs a build. `--clobber` makes a
+      # re-run idempotent.
+      - name: Attach to the release
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh release upload "v${{ needs.release-and-publish.outputs.version }}" \
+            "${{ matrix.asset }}" "${{ matrix.asset }}.sha256" --clobber

.gitignore

@@ -1,4 +1,5 @@
 .venv/
+.venv-freeze/
 __pycache__/
 *.egg-info/
 .pytest_cache/

README.md

@@ -18,6 +18,26 @@ instead.
 
 Or run it without installing: `uvx --from git+https://github.com/Zipstack/unstract-cli unstract --discover groups`.
 
+Prefer one file and no Python at all? Every release attaches a standalone binary
+for Linux (x86_64 and arm64) and Apple Silicon:
+
+```bash
+curl -Lo unstract https://github.com/Zipstack/unstract-cli/releases/latest/download/unstract-linux-x86_64
+chmod +x unstract && sudo mv unstract /usr/local/bin/
+```
+
+Substitute `unstract-linux-arm64` or `unstract-macos-arm64`; each asset has a
+`.sha256` beside it. Keep the name `unstract` when you move it into place — the
+CLI reports the name it was invoked as, so a binary left called
+`unstract-macos-arm64` says exactly that in `--version` and in every usage line.
+
+The binaries are unsigned. `curl` does not quarantine what it downloads, so
+macOS runs one as-is; a browser download does get quarantined, and needs
+`xattr -d com.apple.quarantine /usr/local/bin/unstract` once. The Linux binaries
+are built on Ubuntu 22.04, so they need glibc 2.35 or newer — on anything older
+(RHEL 9 and Amazon Linux 2023 are 2.34), the `uv` install above is the way in.
+There is no Windows or Intel-Mac binary; both are `uv tool install`.
+
 ## Output
 
 `unstract` prints a table by default — in a terminal and in a pipe alike, so
@@ -139,3 +159,27 @@ uv venv && uv pip install -e '.[dev]'
 uv run pytest   # offline; no network, no credentials
 uv run ruff check .
 ```
+
+The standalone binaries are built from `unstract.spec`, which is committed and
+hand-edited — `pyinstaller` regenerating it would drop the comments explaining
+why each option is set. Every pull request builds it, and a release builds one
+per platform. To reproduce one locally:
+
+```bash
+python3.12 -m venv .venv-freeze
+./.venv-freeze/bin/python -m pip install . 'pyinstaller==6.22.2'
+./.venv-freeze/bin/pyinstaller --clean --noconfirm unstract.spec
+scripts/smoke-binary.sh dist/unstract
+```
+
+The install is for the dependencies. `unstract_cli` itself is frozen from
+`src/`, because PyInstaller puts the entry script's own tree on the module
+search path ahead of anything installed — so an edit is picked up by a rebuild
+alone, and the spec reads the packaged specs and overlay out of `src/` too
+rather than out of site-packages, so the two cannot drift apart.
+
+`smoke-binary.sh` runs the binary under `env -i` with an empty `PATH`, which is
+the only way to see a missing module: a dev box has a Python that would answer
+the import. `--version` is not a trivial check there — importing the command
+modules derives every flag from the bundled specs, so it fails outright if the
+spec's `datas` came out wrong.

scripts/smoke-binary.sh

@@ -0,0 +1,42 @@
+#!/bin/sh
+# Exercise a built `unstract` binary with no interpreter in reach.
+#
+#   scripts/smoke-binary.sh dist/unstract [expected-version]
+#
+# Run by ci.yml on every pull request and by release.yml before a binary is
+# attached to a release, so the same checks decide both. A dev box has a Python
+# that would answer an import the bundle is missing, which is why every command
+# below runs under `env -i` with an empty PATH.
+set -eu
+
+BIN=$(cd "$(dirname "$1")" && pwd)/$(basename "$1")
+EXPECTED_VERSION="${2:-}"
+
+mkdir -p /tmp/emptybin
+run() { env -i PATH=/tmp/emptybin HOME="$HOME" "$BIN" "$@"; }
+
+# Not a trivial path: importing the command modules applies the `@spec_options`
+# decorators, which read `overlay.toml` and both vendored specs before Click
+# parses anything. A bundle missing its data files fails here.
+run --version
+run --discover full >/dev/null
+
+# Click wraps help text to the terminal width, so a phrase can arrive split
+# across lines; squeeze the whitespace rather than pin the wrapping.
+help_text() { run "$@" --help | tr -s '[:space:]' ' '; }
+
+# One derived flag per vendored spec, proving each was reachable...
+help_text whisper extract | grep -q -- '--add-line-nos'
+help_text docstudio deployment run | grep -q -- '--hitl-packet-id'
+# ...and one help string, which comes from the published client's docstring
+# rather than from the spec. A build made with `-OO` passes everything above and
+# fails only this.
+help_text whisper extract | grep -q 'Adds line numbers'
+
+# The release job stamps the version it published; a binary that disagrees with
+# the release it is attached to is worse than no binary.
+if [ -n "$EXPECTED_VERSION" ]; then
+    run --version | grep -q "$EXPECTED_VERSION"
+fi
+
+echo "OK  $(basename "$BIN")"

src/unstract_cli/app.py

@@ -12,6 +12,7 @@
 
 import click
 
+from unstract_cli import __version__
 from unstract_cli.commands.config_cmd import config_group
 from unstract_cli.config import (
     DOCSTUDIO,
@@ -139,7 +140,10 @@ def secrets(self) -> list[str]:
     default=None,
     help="Describe this CLI as JSON instead of running a command, useful for agents.",
 )
-@click.version_option(package_name="unstract-cli")
+# The version is passed rather than looked up: `importlib.metadata` has no
+# distribution to read inside a frozen binary, and `__init__.py` is already the
+# one place the release workflow bumps.
+@click.version_option(__version__, package_name="unstract-cli")
 @click.pass_context
 def cli(
     ctx: click.Context,

src/unstract_cli/core/params.py

@@ -53,7 +53,9 @@ def load_spec(product: str) -> dict[str, Any]:
         filename = SPEC_FILES[product]
     except KeyError:
         raise KeyError(f"No spec vendored for product {product!r}") from None
-    text = (resources.files("unstract_cli.specs") / filename).read_text(encoding="utf-8")
+    text = (resources.files("unstract_cli") / "specs" / filename).read_text(
+        encoding="utf-8"
+    )
     return json.loads(text)
 
 

tests/test_specs.py

@@ -15,13 +15,13 @@
 from unstract_cli.core.params import SPEC_FILES
 
 PROVENANCE = json.loads(
-    (resources.files("unstract_cli.specs") / "provenance.json").read_text("utf-8")
+    (resources.files("unstract_cli") / "specs" / "provenance.json").read_text("utf-8")
 )
 
 
 @pytest.mark.parametrize("filename", sorted(SPEC_FILES.values()))
 def test_each_vendored_spec_is_the_pinned_one(filename):
-    blob = (resources.files("unstract_cli.specs") / filename).read_bytes()
+    blob = (resources.files("unstract_cli") / "specs" / filename).read_bytes()
     assert hashlib.sha256(blob).hexdigest() == PROVENANCE[filename]["sha256"]
 
 

unstract.spec

@@ -0,0 +1,112 @@
+# -*- mode: python ; coding: utf-8 -*-
+"""One-file build of the `unstract` CLI.
+
+Generated once with
+
+    pyinstaller --onefile --console --name unstract src/unstract_cli/__main__.py
+
+and hand-edited since. This file is the build, not that command line: the
+options below are decisions, and a regenerated spec would drop them silently.
+"""
+
+# The packaged files that are read through `importlib.resources` -- `overlay.toml`
+# and the vendored specs -- and read at *import* time, by the `@spec_options`
+# decorators the command modules apply at module scope. They are data, not
+# modules, so nothing puts them in the PYZ, and a bundle without them builds
+# clean and then fails on every invocation.
+#
+# Taken from `src/` rather than from `collect_data_files("unstract_cli")`, which
+# reads the *installed* package. PyInstaller prepends the entry script's parent
+# package directory to the module search path, so the code is frozen from `src/`
+# either way; sourcing the data from site-packages would let an edited module
+# ship beside a stale spec. One tree decides both.
+#
+# A new data file needs a line here. `ci.yml` builds this spec on every pull
+# request, so one that is read at import time fails the gate rather than a
+# release.
+datas = [
+    ("src/unstract_cli/overlay.toml", "unstract_cli"),
+    ("src/unstract_cli/specs", "unstract_cli/specs"),
+]
+
+hiddenimports = [
+    # `unstract.clone.report.CloneReport.render` imports these inside the
+    # function, behind `except ImportError: return self._render_plain()`. The
+    # module graph does follow function-level imports, but a miss here degrades
+    # `unstract clone`'s table to plain text without failing anything, so the
+    # dependency is stated rather than inferred.
+    "rich.console",
+    "rich.table",
+]
+
+# A local build runs in a `.[dev]` venv, so the test and lint tooling is on the
+# path even though nothing reaches it from the entry point. CI installs only the
+# runtime dependencies, where these are no-ops -- they keep the two builds the
+# same size rather than being load-bearing. `unittest` is deliberately absent:
+# the size it saves is small, and libraries reach for `unittest.mock` in
+# surprising places.
+excludes = [
+    "pytest",
+    "_pytest",
+    "pluggy",
+    "iniconfig",
+    "ruff",
+    "setuptools",
+    "pkg_resources",
+    "tkinter",
+]
+
+a = Analysis(
+    ["src/unstract_cli/__main__.py"],
+    pathex=[],
+    binaries=[],
+    datas=datas,
+    hiddenimports=hiddenimports,
+    hookspath=[],
+    hooksconfig={},
+    runtime_hooks=[],
+    excludes=excludes,
+    noarchive=False,
+    # Not 1 or 2, and never build with PYTHONOPTIMIZE set. Every derived flag's
+    # help text comes from `inspect.getdoc()` on the published clients' methods
+    # -- the specs carry no parameter descriptions -- so stripping docstrings
+    # empties `--help` across the whole generated surface without failing a
+    # single check.
+    optimize=0,
+)
+
+pyz = PYZ(a.pure)
+
+exe = EXE(
+    pyz,
+    a.scripts,
+    # One file: the binaries and the data are folded into the executable rather
+    # than collected beside it, so there is no COLLECT and nothing to unpack.
+    a.binaries,
+    a.datas,
+    [],
+    name="unstract",
+    debug=False,
+    bootloader_ignore_signals=False,
+    # Stripping invalidates the ad-hoc signature an arm64 macOS binary needs in
+    # order to run at all, and occasionally produces unloadable shared objects
+    # on Linux. It saves a couple of megabytes out of twenty.
+    strip=False,
+    # UPX is unusable on macOS arm64, and on Linux it buys size back by adding
+    # decompression to every start and by looking like packed malware to EDR.
+    upx=False,
+    upx_exclude=[],
+    runtime_tmpdir=None,
+    console=True,
+    disable_windowed_traceback=False,
+    argv_emulation=False,
+    # The building interpreter's architecture. The runners are native, and no
+    # universal binary is shipped.
+    target_arch=None,
+    # Unsigned by design. PyInstaller still applies the ad-hoc signature Apple
+    # Silicon requires to execute a Mach-O at all; what is absent is a Developer
+    # ID signature and notarisation, which is why a browser download needs its
+    # quarantine attribute cleared.
+    codesign_identity=None,
+    entitlements_file=None,
+)