diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a68d3a2..4c70ad6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,3 +23,19 @@ jobs: # what discovery reads the internals of. - run: uv pip install -U click - run: .venv/bin/python -m pytest -q + + # The binaries are built and attached only after a release has published, so + # without this a broken spec -- a PyInstaller bump, a client re-pin, a data + # file added without a `datas` entry -- would surface as a live release whose + # advertised download 404s. One platform is enough to catch that; the release + # matrix covers the other two. + binary: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python -m pip install . 'pyinstaller==6.22.2' + - run: pyinstaller --clean --noconfirm unstract.spec + - run: scripts/smoke-binary.sh dist/unstract diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1dc86fd..36b1926 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,6 +26,10 @@ on: jobs: release-and-publish: runs-on: ubuntu-latest + outputs: + # Consumed by `build-binaries`, which checks out the tag this run created + # rather than whatever the branch has moved on to. + version: ${{ steps.version.outputs.version }} permissions: contents: write # Publishing is by PyPI Trusted Publisher, so there is no API token. @@ -170,3 +174,81 @@ jobs: echo "Published ${{ steps.version.outputs.version }} to PyPI with uv publish using Trusted Publishers" echo "Release: https://github.com/${{ github.repository }}/releases/tag/v${{ steps.version.outputs.version }}" echo "PyPI: https://pypi.org/project/unstract-cli/${{ steps.version.outputs.version }}/" + + # `needs`, not a tag trigger: the release has to exist before anything attaches + # an asset to it. A tag-push workflow would race the `gh release create` above, + # and the loser is whichever calls POST /releases second -- which for `gh` + # means failing after PyPI has already published. + # + # PyInstaller does not cross-compile, so one runner per OS and architecture. + build-binaries: + needs: [release-and-publish] + permissions: + contents: write + strategy: + # One platform's toolchain breaking is not a reason to lose the other two. + fail-fast: false + matrix: + include: + # 22.04 rather than 24.04: a binary's glibc floor is its builder's, and + # 22.04's 2.35 still covers Ubuntu 22.04 and Debian 12, where 24.04's + # 2.39 would drop both. + - runner: ubuntu-22.04 + asset: unstract-linux-x86_64 + - runner: ubuntu-22.04-arm + asset: unstract-linux-arm64 + - runner: macos-latest # arm64 + asset: unstract-macos-arm64 + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v4 + with: + ref: v${{ needs.release-and-publish.outputs.version }} + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + # A pre-release reverts `__init__.py` before tagging, so `v0.2.0rc1` is a + # tag whose tree still names the last stable version. `--version` reads + # that file, so without this the binary attached to an rc would report a + # version the release does not have. `-i.bak` because BSD sed on the macOS + # runner has no bare `-i`. + - name: Stamp the version being released + run: | + sed -i.bak 's/^__version__ = ".*"/__version__ = "${{ needs.release-and-publish.outputs.version }}"/' \ + src/unstract_cli/__init__.py + rm -f src/unstract_cli/__init__.py.bak + + # The install is for the dependencies: PyInstaller freezes `unstract_cli` + # itself from `src/`, which is why the stamp above lands in the binary. + - name: Install the CLI and PyInstaller + run: | + python -m pip install --upgrade pip + python -m pip install . + python -m pip install 'pyinstaller==6.22.2' + + - run: pyinstaller --clean --noconfirm unstract.spec + + # The same checks ci.yml runs on every pull request, plus the version, + # which only a release knows. + - name: Check the binary + run: | + scripts/smoke-binary.sh dist/unstract \ + "${{ needs.release-and-publish.outputs.version }}" + + - name: Name and checksum + run: | + mv dist/unstract "${{ matrix.asset }}" + chmod +x "${{ matrix.asset }}" + shasum -a 256 "${{ matrix.asset }}" > "${{ matrix.asset }}.sha256" + + # The release already exists, so the default token is enough and the App + # credential never reaches a job that runs a build. `--clobber` makes a + # re-run idempotent. + - name: Attach to the release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release upload "v${{ needs.release-and-publish.outputs.version }}" \ + "${{ matrix.asset }}" "${{ matrix.asset }}.sha256" --clobber diff --git a/.gitignore b/.gitignore index 130baad..3b15acd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .venv/ +.venv-freeze/ __pycache__/ *.egg-info/ .pytest_cache/ diff --git a/README.md b/README.md index 3c59880..9a56de3 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,26 @@ instead. Or run it without installing: `uvx --from git+https://github.com/Zipstack/unstract-cli unstract --discover groups`. +Prefer one file and no Python at all? Every release attaches a standalone binary +for Linux (x86_64 and arm64) and Apple Silicon: + +```bash +curl -Lo unstract https://github.com/Zipstack/unstract-cli/releases/latest/download/unstract-linux-x86_64 +chmod +x unstract && sudo mv unstract /usr/local/bin/ +``` + +Substitute `unstract-linux-arm64` or `unstract-macos-arm64`; each asset has a +`.sha256` beside it. Keep the name `unstract` when you move it into place — the +CLI reports the name it was invoked as, so a binary left called +`unstract-macos-arm64` says exactly that in `--version` and in every usage line. + +The binaries are unsigned. `curl` does not quarantine what it downloads, so +macOS runs one as-is; a browser download does get quarantined, and needs +`xattr -d com.apple.quarantine /usr/local/bin/unstract` once. The Linux binaries +are built on Ubuntu 22.04, so they need glibc 2.35 or newer — on anything older +(RHEL 9 and Amazon Linux 2023 are 2.34), the `uv` install above is the way in. +There is no Windows or Intel-Mac binary; both are `uv tool install`. + ## Output `unstract` prints a table by default — in a terminal and in a pipe alike, so @@ -139,3 +159,27 @@ uv venv && uv pip install -e '.[dev]' uv run pytest # offline; no network, no credentials uv run ruff check . ``` + +The standalone binaries are built from `unstract.spec`, which is committed and +hand-edited — `pyinstaller` regenerating it would drop the comments explaining +why each option is set. Every pull request builds it, and a release builds one +per platform. To reproduce one locally: + +```bash +python3.12 -m venv .venv-freeze +./.venv-freeze/bin/python -m pip install . 'pyinstaller==6.22.2' +./.venv-freeze/bin/pyinstaller --clean --noconfirm unstract.spec +scripts/smoke-binary.sh dist/unstract +``` + +The install is for the dependencies. `unstract_cli` itself is frozen from +`src/`, because PyInstaller puts the entry script's own tree on the module +search path ahead of anything installed — so an edit is picked up by a rebuild +alone, and the spec reads the packaged specs and overlay out of `src/` too +rather than out of site-packages, so the two cannot drift apart. + +`smoke-binary.sh` runs the binary under `env -i` with an empty `PATH`, which is +the only way to see a missing module: a dev box has a Python that would answer +the import. `--version` is not a trivial check there — importing the command +modules derives every flag from the bundled specs, so it fails outright if the +spec's `datas` came out wrong. diff --git a/scripts/smoke-binary.sh b/scripts/smoke-binary.sh new file mode 100755 index 0000000..9a6eaaa --- /dev/null +++ b/scripts/smoke-binary.sh @@ -0,0 +1,42 @@ +#!/bin/sh +# Exercise a built `unstract` binary with no interpreter in reach. +# +# scripts/smoke-binary.sh dist/unstract [expected-version] +# +# Run by ci.yml on every pull request and by release.yml before a binary is +# attached to a release, so the same checks decide both. A dev box has a Python +# that would answer an import the bundle is missing, which is why every command +# below runs under `env -i` with an empty PATH. +set -eu + +BIN=$(cd "$(dirname "$1")" && pwd)/$(basename "$1") +EXPECTED_VERSION="${2:-}" + +mkdir -p /tmp/emptybin +run() { env -i PATH=/tmp/emptybin HOME="$HOME" "$BIN" "$@"; } + +# Not a trivial path: importing the command modules applies the `@spec_options` +# decorators, which read `overlay.toml` and both vendored specs before Click +# parses anything. A bundle missing its data files fails here. +run --version +run --discover full >/dev/null + +# Click wraps help text to the terminal width, so a phrase can arrive split +# across lines; squeeze the whitespace rather than pin the wrapping. +help_text() { run "$@" --help | tr -s '[:space:]' ' '; } + +# One derived flag per vendored spec, proving each was reachable... +help_text whisper extract | grep -q -- '--add-line-nos' +help_text docstudio deployment run | grep -q -- '--hitl-packet-id' +# ...and one help string, which comes from the published client's docstring +# rather than from the spec. A build made with `-OO` passes everything above and +# fails only this. +help_text whisper extract | grep -q 'Adds line numbers' + +# The release job stamps the version it published; a binary that disagrees with +# the release it is attached to is worse than no binary. +if [ -n "$EXPECTED_VERSION" ]; then + run --version | grep -q "$EXPECTED_VERSION" +fi + +echo "OK $(basename "$BIN")" diff --git a/src/unstract_cli/app.py b/src/unstract_cli/app.py index 2eb6306..5cb0210 100644 --- a/src/unstract_cli/app.py +++ b/src/unstract_cli/app.py @@ -12,6 +12,7 @@ import click +from unstract_cli import __version__ from unstract_cli.commands.config_cmd import config_group from unstract_cli.config import ( DOCSTUDIO, @@ -139,7 +140,10 @@ def secrets(self) -> list[str]: default=None, help="Describe this CLI as JSON instead of running a command, useful for agents.", ) -@click.version_option(package_name="unstract-cli") +# The version is passed rather than looked up: `importlib.metadata` has no +# distribution to read inside a frozen binary, and `__init__.py` is already the +# one place the release workflow bumps. +@click.version_option(__version__, package_name="unstract-cli") @click.pass_context def cli( ctx: click.Context, diff --git a/src/unstract_cli/core/params.py b/src/unstract_cli/core/params.py index 971876f..1e6bb70 100644 --- a/src/unstract_cli/core/params.py +++ b/src/unstract_cli/core/params.py @@ -53,7 +53,9 @@ def load_spec(product: str) -> dict[str, Any]: filename = SPEC_FILES[product] except KeyError: raise KeyError(f"No spec vendored for product {product!r}") from None - text = (resources.files("unstract_cli.specs") / filename).read_text(encoding="utf-8") + text = (resources.files("unstract_cli") / "specs" / filename).read_text( + encoding="utf-8" + ) return json.loads(text) diff --git a/tests/test_specs.py b/tests/test_specs.py index 1537b42..0f13fe8 100644 --- a/tests/test_specs.py +++ b/tests/test_specs.py @@ -15,13 +15,13 @@ from unstract_cli.core.params import SPEC_FILES PROVENANCE = json.loads( - (resources.files("unstract_cli.specs") / "provenance.json").read_text("utf-8") + (resources.files("unstract_cli") / "specs" / "provenance.json").read_text("utf-8") ) @pytest.mark.parametrize("filename", sorted(SPEC_FILES.values())) def test_each_vendored_spec_is_the_pinned_one(filename): - blob = (resources.files("unstract_cli.specs") / filename).read_bytes() + blob = (resources.files("unstract_cli") / "specs" / filename).read_bytes() assert hashlib.sha256(blob).hexdigest() == PROVENANCE[filename]["sha256"] diff --git a/unstract.spec b/unstract.spec new file mode 100644 index 0000000..4a3bf16 --- /dev/null +++ b/unstract.spec @@ -0,0 +1,112 @@ +# -*- mode: python ; coding: utf-8 -*- +"""One-file build of the `unstract` CLI. + +Generated once with + + pyinstaller --onefile --console --name unstract src/unstract_cli/__main__.py + +and hand-edited since. This file is the build, not that command line: the +options below are decisions, and a regenerated spec would drop them silently. +""" + +# The packaged files that are read through `importlib.resources` -- `overlay.toml` +# and the vendored specs -- and read at *import* time, by the `@spec_options` +# decorators the command modules apply at module scope. They are data, not +# modules, so nothing puts them in the PYZ, and a bundle without them builds +# clean and then fails on every invocation. +# +# Taken from `src/` rather than from `collect_data_files("unstract_cli")`, which +# reads the *installed* package. PyInstaller prepends the entry script's parent +# package directory to the module search path, so the code is frozen from `src/` +# either way; sourcing the data from site-packages would let an edited module +# ship beside a stale spec. One tree decides both. +# +# A new data file needs a line here. `ci.yml` builds this spec on every pull +# request, so one that is read at import time fails the gate rather than a +# release. +datas = [ + ("src/unstract_cli/overlay.toml", "unstract_cli"), + ("src/unstract_cli/specs", "unstract_cli/specs"), +] + +hiddenimports = [ + # `unstract.clone.report.CloneReport.render` imports these inside the + # function, behind `except ImportError: return self._render_plain()`. The + # module graph does follow function-level imports, but a miss here degrades + # `unstract clone`'s table to plain text without failing anything, so the + # dependency is stated rather than inferred. + "rich.console", + "rich.table", +] + +# A local build runs in a `.[dev]` venv, so the test and lint tooling is on the +# path even though nothing reaches it from the entry point. CI installs only the +# runtime dependencies, where these are no-ops -- they keep the two builds the +# same size rather than being load-bearing. `unittest` is deliberately absent: +# the size it saves is small, and libraries reach for `unittest.mock` in +# surprising places. +excludes = [ + "pytest", + "_pytest", + "pluggy", + "iniconfig", + "ruff", + "setuptools", + "pkg_resources", + "tkinter", +] + +a = Analysis( + ["src/unstract_cli/__main__.py"], + pathex=[], + binaries=[], + datas=datas, + hiddenimports=hiddenimports, + hookspath=[], + hooksconfig={}, + runtime_hooks=[], + excludes=excludes, + noarchive=False, + # Not 1 or 2, and never build with PYTHONOPTIMIZE set. Every derived flag's + # help text comes from `inspect.getdoc()` on the published clients' methods + # -- the specs carry no parameter descriptions -- so stripping docstrings + # empties `--help` across the whole generated surface without failing a + # single check. + optimize=0, +) + +pyz = PYZ(a.pure) + +exe = EXE( + pyz, + a.scripts, + # One file: the binaries and the data are folded into the executable rather + # than collected beside it, so there is no COLLECT and nothing to unpack. + a.binaries, + a.datas, + [], + name="unstract", + debug=False, + bootloader_ignore_signals=False, + # Stripping invalidates the ad-hoc signature an arm64 macOS binary needs in + # order to run at all, and occasionally produces unloadable shared objects + # on Linux. It saves a couple of megabytes out of twenty. + strip=False, + # UPX is unusable on macOS arm64, and on Linux it buys size back by adding + # decompression to every start and by looking like packed malware to EDR. + upx=False, + upx_exclude=[], + runtime_tmpdir=None, + console=True, + disable_windowed_traceback=False, + argv_emulation=False, + # The building interpreter's architecture. The runners are native, and no + # universal binary is shipped. + target_arch=None, + # Unsigned by design. PyInstaller still applies the ad-hoc signature Apple + # Silicon requires to execute a Mach-O at all; what is absent is a Developer + # ID signature and notarisation, which is why a browser download needs its + # quarantine attribute cleared. + codesign_identity=None, + entitlements_file=None, +)