chore: optimize CI test coverage

.github/scripts/binaries-matrix.json

@@ -2,6 +2,7 @@
   "include": [
     {
       "name": "linux-x86_64-gnu",
+      "pr": true,
       "build_runner": "ubuntu-24.04",
       "mode": "glibc",
       "build_image": "almalinux:8@sha256:4a87d2615a770506e204c27d6248ac97f4df67f4e41e2e9c47c81f0ed0be98cb",
@@ -42,6 +43,7 @@
     },
     {
       "name": "macos-arm64",
+      "pr": true,
       "build_runner": "macos-14",
       "mode": "native",
       "build_image": "",
@@ -62,6 +64,7 @@
     },
     {
       "name": "windows-x86_64",
+      "pr": true,
       "build_runner": "windows-2025",
       "mode": "native",
       "build_image": "",

.github/workflows/binaries.yml

@@ -18,6 +18,18 @@ on:
       CLOUDSMITH_API_KEY:
         description: Read-only API key for the online smoketests
         required: false
+  push:
+    branches:
+      - master
+    paths:
+      - cloudsmith_cli/**
+      - packaging/**
+      - Dockerfile
+      - pyproject.toml
+      - uv.lock
+      - .github/scripts/**
+      - .github/workflows/binaries.yml
+      - .github/workflows/release.yml
   pull_request:
     paths:
       - cloudsmith_cli/**
@@ -77,12 +89,21 @@ jobs:
           persist-credentials: false
 
       - id: gen
-        run: echo "include=$(jq -c . .github/scripts/binaries-matrix.json)" >> "$GITHUB_OUTPUT"
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+        run: |
+          MATRIX=$(
+            jq -c --arg event_name "${EVENT_NAME}" \
+              '{include: [.include[] | select($event_name != "pull_request" or .pr) | del(.pr)]}' \
+              .github/scripts/binaries-matrix.json
+          )
+          test "$(echo "${MATRIX}" | jq '.include | length')" -gt 0
+          echo "include=${MATRIX}" >> "$GITHUB_OUTPUT"
 
   build:
     needs: matrix
     strategy:
-      fail-fast: false
+      fail-fast: ${{ github.event_name == 'pull_request' }}
       matrix: ${{ fromJson(needs.matrix.outputs.include) }}
     runs-on: ${{ matrix.build_runner }}
     timeout-minutes: 45
@@ -207,7 +228,7 @@ jobs:
   test:
     needs: [matrix, build]
     strategy:
-      fail-fast: false
+      fail-fast: ${{ github.event_name == 'pull_request' }}
       matrix: ${{ fromJson(needs.matrix.outputs.include) }}
     runs-on: ${{ matrix.test_runner }}
     timeout-minutes: 25

.github/workflows/image-build.yml

@@ -8,20 +8,35 @@ on:
       - "**"
 permissions:
   contents: read
+
+concurrency:
+  group: image-build-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   image-build:
     name: Build container image
     runs-on: ubuntu-24.04
+    timeout-minutes: 30
     steps:
       - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
         with:
           persist-credentials: false
+
+      - uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
+
+      - uses: crazy-max/ghaction-github-runtime@04d248b84655b509d8c44dc1d6f990c879747487 # v4.0.0
+
       - name: Build image
         run: |
-          docker build -t cloudsmith-io/cli-dev \
-          --file dev.Dockerfile \
-          --build-arg CLOUDSMITH_CLI_VERSION="$(cat VERSION)" \
-          --build-arg VCS_REF="${GITHUB_SHA}" \
-          .
+          docker buildx build \
+            --load \
+            --cache-from type=gha \
+            --cache-to type=gha,mode=max \
+            --tag cloudsmith-io/cli-dev \
+            --file dev.Dockerfile \
+            --build-arg CLOUDSMITH_CLI_VERSION="$(cat VERSION)" \
+            --build-arg VCS_REF="${GITHUB_SHA}" \
+            .
       - name: Test image
         run: docker run -t cloudsmith-io/cli-dev --help

.github/workflows/test.yml

@@ -17,10 +17,11 @@ concurrency:
 
 jobs:
   pytest:
-    name: Run tests (Python ${{ matrix.python-version }})
+    name: Unit tests (Python ${{ matrix.python-version }})
     runs-on: ubuntu-24.04
+    timeout-minutes: 15
     strategy:
-      fail-fast: false
+      fail-fast: ${{ github.event_name == 'pull_request' }}
       matrix:
         python-version: ['3.10', '3.11', '3.12', '3.13', '3.14']
 
@@ -43,16 +44,80 @@ jobs:
         run: uv sync --locked --group dev --python ${{ matrix.python-version }}
 
       - name: Run pytest
+        id: pytest
+        env:
+          COVERAGE_ARGS: ${{ matrix.python-version == '3.12' && '--cov=cloudsmith_cli' || '' }}
+        run: uv run pytest -m "not integration" ${COVERAGE_ARGS} --junitxml=./reports/pytest.xml
+
+      - name: Upload test results
+        if: ${{ !cancelled() && steps.pytest.outcome != 'skipped' }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: test-results-${{ matrix.python-version }}
+          path: ./reports/pytest.xml
+          retention-days: 30
+          if-no-files-found: error
+
+  integration-gate:
+    name: Check integration credentials
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    outputs:
+      available: ${{ steps.credentials.outputs.available }}
+    steps:
+      - name: Check integration credentials
+        id: credentials
+        env:
+          API_KEY: ${{ secrets.PYTEST_CLOUDSMITH_API_KEY }}
+          API_HOST: ${{ vars.PYTEST_CLOUDSMITH_API_HOST }}
+          ORGANIZATION: ${{ vars.PYTEST_CLOUDSMITH_ORGANIZATION }}
+        run: |
+          if [ -n "${API_KEY}" ] && [ -n "${API_HOST}" ] && [ -n "${ORGANIZATION}" ]; then
+            echo "available=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "available=false" >> "$GITHUB_OUTPUT"
+            echo "Skipping live integration tests because credentials are unavailable."
+          fi
+
+  integration:
+    name: Live integration tests (Python 3.12)
+    needs: integration-gate
+    if: needs.integration-gate.outputs.available == 'true'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          persist-credentials: false
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+        with:
+          enable-cache: true
+
+      - name: Install dependencies
+        run: uv sync --locked --group dev --python 3.12
+
+      - name: Run live integration tests
+        id: pytest
         env:
           PYTEST_CLOUDSMITH_API_KEY: ${{ secrets.PYTEST_CLOUDSMITH_API_KEY }}
           PYTEST_CLOUDSMITH_API_HOST: ${{ vars.PYTEST_CLOUDSMITH_API_HOST }}
           PYTEST_CLOUDSMITH_ORGANIZATION: ${{ vars.PYTEST_CLOUDSMITH_ORGANIZATION }}
-        run: uv run pytest --junitxml=./reports/pytest.xml
+          PYTEST_CLOUDSMITH_USERNAME: ${{ secrets.PYTEST_CLOUDSMITH_USERNAME }}
+          PYTEST_CLOUDSMITH_PASSWORD: ${{ secrets.PYTEST_CLOUDSMITH_PASSWORD }}
+        run: uv run pytest -m integration --junitxml=./reports/pytest.xml
 
       - name: Upload test results
-        if: always()
+        if: ${{ !cancelled() && steps.pytest.outcome != 'skipped' }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: test-results-${{ matrix.python-version }}
+          name: test-results-integration-3.12
           path: ./reports/pytest.xml
           retention-days: 30
+          if-no-files-found: error

.github/workflows/ty-check.yml

@@ -1,10 +1,18 @@
 name: Type check
 on:
+  push:
+    branches:
+      - master
   pull_request:
     branches:
       - "**"
 permissions:
   contents: read
+
+concurrency:
+  group: ty-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   ty:
     runs-on: ubuntu-24.04

AGENTS.md

@@ -18,7 +18,8 @@ Python `>=3.10` is required (CI tests 3.10–3.14).
 ## Common commands
 
 - Run the CLI locally: `cloudsmith ...` (console_script) or `python -m cloudsmith_cli ...`.
-- Run tests: `pytest` (configured via `pyproject.toml` `[tool.pytest.ini_options]` — adds `--cov=cloudsmith_cli`).
+- Run the unit tests: `pytest -m "not integration"`. Run with coverage: `pytest --cov=cloudsmith_cli`.
+- Run the live-service tests: `pytest -m integration` (requires the `PYTEST_CLOUDSMITH_*` environment variables). Mark each test that calls the live Cloudsmith service with `@pytest.mark.integration`.
 - Run a single test: `pytest cloudsmith_cli/cli/tests/test_push.py::TestClass::test_name` or by node id / `-k <expr>`.
 - Lint/format (all run via pre-commit): `pre-commit run --all-files`. Individual tools: `black .`, `isort .`, `flake8 --config=.flake8`, `pylint --rcfile=.pylintrc <path>`, `pyupgrade --py310-plus <files>`.
 - Release: `bumpversion <major|minor|revision>` then `git push origin <tag>`. The `VERSION` symlink in repo root points at `cloudsmith_cli/data/VERSION`.
@@ -59,7 +60,7 @@ Three auth paths feed `core.api.init.initialise_api`:
 
 ## Tests
 
-Tests live alongside code: `cloudsmith_cli/cli/tests/` and `cloudsmith_cli/core/tests/`. The CLI tests use Click's `CliRunner`; API tests stub HTTP with `httpretty` and freeze time with `freezegun`. `bin/` and `.venv/` are excluded from pytest discovery (`norecursedirs` in `setup.cfg`).
+Tests live alongside code: `cloudsmith_cli/cli/tests/` and `cloudsmith_cli/core/tests/`. The CLI tests use Click's `CliRunner`; API tests stub HTTP with `httpretty` and freeze time with `freezegun`. `bin/` and `.venv/` are excluded from pytest discovery (`norecursedirs` in `pyproject.toml`). Tests that call the live Cloudsmith service carry the `integration` marker.
 
 ## Style notes specific to this repo
 

cloudsmith_cli/cli/tests/commands/conftest.py

@@ -1,5 +1,13 @@
 import pytest
 
+FAKE_API_HOST = "https://api.example.com"
+
+
+@pytest.fixture()
+def set_fake_api_host_env_var(monkeypatch):
+    """Set CLOUDSMITH_API_HOST to a host that no test can reach."""
+    monkeypatch.setenv("CLOUDSMITH_API_HOST", FAKE_API_HOST)
+
 
 class MockToken:
     """Mock Token object with the properties needed for testing."""

cloudsmith_cli/cli/tests/commands/policy/test_deny.py

@@ -74,6 +74,7 @@ def assert_output_matches_policy_config(output, config_file_path):
 
 
 @pytest.mark.usefixtures("set_api_key_env_var", "set_api_host_env_var")
+@pytest.mark.integration
 def test_deny_policy_commands(runner, organization, tmp_path):
     """Test CRUD operations for deny policies."""
     # Generate the deny policy configuration file.

cloudsmith_cli/cli/tests/commands/policy/test_licence.py

@@ -86,6 +86,7 @@ def assert_output_matches_policy_config(output, config_file_path):
 
 
 @pytest.mark.usefixtures("set_api_key_env_var", "set_api_host_env_var")
+@pytest.mark.integration
 def test_license_policy_commands(runner, organization, tmp_path):
     """Test CRUD operations for license policies."""
 

cloudsmith_cli/cli/tests/commands/policy/test_vulnerability.py

@@ -88,6 +88,7 @@ def assert_output_matches_policy_config(output, config_file_path):
 
 
 @pytest.mark.usefixtures("set_api_key_env_var", "set_api_host_env_var")
+@pytest.mark.integration
 def test_vulnerability_policy_commands(runner, organization, tmp_path):
     """Test CRUD operations for vulnerability policies."""
 

cloudsmith_cli/cli/tests/commands/test_check.py

@@ -4,9 +4,9 @@
 
 from ....cli.commands.check import check
 from ....cli.tests.utils import random_str
+from .conftest import FAKE_API_HOST
 
 
-@pytest.mark.usefixtures("set_api_host_env_var")
 class TestCheckServiceCommand:
     @pytest.mark.parametrize(
         "service_version,api_binding_version",
@@ -16,8 +16,9 @@ class TestCheckServiceCommand:
             ("1.0.0", "2.0.0"),
         ],
     )
+    @pytest.mark.usefixtures("set_fake_api_host_env_var")
     def test_check_service_command_output(
-        self, runner, api_host, service_version, api_binding_version
+        self, runner, service_version, api_binding_version
     ):
         """Unit test the command output given different combinations of service/binding version."""
         service_status = random_str()
@@ -39,7 +40,7 @@ def test_check_service_command_output(
 
         assert output[0] == "Retrieving service status ... OK"
         assert output[1] == ""
-        assert output[2] == f"The service endpoint is: {api_host}"
+        assert output[2] == f"The service endpoint is: {FAKE_API_HOST}"
         assert output[3] == f"The service status is:   {service_status}"
         assert (
             output[4]
@@ -52,6 +53,8 @@ def test_check_service_command_output(
             else "The API library used by this CLI tool seems to be up-to-date."
         )
 
+    @pytest.mark.integration
+    @pytest.mark.usefixtures("set_api_host_env_var")
     def test_check_service_command(self, runner, api_host):
         """Integration test the `cloudsmith check service` command (actually hit the API)."""
         result = runner.invoke(check, args="service", catch_exceptions=False)

cloudsmith_cli/cli/tests/commands/test_credential_helper.py

@@ -234,6 +234,11 @@ def _cache_dir(tmp_path, monkeypatch):
     )
 
 
+@pytest.fixture(autouse=True)
+def _no_retry_sleep(monkeypatch):
+    monkeypatch.setattr("time.sleep", lambda _: None)
+
+
 @pytest.mark.parametrize(
     "status,expect_domains,expect_cached",
     [

cloudsmith_cli/cli/tests/commands/test_entitlements.py

@@ -7,6 +7,7 @@
 
 
 @pytest.mark.usefixtures("set_api_key_env_var", "set_api_host_env_var")
+@pytest.mark.integration
 def test_entitlements_list_with_show_all(runner, organization, tmp_repository):
     """Test listing entitlements with --show-all flag."""
     org_repo = f"{organization}/{tmp_repository['slug']}"

cloudsmith_cli/cli/tests/commands/test_login.py

@@ -4,6 +4,7 @@
 
 
 @pytest.mark.usefixtures("set_api_host_env_var")
+@pytest.mark.integration
 class TestLoginCommand:
     def test_login_via_prompt(self, runner, username, password, api_key):
         """Test that a user can `cloudsmith login` with interactive prompts."""

cloudsmith_cli/cli/tests/commands/test_package_commands.py

@@ -12,6 +12,7 @@
 
 
 @pytest.mark.usefixtures("set_api_key_env_var", "set_api_host_env_var")
+@pytest.mark.integration
 @pytest.mark.parametrize(
     "filesize",
     [
@@ -88,6 +89,7 @@ def test_push_and_delete_raw_package(
 
 
 @pytest.mark.usefixtures("set_api_key_env_var", "set_api_host_env_var")
+@pytest.mark.integration
 def test_list_packages_with_sort(runner, organization, tmp_repository, tmp_path):
     """Test listing packages with different sort options."""
     org_repo = f"{organization}/{tmp_repository['slug']}"

cloudsmith_cli/cli/tests/commands/test_repos.py

@@ -74,6 +74,7 @@ def assert_output_is_equal_to_repo_config(output, organisation, repo_config_file
 
 
 @pytest.mark.usefixtures("set_api_key_env_var", "set_api_host_env_var")
+@pytest.mark.integration
 def test_repos_commands(runner, organization, tmp_path):
     """Test CRUD operations for repositories."""