Reproducible/deployable environment for LiteLLM proxy/admin server

.gitattributes

@@ -0,0 +1,2 @@
+# SCM syntax highlighting & preventing 3-way merges
+pixi.lock merge=binary linguist-language=YAML linguist-generated=true -diff

.gitignore

@@ -23,3 +23,6 @@ notifiers/*
 
 # This installation's own settings: gateways, credentials, what is available here.
 lab.env
+# pixi environments
+.pixi/*
+!.pixi/config.toml

docs/llm.md

@@ -30,20 +30,29 @@ SDK's, unchanged.
 format and routes it to any provider it supports. Running it in front of an
 OpenAI-style backend lets a campaign run on that backend through the same SDK path.
 
-Install it in its own environment, since its proxy extra pulls in a large dependency
-set:
+The repository's pixi environment includes LiteLLM, its proxy dependencies, PostgreSQL,
+and Prisma. From the repository root, start the proxy and its local database with:
 
 ```
-python -m venv ~/venvs/litellm && ~/venvs/litellm/bin/pip install "litellm[proxy]" "fastapi<0.140.7"
+pixi install
+pixi run litellm-proxy-start
 ```
 
-The FastAPI cap is needed as of LiteLLM 1.97.0. LiteLLM imports
-`fastapi.dependencies.utils.get_flat_dependant`, which FastAPI removed in 0.140.7, and
-LiteLLM declares `fastapi>=0.136.3,<1.0` — no upper bound below the removal — so an
-uncapped install takes a FastAPI the proxy cannot import. The working range is 0.136.3
-to 0.140.6. Drop the cap once a LiteLLM release no longer needs it; keep it inside that
-range rather than pinning further back, since below 0.136.3 is outside what LiteLLM
-supports.
+This initializes PostgreSQL under `scratch/litellm-postgres`, uses TCP port 5433 for the
+database, and serves the proxy on port 4000. The database is local and gitignored. 
+The first start generates LiteLLM's Prisma client and runs
+its migrations. Stop the proxy cleanly with Ctrl-C in the foreground terminal. The launcher stops
+LiteLLM first and PostgreSQL second. You can also stop both services from another
+terminal with:
+
+```
+pixi run litellm-proxy-stop
+```
+
+Set `LITELLM_MASTER_KEY` and `LITELLM_SALT_KEY` before starting when using this beyond
+local development. The defaults are intentionally only suitable for a local machine.
+Use `LITELLM_PORT`, `LITELLM_DB_PORT`, or `LITELLM_PGDATA` to change the proxy port,
+database port, or database directory.
 
 Write a config naming the upstream model, its endpoint, and the key to reach it:
 

framework/run_litellm.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+# Run LiteLLM with a local Postgres database managed by pixi.
+#
+# Usage:
+#   pixi run litellm-proxy-start  # start PostgreSQL and the proxy
+#   pixi run litellm-proxy-stop   # stop the proxy, then PostgreSQL
+#
+# Press Ctrl-C in the start command to stop the proxy before PostgreSQL.
+# Override LITELLM_PORT, LITELLM_DB_PORT, LITELLM_MASTER_KEY,
+# LITELLM_SALT_KEY, or LITELLM_PGDATA when needed.
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
+PGDATA="${LITELLM_PGDATA:-$ROOT_DIR/scratch/litellm-postgres}"
+PGLOG="${LITELLM_PGLOG:-$ROOT_DIR/scratch/litellm-postgres.log}"
+PIDFILE="${LITELLM_PIDFILE:-$ROOT_DIR/scratch/litellm-proxy.pid}"
+DB_PORT="${LITELLM_DB_PORT:-5433}"
+PROXY_PORT="${LITELLM_PORT:-4000}"
+DB_USER="${LITELLM_DB_USER:-litellm}"
+DB_NAME="${LITELLM_DB_NAME:-litellm}"
+DB_URL="${DATABASE_URL:-postgresql://${DB_USER}@127.0.0.1:${DB_PORT}/${DB_NAME}}"
+
+stop_proxy() {
+    local pid=""
+    if [ -f "$PIDFILE" ]; then
+        pid="$(cat "$PIDFILE")"
+    fi
+    if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then
+        kill -TERM "$pid"
+        for _ in $(seq 1 30); do
+            kill -0 "$pid" 2>/dev/null || break
+            sleep 1
+        done
+        if kill -0 "$pid" 2>/dev/null; then
+            echo "LiteLLM did not stop within 30 seconds" >&2
+            return 1
+        fi
+    fi
+    rm -f "$PIDFILE"
+}
+
+stop_db() {
+    if [ -d "$PGDATA" ] && pg_ctl -D "$PGDATA" status >/dev/null 2>&1; then
+        pg_ctl -D "$PGDATA" -m fast stop
+    fi
+}
+
+if [ "${1:-start}" = "stop" ]; then
+    stop_proxy
+    stop_db
+    exit 0
+fi
+
+if [ "${1:-start}" != "start" ]; then
+    echo "usage: $0 [start|stop]" >&2
+    exit 2
+fi
+
+mkdir -p "$(dirname "$PGDATA")"
+if [ ! -f "$PGDATA/PG_VERSION" ]; then
+    initdb -D "$PGDATA" -U "$DB_USER" --auth=trust --no-instructions
+fi
+
+if pg_ctl -D "$PGDATA" status >/dev/null 2>&1; then
+    if ! pg_isready -h 127.0.0.1 -p "$DB_PORT" >/dev/null 2>&1; then
+        echo "Postgres data directory is running, but not on port $DB_PORT: $PGDATA" >&2
+        exit 1
+    fi
+else
+    pg_ctl -D "$PGDATA" -o "-p $DB_PORT -h 127.0.0.1" -l "$PGLOG" start
+fi
+
+until pg_isready -h 127.0.0.1 -p "$DB_PORT" >/dev/null 2>&1; do
+    sleep 1
+done
+
+if ! psql -h 127.0.0.1 -p "$DB_PORT" -U "$DB_USER" -d postgres -Atqc \
+    "SELECT 1 FROM pg_database WHERE datname = '$DB_NAME'" | grep -q '^1$'; then
+    createdb -h 127.0.0.1 -p "$DB_PORT" -U "$DB_USER" "$DB_NAME"
+fi
+
+SCHEMA="$(python -c 'import os, litellm; print(os.path.join(os.path.dirname(litellm.__file__), "proxy", "schema.prisma"))')"
+if [ ! -f "$(dirname "$SCHEMA")/client.py" ]; then
+    prisma generate --schema="$SCHEMA"
+fi
+
+export DATABASE_URL="$DB_URL"
+export LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY:-sk-1234}"
+export LITELLM_SALT_KEY="${LITELLM_SALT_KEY:-sk-local-development-only}"
+export STORE_MODEL_IN_DB="${STORE_MODEL_IN_DB:-True}"
+
+litellm --config "$ROOT_DIR/config.yaml" --port "$PROXY_PORT" &
+proxy_pid=$!
+printf '%s\n' "$proxy_pid" > "$PIDFILE"
+cleanup() {
+    trap - INT TERM EXIT
+    stop_proxy || true
+    stop_db || true
+}
+trap cleanup INT TERM EXIT
+
+set +e
+wait "$proxy_pid"
+proxy_status=$?
+set -e
+cleanup
+exit "$proxy_status"