Implement Stage 9 v2 metadata catalog and preview routes

.github/scripts/cloud_run_env.sh

@@ -43,6 +43,7 @@ cloud_run_set_defaults() {
   CLOUD_RUN_RUNTIME_CACHE_URL_SECRET="${CLOUD_RUN_RUNTIME_CACHE_URL_SECRET:-policyengine-api-prod-runtime-cache-url:latest}"
   CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET="${CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET:-policyengine-api-prod-runtime-cache-ca:latest}"
   CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT="${CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT:-production}"
+  V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE="${V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE:-}"
   CLOUD_RUN_VPC_NETWORK="${CLOUD_RUN_VPC_NETWORK:-default}"
   CLOUD_RUN_VPC_SUBNET="${CLOUD_RUN_VPC_SUBNET:-default}"
   CLOUD_RUN_VPC_EGRESS="${CLOUD_RUN_VPC_EGRESS:-private-ranges-only}"
@@ -79,6 +80,7 @@ cloud_run_set_defaults() {
   export CLOUD_RUN_RUNTIME_CACHE_URL_SECRET
   export CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET
   export CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT
+  export V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE
   export CLOUD_RUN_VPC_NETWORK
   export CLOUD_RUN_VPC_SUBNET
   export CLOUD_RUN_VPC_EGRESS

.github/scripts/deploy_cloud_run_candidate.sh

@@ -29,6 +29,7 @@ env_vars=(
   "RUNTIME_CACHE_SERVICE=api"
   "V2_SUPABASE_PROJECT_REF=${V2_SUPABASE_PROJECT_REF}"
   "V2_SUPABASE_ENVIRONMENT=${V2_SUPABASE_ENVIRONMENT}"
+  "V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE=${V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE}"
 )
 
 if [[ -n "${OLD_SIMULATION_GATEWAY_URL:-}" ]]; then

.github/scripts/migrate_v2_metadata_schema.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+set +x
+
+uv run alembic -c alembic-v2.ini upgrade head
+uv run alembic -c alembic-v2.ini current --check-heads
+uv run alembic -c alembic-v2.ini check

.github/scripts/validate_cloud_run_deploy_env.sh

@@ -36,6 +36,7 @@ cloud_run_require_env \
   CLOUD_RUN_VPC_EGRESS \
   V2_SUPABASE_PROJECT_REF \
   V2_SUPABASE_ENVIRONMENT \
+  V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE \
   SIM_ENTRYPOINT \
   ROUTE_IMPL_HEALTH \
   ROUTE_IMPL_SPECIFICATION \

.github/workflows/alembic-v2-check.yml

@@ -1,12 +1,12 @@
-name: Alembic v2 and runtime-cache checks
+name: Alembic v2 schema checks
 
 on:
   workflow_call:
   workflow_dispatch:
 
 jobs:
-  postgres-redis-lifecycle:
-    name: V2 Postgres and Redis lifecycle
+  postgres-lifecycle:
+    name: V2 Postgres migration lifecycle
     runs-on: ubuntu-latest
     services:
       postgres:
@@ -22,19 +22,9 @@ jobs:
           --health-interval=5s
           --health-timeout=5s
           --health-retries=20
-      redis:
-        image: redis:7.2-alpine
-        ports:
-          - 6379:6379
-        options: >-
-          --health-cmd="redis-cli ping"
-          --health-interval=5s
-          --health-timeout=5s
-          --health-retries=20
     env:
       V2_MIGRATION_DATABASE_URL: postgresql+psycopg://postgres:policyengine_v2_test@127.0.0.1:5432/policyengine_v2_alembic_test
       V2_ALEMBIC_DISPOSABLE_TEST: "1"
-      RUNTIME_CACHE_TEST_URL: redis://127.0.0.1:6379/0
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -52,5 +42,3 @@ jobs:
         run: uv run alembic -c alembic-v2.ini current --check-heads
       - name: Require no ungenerated v2 schema or data operations
         run: uv run alembic -c alembic-v2.ini check
-      - name: Test real Redis cross-instance semantics
-        run: uv run pytest -q tests/integration/test_runtime_cache_redis.py

.github/workflows/pr.yml

@@ -55,8 +55,13 @@ jobs:
     name: Alembic v1 qualification
     uses: ./.github/workflows/alembic-v1-check.yml
   alembic-v2-check:
-    name: Alembic v2 and Redis qualification
+    name: Alembic v2 qualification
     uses: ./.github/workflows/alembic-v2-check.yml
+  v2-integration-check:
+    name: V2 integration qualification
+    uses: ./.github/workflows/v2-integration-check.yml
+    secrets:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
   check-changelog:
     name: Check changelog fragment
     runs-on: ubuntu-latest

.github/workflows/push.yml

@@ -34,10 +34,17 @@ jobs:
     uses: ./.github/workflows/alembic-v1-check.yml
 
   alembic-v2-check:
-    name: Alembic v2 and Redis qualification
+    name: Alembic v2 qualification
     if: github.repository == 'PolicyEngine/policyengine-api'
     uses: ./.github/workflows/alembic-v2-check.yml
 
+  v2-integration-check:
+    name: V2 integration qualification
+    if: github.repository == 'PolicyEngine/policyengine-api'
+    uses: ./.github/workflows/v2-integration-check.yml
+    secrets:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
   ensure-staging-model-version-aligns-with-sim-api:
     name: Ensure staging model version aligns with simulation API
     runs-on: ubuntu-latest
@@ -58,7 +65,11 @@ jobs:
 
   versioning:
     name: Update versioning
-    needs: [lint, alembic-v1-check, alembic-v2-check]
+    needs:
+      - lint
+      - alembic-v1-check
+      - alembic-v2-check
+      - v2-integration-check
     if: |
       (github.repository == 'PolicyEngine/policyengine-api')
       && !(github.event.head_commit.message == 'Update PolicyEngine API')
@@ -101,6 +112,7 @@ jobs:
       - lint
       - alembic-v1-check
       - alembic-v2-check
+      - v2-integration-check
     if: |
       (github.repository == 'PolicyEngine/policyengine-api')
       && (github.event.head_commit.message == 'Update PolicyEngine API')
@@ -153,13 +165,27 @@ jobs:
         if: always()
         run: bash .github/scripts/stop_cloud_sql_proxy.sh
 
+  seed-v2-staging-database:
+    name: Seed staging v2 database
+    needs:
+      - publish-git-tag
+      - migrate-v1-cloud-sql
+    if: |
+      (github.repository == 'PolicyEngine/policyengine-api')
+      && (github.event.head_commit.message == 'Update PolicyEngine API')
+    uses: ./.github/workflows/seed-v2-database.yml
+    with:
+      deployment_environment: staging
+    secrets: inherit
+
   deploy-staging:
     name: Deploy staging App Engine version
     runs-on: ubuntu-latest
     needs:
       - ensure-staging-model-version-aligns-with-sim-api
       - publish-git-tag
       - migrate-v1-cloud-sql
+      - seed-v2-staging-database
     if: |
       (github.repository == 'PolicyEngine/policyengine-api')
       && (github.event.head_commit.message == 'Update PolicyEngine API')
@@ -255,6 +281,7 @@ jobs:
       - ensure-staging-model-version-aligns-with-sim-api
       - publish-git-tag
       - migrate-v1-cloud-sql
+      - seed-v2-staging-database
     if: |
       (github.repository == 'PolicyEngine/policyengine-api')
       && (github.event.head_commit.message == 'Update PolicyEngine API')
@@ -272,6 +299,7 @@ jobs:
       CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET: policyengine-api-staging-runtime-cache-ca:latest
       V2_SUPABASE_PROJECT_REF: ${{ vars.V2_SUPABASE_PROJECT_REF }}
       V2_SUPABASE_ENVIRONMENT: ${{ vars.V2_SUPABASE_ENVIRONMENT }}
+      V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE: ${{ secrets.V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE }}
       # Staging stays scale-to-zero, single instance: it exists for per-push
       # validation, not capacity. Both the revision-level (--min-instances) and
       # service-level (--min) floors are 0.
@@ -404,7 +432,7 @@ jobs:
       - name: Install staging test dependencies
         run: pip install pytest httpx
       - name: Run staging smoke test
-        run: python -m pytest tests/integration/test_cloud_run_candidate.py tests/integration/test_live_calculate.py tests/integration/test_live_economy.py tests/integration/test_live_budget_window_cache.py -v
+        run: python -m pytest tests/integration/test_cloud_run_candidate.py tests/integration/test_live_v2_metadata.py tests/integration/test_live_calculate.py tests/integration/test_live_economy.py tests/integration/test_live_budget_window_cache.py -v
         env:
           API_BASE_URL: ${{ needs.deploy-cloud-run-staging.outputs.url }}
           STAGING_API_TEST_PROBE_ID: cloud-run-${{ needs.deploy-cloud-run-staging.outputs.tag }}
@@ -491,10 +519,21 @@ jobs:
       - name: Check simulation API supports PolicyEngine bundle
         run: bash .github/check-policyengine-bundle-supported.sh
 
+  seed-v2-production-database:
+    name: Seed production v2 database
+    needs: ensure-production-model-version-aligns-with-sim-api
+    if: |
+      (github.repository == 'PolicyEngine/policyengine-api')
+      && (github.event.head_commit.message == 'Update PolicyEngine API')
+    uses: ./.github/workflows/seed-v2-database.yml
+    with:
+      deployment_environment: production
+    secrets: inherit
+
   deploy-production-candidate:
     name: Deploy production App Engine candidate
     runs-on: ubuntu-latest
-    needs: ensure-production-model-version-aligns-with-sim-api
+    needs: seed-v2-production-database
     if: |
       (github.repository == 'PolicyEngine/policyengine-api')
       && (github.event.head_commit.message == 'Update PolicyEngine API')
@@ -620,7 +659,7 @@ jobs:
   deploy-cloud-run-candidate:
     name: Deploy production Cloud Run candidate
     runs-on: ubuntu-latest
-    needs: ensure-production-model-version-aligns-with-sim-api
+    needs: seed-v2-production-database
     if: |
       (github.repository == 'PolicyEngine/policyengine-api')
       && (github.event.head_commit.message == 'Update PolicyEngine API')
@@ -638,6 +677,7 @@ jobs:
       CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET: policyengine-api-prod-runtime-cache-ca:latest
       V2_SUPABASE_PROJECT_REF: ${{ vars.V2_SUPABASE_PROJECT_REF }}
       V2_SUPABASE_ENVIRONMENT: ${{ vars.V2_SUPABASE_ENVIRONMENT }}
+      V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE: ${{ secrets.V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE }}
       # Sized by the Stage 2 qualification and the PR 4 host cutover — rationale
       # and numbers in docs/migration/cloud-run-operations.md ("Runtime shape and
       # scaling"). Warm capacity is expressed service-level (--min); the
@@ -692,7 +732,7 @@ jobs:
       - name: Install Cloud Run smoke test dependencies
         run: pip install pytest httpx
       - name: Run Cloud Run candidate smoke tests
-        run: python -m pytest tests/integration/test_cloud_run_candidate.py -v
+        run: python -m pytest tests/integration/test_cloud_run_candidate.py tests/integration/test_live_v2_metadata.py -v
         env:
           API_BASE_URL: ${{ steps.candidate.outputs.url }}
           STAGING_API_TEST_PROBE_ID: cloud-run-${{ steps.cloud_run.outputs.revision_tag }}

.github/workflows/seed-v2-database.yml

@@ -0,0 +1,44 @@
+name: Seed v2 database
+
+on:
+  workflow_call:
+    inputs:
+      deployment_environment:
+        description: GitHub environment containing one v2 database target
+        required: true
+        type: string
+  workflow_dispatch:
+    inputs:
+      deployment_environment:
+        description: GitHub environment containing one v2 database target
+        required: true
+        type: environment
+
+jobs:
+  seed:
+    name: Upgrade schema and seed v2 database
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    environment: ${{ inputs.deployment_environment }}
+    env:
+      V2_SUPABASE_PROJECT_REF: ${{ vars.V2_SUPABASE_PROJECT_REF }}
+      V2_SUPABASE_ENVIRONMENT: ${{ vars.V2_SUPABASE_ENVIRONMENT }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v6
+      - name: Install locked dependencies
+        run: uv sync --frozen
+      - name: Upgrade and verify the v2 schema
+        run: bash .github/scripts/migrate_v2_metadata_schema.sh
+        env:
+          V2_MIGRATION_DATABASE_URL: ${{ secrets.V2_MIGRATION_DATABASE_URL }}
+      - name: Seed and validate the v2 metadata catalog
+        run: uv run python scripts/publish_v2_metadata_catalog.py
+        env:
+          V2_DATA_WRITE_DATABASE_URL: ${{ secrets.V2_DATA_WRITE_DATABASE_URL }}

.github/workflows/v2-integration-check.yml

@@ -0,0 +1,73 @@
+name: V2 integration checks
+
+on:
+  workflow_call:
+    secrets:
+      CODECOV_TOKEN:
+        required: false
+  workflow_dispatch:
+
+jobs:
+  catalog-runtime-integration:
+    name: V2 PostgreSQL and Redis integration
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:17
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: policyengine_v2_test
+          POSTGRES_DB: policyengine_v2_alembic_test
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd="pg_isready -U postgres"
+          --health-interval=5s
+          --health-timeout=5s
+          --health-retries=20
+      redis:
+        image: redis:7.2-alpine
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd="redis-cli ping"
+          --health-interval=5s
+          --health-timeout=5s
+          --health-retries=20
+    env:
+      V2_MIGRATION_DATABASE_URL: postgresql+psycopg://postgres:policyengine_v2_test@127.0.0.1:5432/policyengine_v2_alembic_test
+      V2_ALEMBIC_DISPOSABLE_TEST: "1"
+      RUNTIME_CACHE_TEST_URL: redis://127.0.0.1:6379/0
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v6
+      - name: Install locked dependencies
+        run: uv sync --frozen
+      - name: Prepare the disposable v2 schema
+        run: uv run alembic -c alembic-v2.ini upgrade head
+      - name: Verify the installed PolicyEngine.py catalog interface
+        run: uv run coverage run --branch -m pytest -q tests/integration/test_v2_catalog_installed.py
+        env:
+          RUN_V2_CATALOG_COMPATIBILITY: "1"
+      - name: Test v2 metadata publication and resource routes
+        run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_catalog_publication.py tests/integration/test_v2_metadata_routes.py
+      - name: Qualify production-scale v2 metadata publication
+        run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_catalog_publication_qualification.py
+        env:
+          RUN_V2_CATALOG_PUBLICATION_QUALIFICATION: "1"
+      - name: Test real Redis cross-instance semantics
+        run: uv run coverage run -a --branch -m pytest -q tests/integration/test_runtime_cache_redis.py
+      - name: Export v2 integration coverage
+        run: uv run coverage xml -i -o coverage-v2.xml
+      - name: Upload v2 integration coverage to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          slug: PolicyEngine/policyengine-api
+          files: coverage-v2.xml

changelog.d/stage-9-v2-metadata.added.md

@@ -0,0 +1 @@
+Add the versioned API v2 metadata catalog and Cloud Run preview endpoints.