deploy-changed-samples.yml

name: Deploy Changed Samples

on:
  pull_request:
    paths:
      - "samples/**"

permissions:
  contents: read
  id-token: write

jobs:
  deploy_changed_samples:
    runs-on: ubuntu-latest
    concurrency:
      group: deploy_samples-group
    environment: deploy-changed-samples
    timeout-minutes: 90
    steps:
      - name: Checkout code
        uses: actions/checkout@v5
        with:
          fetch-depth: 0

      - name: Identify changed samples
        run: |
          git fetch origin main
          LAST_ANCESTOR=$(git merge-base HEAD origin/main)
          # This BYOC-only sample exceeds the retired Playground staging quota.
          git diff --name-only HEAD $LAST_ANCESTOR \
            | grep '^samples/' \
            | awk -F'/' '$2 != "self-improving" {print $1"/"$2}' \
            | sort \
            | uniq \
            | while read -r sample; do
                # A renamed or deleted sample still shows up in the diff, but
                # there is nothing left to deploy. Without this the job fails
                # with "no results found".
                if [ -d "$sample" ]; then echo "$sample"; fi
              done > changed_samples.txt
          if [ -s changed_samples.txt ]; then
            echo "should_continue=true" >> $GITHUB_ENV
            echo "The following samples have changed:"
            cat changed_samples.txt
          else
            echo "should_continue=false" >> $GITHUB_ENV
            echo "No samples have changed. Exiting..."
          fi
      - name: Install Golang
        uses: actions/setup-go@v5
        with:
          go-version-file: tools/testing/go.mod
          cache-dependency-path: |
            tools/testing/go.sum
        if: env.should_continue == 'true'

      - name: Build the test tool using Go
        run: |
          go mod tidy
          go build ./cmd/loadtest
        working-directory: tools/testing/
        if: env.should_continue == 'true'

      - name: Install Defang
        run: |
          eval "$(curl -fsSL s.defang.io/install)"
        if: env.should_continue == 'true'

      - name: Deploy changed samples to staging
        id: deploy-samples
        shell: bash # implies set -o pipefail, so pipe below will keep the exit code from loadtest
        # Dependabot-triggered pull_request runs get no access to Actions secrets
        # (GitHub restricts this the same way it does for fork PRs), so this step
        # would always fail here. Skip it so the required check reports "success"
        # (skipped) instead of blocking dependabot-automerge on every bump.
        if: env.should_continue == 'true' && github.actor != 'dependabot[bot]'
        env:
          FIXED_VERIFIER_PK: ${{ secrets.FIXED_VERIFIER_PK }}
          TEST_ALLOWED_HOSTS: ${{ secrets.TEST_ALLOWED_HOSTS }}
          TEST_ANTHROPIC_API_KEY: ${{ secrets.TEST_ANTHROPIC_API_KEY }}
          TEST_AWS_ACCESS_KEY: ${{ secrets.TEST_AWS_ACCESS_KEY }}
          TEST_AWS_SECRET_KEY: ${{ secrets.TEST_AWS_SECRET_KEY }}
          TEST_BETTER_AUTH_SECRET: ${{ secrets.TEST_SESSION_SECRET }}
          TEST_BOARD_PASSWORD: ${{ secrets.TEST_BOARD_PASSWORD }}
          TEST_DATABASE_HOST: ${{ secrets.TEST_DATABASE_HOST }}
          TEST_DATABASE_NAME: ${{ secrets.TEST_DATABASE_NAME }}
          TEST_DATABASE_PASSWORD: ${{ secrets.TEST_DATABASE_PASSWORD }}
          TEST_DATABASE_URL: ${{ secrets.TEST_DATABASE_URL }}
          TEST_DATABASE_USERNAME: ${{ secrets.TEST_DATABASE_USERNAME }}
          TEST_DB_SSL: ${{ secrets.TEST_DB_SSL }}
          TEST_DB_POSTGRESDB_PASSWORD: ${{ secrets.TEST_POSTGRES_PASSWORD }}
          TEST_DB_POSTGRESDB_SSL_ENABLED: ${{ secrets.TEST_DB_POSTGRESDB_SSL_ENABLED }}
          TEST_DB_POSTGRESDB_SSL_REJECT_UNAUTHORIZED: ${{ secrets.TEST_DB_POSTGRESDB_SSL_REJECT_UNAUTHORIZED }}
          TEST_GITHUB_TOKEN: ${{ secrets.TEST_GITHUB_TOKEN }}
          TEST_HASURA_GRAPHQL_ADMIN_SECRET: ${{ secrets.TEST_HASURA_GRAPHQL_ADMIN_SECRET }}
          TEST_HASURA_GRAPHQL_DATABASE_URL: ${{ secrets.TEST_HASURA_GRAPHQL_DATABASE_URL }}
          TEST_JUPYTER_TOKEN: ${{ secrets.TEST_JUPYTER_TOKEN }}
          TEST_HF_TOKEN: ${{ secrets.TEST_HF_TOKEN }}
          TEST_MB_DB_DBNAME: ${{ secrets.TEST_MB_DB_DBNAME }}
          TEST_MB_DB_HOST: ${{ secrets.TEST_MB_DB_HOST }}
          TEST_MB_DB_PASS: ${{ secrets.TEST_MB_DB_PASS }}
          TEST_MB_DB_PORT: ${{ secrets.TEST_MB_DB_PORT }}
          TEST_MB_DB_USER: ${{ secrets.TEST_MB_DB_USER }}
          TEST_MISTRAL_API_KEY: ${{ secrets.TEST_MISTRAL_API_KEY }}
          TEST_MODEL: ${{ secrets.TEST_MODEL }}
          TEST_MONGO_INITDB_ROOT_USERNAME: ${{ secrets.TEST_MONGO_INITDB_ROOT_USERNAME }}
          TEST_MONGO_INITDB_ROOT_PASSWORD: ${{ secrets.TEST_MONGO_INITDB_ROOT_PASSWORD }}
          TEST_LLM_MODEL: ${{ secrets.TEST_LLM_MODEL }}
          TEST_EMBEDDING_MODEL: ${{ secrets.TEST_EMBEDDING_MODEL }}
          TEST_N8N_ENCRYPTION_KEY: ${{ secrets.TEST_N8N_ENCRYPTION_KEY }}
          TEST_NC_DB: ${{ secrets.TEST_NC_DB }}
          TEST_NC_S3_ENDPOINT: ${{ secrets.TEST_NC_S3_ENDPOINT }}
          TEST_NC_S3_BUCKET_NAME: ${{ secrets.TEST_NC_S3_BUCKET_NAME }}
          TEST_NC_S3_REGION: ${{ secrets.TEST_NC_S3_REGION }}
          TEST_NC_S3_ACCESS_KEY: ${{ secrets.TEST_NC_S3_ACCESS_KEY }}
          TEST_NC_S3_ACCESS_SECRET: ${{ secrets.TEST_NC_S3_ACCESS_SECRET }}
          TEST_OPENAI_KEY: ${{ secrets.TEST_OPENAI_KEY }}
          TEST_POSTGRES_PASSWORD: ${{ secrets.TEST_POSTGRES_PASSWORD }}
          TEST_POSTGRES_SSL: ${{ secrets.TEST_POSTGRES_SSL }}
          TEST_PROJECT_HONEYPOT_KEY: ${{ secrets.TEST_PROJECT_HONEYPOT_KEY}}
          TEST_QUEUE: ${{ secrets.TEST_QUEUE }}
          TEST_SECRET_KEY: ${{ secrets.TEST_SECRET_KEY }}
          TEST_SECRET_KEY_BASE: ${{ secrets.TEST_SECRET_KEY_BASE }}
          TEST_SESSION_SECRET: ${{ secrets.TEST_SESSION_SECRET }}
          TEST_SLACK_CHANNEL_ID: ${{ secrets.TEST_SLACK_CHANNEL_ID }}
          TEST_SLACK_TOKEN: ${{ secrets.TEST_SLACK_TOKEN }}
          TEST_SHARED_SECRETS: ${{ secrets.TEST_SHARED_SECRETS}}
          TEST_SSL_MODE: ${{ secrets.TEST_SSL_MODE }}
          TEST_TAVILY_API_KEY: ${{ secrets.TEST_TAVILY_API_KEY }}
        run: |
          # Escape regex special characters and anchor patterns to ensure exact matching of sample names
          SAMPLES=$(sed 's|^samples/||' changed_samples.txt | awk '{gsub(/[.*+?^${}()|[\]\\]/, "\\\\&"); print "^" $0 "$"}' | paste -s -d ',' -)
          echo "Running tests for samples: $SAMPLES"
          mkdir output
          # concurrency is set to 10 to avoid exhausting our fargate vCPU limits when running
          # kaniko builds, which consume 4 vCPUs each. the limit is currently set to 60--6.5 of
          # which are used to run the staging stack. So floor((60-6.5)/4) = 13 is our upper limit
          ./tools/testing/loadtest -c fabric-staging.defang.dev:443 --timeout=15m --concurrency=10 -s $SAMPLES -o output --markdown=true | tee output/summary.log | grep -v '^\s*[-*]' # removes load sample log lines
      - name: Upload Output as Artifact
        uses: actions/upload-artifact@v4
        if: env.should_continue == 'true' && github.actor != 'dependabot[bot]' && (success() || steps.deploy-samples.outcome == 'failure') # Always upload result unless cancelled
        with:
          name: program-output
          path: output/**