diff --git a/.github/workflows/appmap-comparison-dogfood.yml b/.github/workflows/appmap-comparison-dogfood.yml new file mode 100644 index 0000000..f9d15f2 --- /dev/null +++ b/.github/workflows/appmap-comparison-dogfood.yml @@ -0,0 +1,258 @@ +name: Dogfood AppMap PR comparison + +on: + push: + branches: + - 'agent/**' + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: appmap-comparison-${{ github.ref }} + cancel-in-progress: true + +env: + COMPARISON_CLI_SHA: aab690a5c4c01802551993718b508718c1139c11 + COMPARISON_VIEWER_SHA: cfd3a0fef35b016cb0af20ee02ae46fd35414069 + COMPARISON_REVIEW_ACTION_SHA: 67060904f03c0c8a1492e610f845eefd29e53a78 + COMPARISON_SKILLS_SHA: 167db8d4b7eb56086c74a9ef9ffe7b2d3c8ab627 + +jobs: + comparison-bundle: + name: Record and compare base/head behavior + runs-on: ubuntu-latest + steps: + - name: Check out base revision + uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.base.sha || github.event.repository.default_branch }} + path: base + fetch-depth: 1 + persist-credentials: false + + - name: Check out head revision + uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + path: head + fetch-depth: 1 + persist-credentials: false + + - name: Check out frozen comparison CLI contract + uses: actions/checkout@v6 + with: + repository: tryingET/appmap-js + ref: ${{ env.COMPARISON_CLI_SHA }} + path: appmap-js + fetch-depth: 1 + persist-credentials: false + + - uses: actions/setup-node@v6 + with: + node-version: 22 + + - name: Install and build both appmap-node revisions + shell: bash + run: | + set -euo pipefail + corepack enable + (cd base && yarn install --immutable && yarn prepack) + (cd head && yarn install --immutable && yarn prepack) + + - name: Record control and visual scenarios + shell: bash + run: | + set -euo pipefail + mkdir -p comparison/appmaps + node head/test/behavioralComparison/record.mjs base control comparison/appmaps/control-base.appmap.json + node head/test/behavioralComparison/record.mjs head control comparison/appmaps/control-head.appmap.json + node head/test/behavioralComparison/record.mjs base before comparison/appmaps/demo-base.appmap.json + node head/test/behavioralComparison/record.mjs head after comparison/appmaps/demo-head.appmap.json + + - name: Build and test comparison contract + shell: bash + run: | + set -euo pipefail + corepack enable + cd appmap-js + test "$(git rev-parse HEAD)" = "$COMPARISON_CLI_SHA" + yarn install --immutable + yarn build + yarn workspace @appland/models test comparison --runInBand + yarn workspace @appland/appmap test sequenceDiagramCompare --runInBand + + - name: Sanitize recordings and build portable comparison bundles + shell: bash + run: | + set -euo pipefail + CLI="node appmap-js/packages/cli/built/cli.js" + BASE_SHA="$(git -C base rev-parse HEAD)" + HEAD_SHA="$(git -C head rev-parse HEAD)" + + $CLI sanitize comparison/appmaps/*.appmap.json + + $CLI sequence-diagram-compare \ + comparison/appmaps/control-base.appmap.json \ + comparison/appmaps/control-head.appmap.json \ + --scenario appmap-node-control \ + --base-revision "$BASE_SHA" \ + --head-revision "$HEAD_SHA" \ + --output-file comparison/control.compare.diff.sequence.json + + $CLI sequence-diagram-compare \ + comparison/appmaps/demo-base.appmap.json \ + comparison/appmaps/demo-head.appmap.json \ + --scenario added-authorization-call \ + --base-revision "$BASE_SHA" \ + --head-revision "$HEAD_SHA" \ + --output-file comparison/authorization.compare.diff.sequence.json + + - name: Verify and publish the frozen contract with the artifact + shell: bash + run: | + set -euo pipefail + node head/test/behavioralComparison/validate.mjs \ + comparison/control.compare.diff.sequence.json \ + comparison/authorization.compare.diff.sequence.json \ + comparison/summary.md + + mkdir -p comparison/contract/examples + cp appmap-js/packages/models/schema/comparison.schema.json comparison/contract/ + cp appmap-js/packages/models/schema/README.md comparison/contract/ + cp appmap-js/packages/models/schema/CHANGELOG.md comparison/contract/ + cp appmap-js/packages/models/schema/examples/*.json comparison/contract/examples/ + + BASE_SHA="$(git -C base rev-parse HEAD)" + HEAD_SHA="$(git -C head rev-parse HEAD)" + cat > comparison/contract-lock.json <> "$GITHUB_STEP_SUMMARY" + + - name: Upload interactive comparison artifact + uses: actions/upload-artifact@v6 + with: + name: appmap-pr-comparison-schema-v1 + path: comparison + retention-days: 14 + if-no-files-found: error + + vscode-viewer: + name: Build and exercise comparison viewer + needs: comparison-bundle + runs-on: ubuntu-latest + steps: + - name: Check out pinned VS Code comparison viewer + uses: actions/checkout@v6 + with: + repository: tryingET/vscode-appland + ref: ${{ env.COMPARISON_VIEWER_SHA }} + fetch-depth: 1 + persist-credentials: false + + - name: Verify the pinned viewer revision + run: test "$(git rev-parse HEAD)" = "$COMPARISON_VIEWER_SHA" + + - name: Download the generated comparison bundle + uses: actions/download-artifact@v8 + with: + name: appmap-pr-comparison-schema-v1 + path: dogfood-comparison + + - name: Install the generated bundle as the editor test fixture + shell: bash + run: | + set -euo pipefail + target="test/fixtures/workspaces/project-diagram-diff/data/diff/minitest/Users_edit_unsuccessful_edit.compare.diff.sequence.json" + mkdir -p "$(dirname "$target")" + cp dogfood-comparison/authorization.compare.diff.sequence.json "$target" + printf 'authorize\n' > "$target.expect" + + - uses: actions/setup-node@v6 + with: + node-version: 18 + cache: yarn + + - name: Install, typecheck, and compile + run: | + corepack enable + yarn install --immutable + yarn pretest + yarn compile + + - name: Prepare Electron integration environment + run: | + yarn test:precache + yarn download-tools + + - name: Open the generated comparison bundle in the extension host + shell: bash + run: | + export XDG_RUNTIME_DIR=/run/user/$(id -u) + export DBUS_SESSION_BUS_ADDRESS=unix:path=$XDG_RUNTIME_DIR/bus + sudo mkdir -p "$XDG_RUNTIME_DIR" + sudo chown "$(id -u):$(id -g)" "$XDG_RUNTIME_DIR" + dbus-daemon --session --address="$DBUS_SESSION_BUS_ADDRESS" --fork --nopidfile + xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- \ + yarn test:integration test/integration/appmapEditor/openSequenceComparison.test.ts + + - name: Package dogfood VSIX + run: yarn vsce package --out appmap-comparison-schema-v1-dogfood.vsix + + - name: Upload comparison viewer + uses: actions/upload-artifact@v6 + with: + name: appmap-comparison-schema-v1-vsix + path: appmap-comparison-schema-v1-dogfood.vsix + retention-days: 14 + if-no-files-found: error + + review-publisher: + name: Verify review artifact publisher + runs-on: ubuntu-latest + steps: + - name: Check out pinned review-action publisher + uses: actions/checkout@v6 + with: + repository: tryingET/review-action + ref: ${{ env.COMPARISON_REVIEW_ACTION_SHA }} + path: review-action + fetch-depth: 1 + persist-credentials: false + + - name: Check out pinned comparison skill + uses: actions/checkout@v6 + with: + repository: tryingET/skills + ref: ${{ env.COMPARISON_SKILLS_SHA }} + path: comparison-skills + fetch-depth: 1 + persist-credentials: false + + - name: Run offline action harness + run: | + test "$(git -C review-action rev-parse HEAD)" = "$COMPARISON_REVIEW_ACTION_SHA" + test "$(git -C comparison-skills rev-parse HEAD)" = "$COMPARISON_SKILLS_SHA" + (cd review-action && test/run.sh) + + - name: Validate publisher and skill contract + shell: bash + run: | + set -euo pipefail + test -s comparison-skills/appmap-comparison/SKILL.md + grep -q 'appmap.comparison' comparison-skills/appmap-comparison/SKILL.md + grep -q 'elementIds' comparison-skills/appmap-comparison/SKILL.md + grep -q 'appmap-comparison' review-action/scripts/install-skills.sh + grep -q 'upload-comparison-artifact' review-action/action.yml diff --git a/test/behavioralComparison/record.mjs b/test/behavioralComparison/record.mjs new file mode 100644 index 0000000..29de2dc --- /dev/null +++ b/test/behavioralComparison/record.mjs @@ -0,0 +1,99 @@ +#!/usr/bin/env node + +import assert from 'node:assert'; +import { spawnSync } from 'node:child_process'; +import { cp, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; + +const [targetRepositoryArg, variant, outputFileArg] = process.argv.slice(2); +if (!targetRepositoryArg || !variant || !outputFileArg) { + console.error('usage: record.mjs '); + process.exit(2); +} +if (!['control', 'before', 'after'].includes(variant)) { + console.error(`unknown variant: ${variant}`); + process.exit(2); +} + +const targetRepository = resolve(targetRepositoryArg); +const outputFile = resolve(outputFileArg); +const appmapNode = join(targetRepository, 'bin', 'appmap-node.js'); +const fixtureDir = await mkdtemp(join(tmpdir(), 'appmap-pr-comparison-')); + +async function findAppMaps(directory) { + const matches = []; + const visit = async (current) => { + for (const entry of await readdir(current, { withFileTypes: true })) { + const path = join(current, entry.name); + if (entry.isDirectory()) await visit(path); + else if (entry.name.endsWith('.appmap.json')) matches.push(path); + } + }; + await visit(directory); + return matches; +} + +function fixtureSource(selectedVariant) { + const authorizationCall = + selectedVariant === 'after' + ? ` +function authorize(request) { + return request.userId > 0; +} +` + : ''; + const authorizationUse = selectedVariant === 'after' ? 'authorize(request);' : ''; + + return ` +function parseRequest() { + return { userId: 42 }; +} +${authorizationCall} +function loadUser(request) { + return { id: request.userId, role: 'reader' }; +} + +const request = parseRequest(); +${authorizationUse} +const user = loadUser(request); +console.log(user.id); +`; +} + +try { + await writeFile( + join(fixtureDir, 'appmap.yml'), + `name: appmap-pr-comparison\nlanguage: javascript\nappmap_dir: tmp/appmap\npackages:\n - path: .\n exclude:\n - node_modules\n` + ); + await writeFile(join(fixtureDir, 'index.mjs'), fixtureSource(variant)); + + const result = spawnSync(process.execPath, [appmapNode, 'index.mjs'], { + cwd: fixtureDir, + env: { + ...process.env, + APPMAP_RECORDER_PROCESS_ALWAYS: 'true', + TZ: 'UTC', + }, + stdio: 'inherit', + }); + assert.equal(result.status, 0, `appmap-node exited with status ${result.status}`); + + const maps = await findAppMaps(join(fixtureDir, 'tmp')); + const processMaps = []; + for (const path of maps) { + const appmap = JSON.parse(await readFile(path, 'utf8')); + if (appmap.metadata?.recorder?.type === 'process') processMaps.push(path); + } + assert.equal( + processMaps.length, + 1, + `expected one process AppMap, found ${processMaps.length}: ${maps.join(', ')}` + ); + + await mkdir(dirname(outputFile), { recursive: true }); + await cp(processMaps[0], outputFile); + console.log(`Recorded ${variant} behavior to ${outputFile}`); +} finally { + await rm(fixtureDir, { recursive: true, force: true }); +} diff --git a/test/behavioralComparison/validate.mjs b/test/behavioralComparison/validate.mjs new file mode 100644 index 0000000..948d3d1 --- /dev/null +++ b/test/behavioralComparison/validate.mjs @@ -0,0 +1,69 @@ +#!/usr/bin/env node + +import assert from 'node:assert'; +import { readFile, writeFile } from 'node:fs/promises'; + +const [controlFile, demoFile, summaryFile] = process.argv.slice(2); +if (!controlFile || !demoFile || !summaryFile) { + console.error('usage: validate.mjs '); + process.exit(2); +} + +const control = JSON.parse(await readFile(controlFile, 'utf8')); +const demo = JSON.parse(await readFile(demoFile, 'utf8')); +const changeIdPattern = /^chg_[0-9a-f]{20}(?:_[2-9][0-9]*|_[1-9][0-9]+)?$/; + +for (const bundle of [control, demo]) { + assert.equal(bundle.kind, 'appmap.comparison'); + assert.equal(bundle.schemaVersion, 1); + assert.equal(bundle.capabilities?.views?.sequence, 1); + assert(bundle.scenario?.id); + assert(bundle.recordings?.base && bundle.recordings?.head); + assert(bundle.views?.sequence?.base?.actors && bundle.views.sequence.base.rootActions); + assert(bundle.views?.sequence?.head?.actors && bundle.views.sequence.head.rootActions); + assert(bundle.views?.sequence?.diff?.actors && bundle.views.sequence.diff.rootActions); + assert(Array.isArray(bundle.views.sequence.alignment?.actorOrder)); + assert(Array.isArray(bundle.changes)); + assert.equal(new Set(bundle.changes.map((change) => change.id)).size, bundle.changes.length); + bundle.changes.forEach((change) => { + assert.match(change.id, changeIdPattern); + assert(change.views?.sequence); + const references = [change.views.sequence.base, change.views.sequence.head] + .filter(Boolean) + .flatMap((reference) => [reference.eventIds, reference.elementIds]) + .filter(Boolean); + assert(references.length > 0, `change ${change.id} has no navigable sequence reference`); + }); +} + +assert.equal( + control.changes.length, + 0, + `control recording drifted unexpectedly: ${JSON.stringify(control.changes, null, 2)}` +); +assert(demo.changes.length > 0, 'demo comparison should contain a visible behavioral change'); +assert( + demo.changes.some((change) => { + const name = change.details?.name; + const searchable = [change.summary, name?.before, name?.after] + .filter(Boolean) + .join(' ') + .toLowerCase(); + return change.kind === 'call-added' && searchable.includes('authorize'); + }), + `expected an added authorize call: ${JSON.stringify(demo.changes, null, 2)}` +); + +const summary = + `# AppMap PR comparison dogfood\n\n` + + `- Contract: **appmap.comparison schema v1**.\n` + + `- Control scenario: **no runtime drift** across the base and PR builds.\n` + + `- Visual scenario: **${demo.changes.length} semantic change(s)** detected.\n` + + `- Added authorization call: **confirmed**.\n` + + `- Change IDs: **deterministic, non-positional hashes**.\n` + + `- Structural changes without events: **retained through element IDs**.\n` + + `- Artifact: download \`appmap-pr-comparison\` and open either ` + + `\`*.compare.diff.sequence.json\` file with the companion VS Code extension PR.\n`; + +await writeFile(summaryFile, summary); +console.log(summary);