perf(e2e): adopt a pre-baked toolkit venv instead of provisioning per suite

.github/workflows/e2e.yml

@@ -18,9 +18,13 @@ concurrency:
 
 jobs:
   e2e:
-    name: E2E (ExTester)
+    name: E2E (${{ matrix.group }})
     runs-on: ubuntu-latest
     timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        group: [environments, execution, files, workspace]
     env:
       # Keep ExTester's downloads (test VS Code, ChromeDriver, settings, screenshots) inside the
       # workspace so the artifact-upload paths are predictable. Both this and .test-extensions are
@@ -39,6 +43,7 @@ jobs:
           node-version-file: '.nvmrc'
 
       - name: Setup Python # interpreter the Deepnote environment is created from
+        id: python
         uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: '3.12'
@@ -72,29 +77,94 @@ jobs:
         run: npm run setup:e2e:deps
 
       - name: Cache pip wheel downloads
-        # Provisioning the Deepnote environment pip-installs the toolkit dependency tree into a
-        # fresh venv on first kernel connect — the bulk of the E2E runtime. Caching pip's wheel
-        # cache makes that warm on later runs (the installs use the cache; nothing passes
-        # --no-cache-dir). The key busts when the toolkit version / install set changes; the
-        # restore-keys prefix keeps the cache warm across unrelated changes, since pip's cache is
-        # additive.
+        # github.run_id in the key because actions/cache only writes on a miss — a bare content hash
+        # froze the entry at its first save and never picked up newer wheels.
         uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           path: ~/.cache/pip
-          key: pip-${{ runner.os }}-py312-${{ hashFiles('src/kernels/deepnote/types.ts', 'src/kernels/deepnote/deepnoteToolkitInstaller.node.ts') }}
+          key: pip-${{ runner.os }}-py${{ steps.python.outputs.python-version }}-${{ hashFiles('src/kernels/deepnote/types.ts', 'src/kernels/deepnote/deepnoteToolkitInstaller.node.ts') }}-${{ github.run_id }}
           restore-keys: |
-            pip-${{ runner.os }}-py312-
+            pip-${{ runner.os }}-py${{ steps.python.outputs.python-version }}-${{ hashFiles('src/kernels/deepnote/types.ts', 'src/kernels/deepnote/deepnoteToolkitInstaller.node.ts') }}-
+            pip-${{ runner.os }}-py${{ steps.python.outputs.python-version }}-
+
+      - name: Cache the managed Deepnote toolkit venv
+        # Keyed on the Python version too: pyvenv.cfg records the base interpreter, so a patch bump
+        # would leave a restored venv pointing at a path that is gone.
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: .venv-e2e
+          key: e2e-venv-${{ runner.os }}-py${{ steps.python.outputs.python-version }}-${{ hashFiles('src/kernels/deepnote/types.ts', 'src/kernels/deepnote/deepnoteToolkitInstaller.node.ts') }}
+
+      - name: Warm the managed venv
+        run: npm run setup:e2e:venv
 
       - name: Run E2E
         # VS Code launches with --no-sandbox (no AppArmor sysctl needed). Runs once; Mocha's retries:1
         # and rootHooks.ts (dismiss toasts between tests) handle flakiness in the single shared instance.
-        run: xvfb-run --auto-servernum --server-args='-screen 0 1920x1080x24' npm run test:e2e:prebuilt
+        run: xvfb-run --auto-servernum --server-args='-screen 0 1920x1080x24' npm run test:e2e:ci
+        env:
+          E2E_GROUP: ${{ matrix.group }}
+
+      # Written even when the tests fail, so a shard that ran is never mistaken for one that was
+      # never scheduled. verify-coverage collects these.
+      - name: Record the shard that ran
+        if: always()
+        run: mkdir -p shard-ran && touch "shard-ran/${{ matrix.group }}"
+
+      - name: Upload the shard record
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: e2e-ran-${{ matrix.group }}
+          path: shard-ran/
 
       - name: Upload failure screenshots
         if: failure()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: e2e-screenshots
+          name: e2e-screenshots-${{ matrix.group }}
           path: ${{ env.TEST_RESOURCES }}/screenshots/**/*.png
           if-no-files-found: ignore
           retention-days: 14
+
+  verify-coverage:
+    name: Verify shard coverage
+    needs: e2e
+    if: ${{ !cancelled() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Collect what each shard reported
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
+        with:
+          merge-multiple: true
+          path: shard-ran
+          pattern: e2e-ran-*
+
+      - name: Every suite directory must have been run
+        shell: bash
+        run: |
+          set -uo pipefail
+          echo "Shards that reported in: $(ls shard-ran 2>/dev/null | tr '\n' ' ')"
+
+          status=0
+          for dir in test/e2e/suite/*/; do
+            group="$(basename "${dir}")"
+            if [ ! -e "shard-ran/${group}" ]; then
+              echo "::error::test/e2e/suite/${group}/ was not run by any shard — add it to the matrix in this workflow."
+              status=1
+            fi
+          done
+
+          # A suite outside a group directory is never globbed by any shard either.
+          if compgen -G 'test/e2e/suite/*.e2e.test.ts' > /dev/null; then
+            echo "::error::Suites sit directly in test/e2e/suite/; move them into a group directory:"
+            ls test/e2e/suite/*.e2e.test.ts
+            status=1
+          fi
+
+          exit "${status}"

.vscodeignore

@@ -85,6 +85,7 @@ obj/**
 logs/**
 out/**
 .test-extensions/**
+.venv-e2e/**
 test-resources/**
 precommit.hook
 pythonFiles/.env

package.json

@@ -2672,9 +2672,10 @@
         "compile-e2e-watch": "tsc -p ./test/e2e/tsconfig.json --watch",
         "setup:e2e:vscode": "extest get-vscode -c max && extest get-chromedriver -c max",
         "setup:e2e:deps": "extest install-from-marketplace ms-python.python -e .test-extensions",
-        "setup:e2e": "npm run setup:e2e:vscode && npm run setup:e2e:deps",
-        "test:e2e": "extest setup-and-run \"./out/e2e/suite/*.e2e.test.js\" -c max -o ./test/e2e/settings.json -e .test-extensions -m ./test/e2e/.mocharc.js -i",
-        "test:e2e:prebuilt": "extest run-tests \"./out/e2e/suite/*.e2e.test.js\" -c max -o ./test/e2e/settings.json -e .test-extensions -m ./test/e2e/.mocharc.js",
+        "setup:e2e:venv": "node -e \"require('./out/e2e/helpers/venv').ensureManagedVenv()\"",
+        "setup:e2e": "npm run setup:e2e:vscode && npm run setup:e2e:deps && npm run setup:e2e:venv",
+        "test:e2e": "extest setup-and-run \"./out/e2e/suite/**/*.e2e.test.js\" -c max -o ./test/e2e/settings.json -e .test-extensions -m ./test/e2e/.mocharc.js -i",
+        "test:e2e:ci": "extest run-tests \"./out/e2e/suite/$E2E_GROUP/*.e2e.test.js\" -c max -o ./test/e2e/settings.json -e .test-extensions -m ./test/e2e/.mocharc.js",
         "test:unittests": "mocha --config ./build/.mocha.unittests.js.json ./out/**/*.unit.test.js",
         "test": "npm run test:unittests",
         "typecheck": "tsc -p ./ --noEmit",

src/kernels/deepnote/deepnoteToolkitInstaller.node.ts

@@ -18,7 +18,12 @@ import {
 } from '../../platform/errors/deepnoteKernelErrors';
 import { logger } from '../../platform/logging';
 import { PythonEnvironment } from '../../platform/pythonEnvironments/info';
-import { DEEPNOTE_TOOLKIT_VERSION, IDeepnoteToolkitInstaller, VenvAndToolkitInstallation } from './types';
+import {
+    DEEPNOTE_TOOLKIT_PACKAGES,
+    DEEPNOTE_TOOLKIT_VERSION,
+    IDeepnoteToolkitInstaller,
+    VenvAndToolkitInstallation
+} from './types';
 
 /**
  * Handles installation of the deepnote-toolkit Python package.
@@ -367,9 +372,7 @@ export class DeepnoteToolkitInstaller implements IDeepnoteToolkitInstaller {
                 'install',
                 '--upgrade',
                 `deepnote-toolkit[server]==${DEEPNOTE_TOOLKIT_VERSION}`,
-                'ipykernel',
-                'python-lsp-server[all]',
-                'deepnote-cli'
+                ...DEEPNOTE_TOOLKIT_PACKAGES
             ],
             token
         );

src/kernels/deepnote/types.ts

@@ -414,5 +414,7 @@ export interface IDeepnoteLspClientManager {
 }
 
 export const DEEPNOTE_TOOLKIT_VERSION = '2.1.1';
+
+export const DEEPNOTE_TOOLKIT_PACKAGES = ['ipykernel', 'python-lsp-server[all]', 'deepnote-cli'];
 export const DEEPNOTE_DEFAULT_PORT = 8888;
 export const DEEPNOTE_NOTEBOOK_TYPE = 'deepnote';

test/e2e/helpers/constants.ts

@@ -47,3 +47,15 @@ export const FOLDER_OK_RETRY_DELAY = 400;
 // Selectors that only exist inside the notebook output iframe (`#active-frame`),
 // so reading them cannot accidentally match the cell's source in the editor.
 export const OUTPUT_SELECTOR = '.output_container, .output, .rendered-output';
+
+// Shared by every suite so only one venv is ever provisioned; the deletion suite is the one
+// exception, since it deletes what it creates.
+export const SHARED_ENV_NAME = 'E2E Hello Env';
+
+// Where the managed venv lives, and the substring that identifies it in the interpreter quick pick.
+// The marker matches both the real directory and the `.venv` link the workspace exposes it through.
+export const PREBAKED_VENV_DIR_NAME = '.venv-e2e';
+export const PREBAKED_VENV_MARKER = '.venv';
+
+// Only elapses when the venv is missing, so it stays well under QUICK_PICK_TIMEOUT.
+export const PREBAKED_VENV_FILTER_TIMEOUT = 10_000;

test/e2e/helpers/deepnoteEnvironment.ts

@@ -7,6 +7,8 @@ import {
     KERNEL_CONNECT_TIMEOUT,
     MAX_CREATE_ATTEMPTS,
     OPTIONAL_PROMPT_TIMEOUT,
+    PREBAKED_VENV_MARKER,
+    PREBAKED_VENV_FILTER_TIMEOUT,
     QUICK_PICK_TIMEOUT
 } from './constants';
 import { dismissAllNotifications, waitForNotification } from './notifications';
@@ -16,14 +18,90 @@ import { tryOpenInputBox } from './quickInput';
 const CREATE_ENV_COMMAND = 'Deepnote: Create Environment';
 const SELECT_ENV_COMMAND = 'Deepnote: Select Environment for Notebook';
 
+/**
+ * Picks the managed venv, which the extension adopts instead of provisioning its own.
+ *
+ * `useManagedVenv` picks an interpreter outside it, so the extension creates and owns the venv —
+ * only the deletion suite needs that, since deleteEnvironment removes the directory for managed
+ * environments only.
+ */
+async function selectInterpreter(interpreterPick: InputBox, useManagedVenv: boolean): Promise<void> {
+    const driver = VSBrowser.instance.driver;
+
+    if (!useManagedVenv) {
+        await interpreterPick.setText(PREBAKED_VENV_MARKER);
+        // Wait for the *top row* to be the baked venv, not merely for the list to be non-empty:
+        // VS Code applies the filter asynchronously, so the stale unfiltered list is briefly still
+        // there and confirming against it would pick an arbitrary interpreter.
+        const filtered = await driver
+            .wait(async () => {
+                const picks = await interpreterPick.getQuickPicks();
+                if (picks.length === 0) {
+                    return undefined;
+                }
+                const first = `${await picks[0].getLabel()} ${(await picks[0].getDescription()) ?? ''}`;
+
+                return first.includes(PREBAKED_VENV_MARKER) ? picks[0] : undefined;
+            }, PREBAKED_VENV_FILTER_TIMEOUT)
+            .catch(() => undefined);
+
+        if (filtered) {
+            await interpreterPick.confirm();
+
+            return;
+        }
+
+        console.warn(
+            `[deepnote-e2e] no interpreter under ${PREBAKED_VENV_MARKER} was offered; falling back to ` +
+                'the first entry. The run will provision a venv and take several minutes longer — ' +
+                'check that `npm run setup:e2e:venv` ran.'
+        );
+        await interpreterPick.setText('');
+    }
+
+    const picks = await interpreterPick.getQuickPicks();
+    const labels = await Promise.all(picks.map(async (pick) => pick.getLabel()));
+    // "Not the baked venv" rather than "not any venv": in CI the only other interpreter is the one
+    // actions/setup-python installed, which is not a venv, so this resolves to it.
+    const wanted = labels.find((label) => !label.includes(PREBAKED_VENV_MARKER));
+
+    if (wanted) {
+        // Filter to it and accept with Enter, the same way the baked-venv branch does, rather than
+        // clicking a row or walking the list: rows intercept positional clicks, and an arrow-key walk
+        // silently lands on the wrong entry whenever the list scrolls or reorders under it.
+        await interpreterPick.setText(wanted);
+        const narrowed = await driver
+            .wait(async () => {
+                const filtered = await interpreterPick.getQuickPicks();
+
+                return filtered.length > 0 && !(await filtered[0].getLabel()).includes(PREBAKED_VENV_MARKER);
+            }, PREBAKED_VENV_FILTER_TIMEOUT)
+            .catch(() => false);
+
+        if (narrowed) {
+            await interpreterPick.confirm();
+
+            return;
+        }
+
+        await interpreterPick.setText('');
+    }
+
+    console.warn(
+        `[deepnote-e2e] no interpreter outside ${PREBAKED_VENV_MARKER} could be filtered to; ` +
+            `accepting the first entry. Offered: ${JSON.stringify(labels)}`
+    );
+    await interpreterPick.confirm();
+}
+
 /**
  * Drives `deepnote.environments.create`: pick interpreter -> name -> skip packages -> skip
  * description. Retries when the Python extension has not finished discovering an interpreter yet
  * (the command shows an error and returns instead of opening a quick pick). Idempotent — the
  * "already exists" guard is treated as success so a leftover environment from a previous/retried run
  * is reused rather than colliding.
  */
-export async function createEnvironment(name: string): Promise<void> {
+export async function createEnvironment(name: string, options: { useManagedVenv?: boolean } = {}): Promise<void> {
     const driver = VSBrowser.instance.driver;
     let lastError: unknown;
 
@@ -56,7 +134,7 @@ export async function createEnvironment(name: string): Promise<void> {
             continue;
         }
 
-        await interpreterPick.selectQuickPick(0);
+        await selectInterpreter(interpreterPick, options.useManagedVenv === true);
 
         const nameBox = await InputBox.create();
         await nameBox.setText(name);

test/e2e/helpers/fixtures.ts

@@ -2,27 +2,77 @@ import * as fs from 'fs';
 import * as os from 'os';
 import * as path from 'path';
 
+import { linkManagedVenvInto } from './venv';
+
 export interface FixtureCopy {
     /** Removes the throwaway temp directory and its contents. Idempotent; safe to call more than once. */
     cleanup: () => void;
     /** The absolute path to the copied fixture file inside `tempDir`. */
     filePath: string;
-    /** The throwaway temp directory the fixture was copied into (suitable as a workspace folder). */
+    /** The throwaway directory the fixture was copied into, a child of the shared workspace root. */
     tempDir: string;
 }
 
+const FIXTURES_DIR = path.resolve(process.cwd(), 'test', 'e2e', 'fixtures');
+
+let workspaceRoot: string | undefined;
+
+/**
+ * The single directory every fixture copy lives under. openFolderViaDialog opens it once, on the
+ * first suite that asks, rather than once per suite: opening a folder reloads the workbench, and
+ * that reload dominated suite setup. Each suite still removes its own subdirectory in `after`.
+ */
+export function fixturesWorkspaceRoot(): string {
+    if (!workspaceRoot) {
+        workspaceRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'deepnote-e2e-root-'));
+        // Exposed as `.venv` so the Python extension discovers it with no configuration, which is
+        // what lets the suites run without a settings file written ahead of the window opening.
+        linkManagedVenvInto(workspaceRoot);
+    }
+
+    return workspaceRoot;
+}
+
 /**
- * Copies a fixture from `test/e2e/fixtures` into a fresh throwaway temp directory and returns the
- * paths plus a `cleanup` callback that removes the dir. Execution dirties the notebook, so working
- * on a throwaway copy keeps the committed fixture pristine and avoids save prompts.
+ * Copies a fixture from `test/e2e/fixtures` into a fresh directory under the shared workspace root
+ * and returns the paths plus a `cleanup` callback. Execution dirties the notebook, so working on a
+ * throwaway copy keeps the committed fixture pristine and avoids save prompts.
+ *
+ * Copies keep their committed ids: what isolates one suite from the next is the window reload in
+ * enterFixturesWorkspace(), which restarts the extension host, plus this directory being removed in
+ * `after` — so no two copies of a fixture are ever in the workspace at the same time.
  */
 export function copyFixtureToTempDir(fixtureName: string): FixtureCopy {
-    const source = path.resolve(process.cwd(), 'test', 'e2e', 'fixtures', fixtureName);
-    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'deepnote-e2e-'));
+    const tempDir = fs.mkdtempSync(path.join(fixturesWorkspaceRoot(), 'suite-'));
     const filePath = path.join(tempDir, fixtureName);
-    fs.copyFileSync(source, filePath);
+    fs.copyFileSync(path.join(FIXTURES_DIR, fixtureName), filePath);
 
     const cleanup = () => fs.rmSync(tempDir, { recursive: true, force: true });
 
     return { cleanup, filePath, tempDir };
 }
+
+/**
+ * Copies an additional fixture into an existing copy's directory, which is how a suite assembles a
+ * multi-file project. Returns the written path.
+ */
+export function copyFixtureIntoDir(tempDir: string, fixtureName: string): string {
+    const target = path.join(tempDir, fixtureName);
+    fs.copyFileSync(path.join(FIXTURES_DIR, fixtureName), target);
+
+    return target;
+}
+
+/**
+ * Copies a snapshot fixture into `<tempDir>/snapshots/`, the sibling directory buildSnapshotPath
+ * resolves to. Returns the written path.
+ */
+export function copySnapshotIntoDir(tempDir: string, snapshotName: string): string {
+    const snapshotsDir = path.join(tempDir, 'snapshots');
+    fs.mkdirSync(snapshotsDir, { recursive: true });
+
+    const target = path.join(snapshotsDir, snapshotName);
+    fs.copyFileSync(path.join(FIXTURES_DIR, 'snapshots', snapshotName), target);
+
+    return target;
+}