Run a gene set analysis from a file in the chat

.chainlit/config.toml

@@ -33,7 +33,7 @@ edit_message = true
 
 # Authorize users to spontaneously upload files with messages
 [features.spontaneous_file_upload]
-    enabled = false
+    enabled = true
     # Define accepted file types using MIME types
     # Examples:
     # 1. For specific file types:
@@ -43,9 +43,20 @@ edit_message = true
     # 3. For specific file extensions:
     #    accept = { "application/octet-stream" = [".xyz", ".pdb"] }
     # Note: Using "*/*" is not recommended as it may cause browser warnings
-    accept = ["*/*"]
-    max_files = 20
-    max_size_mb = 500
+    # Expression matrices only. "*/*" is what the template ships and it
+    # invites every other kind of file into a chat that can do nothing with
+    # them.
+    accept = { "text/tab-separated-values" = [".tsv", ".txt"], "text/csv" = [".csv"] }
+    max_files = 1
+    # 20, not the template's 500. This host has one 88 GB volume that sits
+    # around 90-95% full, and ~/update-beta-chat.sh refuses to deploy under
+    # 6 GB free -- so a handful of 500 MB uploads would take the chat down
+    # and block the fix. A 16-sample expression matrix is 1.2 MB.
+    #
+    # src/gsa/upload.py enforces the same ceiling again, because this one is
+    # a client-side convenience: Chainlit is what stops the browser sending
+    # it, not what stops the server accepting it.
+    max_size_mb = 20
 
 [features.audio]
     # Sample rate of the audio

bin/chat-chainlit.py

@@ -3,6 +3,7 @@
 # or get_data_layer. Not fixable here; it needs stubs upstream. The file is
 # named with a hyphen, so it cannot be listed in [[tool.mypy.overrides]].
 import os
+from pathlib import Path
 
 import chainlit as cl
 from chainlit.data.base import BaseDataLayer
@@ -16,6 +17,7 @@
 from agent.profiles import get_chat_profiles
 from agent.profiles.base import OutputState
 from agent.registry import get_graph
+from gsa.chainlit_flow import Attachment, matrix_attachment, run_analysis
 from util.chainlit_helpers import (
     PrefixedS3StorageClient,
     is_feature_enabled,
@@ -139,13 +141,70 @@ async def end() -> None:
     await static_messages(config, TriggerEvent.on_chat_end)
 
 
+async def run_gsa_analysis(attachment: Attachment) -> None:
+    """Drive `gsa.chainlit_flow` with this session's chat operations.
+
+    The flow takes these four as arguments so it can be tested without a
+    browser; this is the only place that knows they are Chainlit.
+    """
+    progress = cl.Message(content="Reading your file…")
+    await progress.send()
+
+    async def ask_for_grouping() -> str | None:
+        answer = await cl.AskUserMessage(
+            content="Which group is each sample in?", timeout=600
+        ).send()
+        return (answer or {}).get("output") if answer else None
+
+    async def send(text: str) -> None:
+        await cl.Message(content=text).send()
+
+    async def update_progress(text: str) -> None:
+        progress.content = text
+        await progress.update()
+
+    async def send_file(path: Path) -> None:
+        await cl.Message(
+            content="",
+            elements=[cl.File(name=path.name, path=str(path), display="inline")],
+        ).send()
+
+    # The progress line is removed rather than marked "Done".
+    #
+    # A `finally` that sets "Done." runs on the failure paths too, so a user
+    # whose analysis died would have been told it finished, one line above
+    # the message explaining that it had not. `run_analysis` says what
+    # happened on every path; this only has to stop the spinner.
+    try:
+        await run_analysis(
+            attachment,
+            ask_for_grouping=ask_for_grouping,
+            send=send,
+            update_progress=update_progress,
+            send_file=send_file,
+        )
+    finally:
+        await progress.remove()
+
+
 @cl.on_message
 async def main(message: cl.Message) -> None:
     if await message_rate_limited(config):
         return
 
     await static_messages(config, TriggerEvent.on_message)
 
+    # An attached matrix routes to the analysis flow instead of the graph.
+    #
+    # Not a tool the agent calls: the run takes minutes, which is longer
+    # than a chat turn, and the matrix is over a megabyte, which must never
+    # enter the model's context. A tool call would put the model in the
+    # middle of both problems.
+    attachment = matrix_attachment(getattr(message, "elements", None))
+    if attachment is not None:
+        await run_gsa_analysis(attachment)
+        return
+
     message_count: int = cl.user_session.get("message_count", 0) + 1
     cl.user_session.set("message_count", message_count)
 

specs/012-run-gene-analysis/spec.md

@@ -4,7 +4,7 @@
 
 **Created**: 2026-09-21
 
-**Status**: Draft
+**Status**: Implemented for uploads; public-dataset path deferred
 
 **Input**: User description: "Run a gene set analysis from an uploaded expression matrix in the chat, and return the pathway results as a downloadable file"
 
@@ -153,3 +153,30 @@ and nothing has been sent to the model.
 - Whether an analysis should survive a chat session ending. Results live
   behind an analysis ID at the service, so resuming is possible; whether it is
   wanted is a product question.
+
+---
+
+## What shipped, 2026-09-22
+
+**User Story 2 (upload) and Story 3 (results without a summary) are done.**
+Story 1 (public datasets by identifier) is built underneath --
+`submit_public_dataset` works and is tested -- but has no chat route yet,
+because reaching it needs the model to choose a dataset and a factor, which
+is a conversation design rather than a plumbing job.
+
+Shipping Story 2 before Story 1 inverts the spec's priorities. The reason is
+that the upload path turned out to be the *simpler* one: a file arrives with
+its own sample names, so there is nothing to search for and nothing to
+disambiguate.
+
+**Nothing reaches OpenAI.** Story 3 was written as "a user who declines the
+summary still gets their results". What shipped is that with no summary at
+all: the result is described from the table and attached as a file, both of
+which are the user's own data returning to the user. `Finished.for_model`
+is computed and sent nowhere. When a summary is added it must be given the
+bounded allow-listed view, behind the existing warning -- not the result.
+
+**FR-006 landed in two places on purpose.** `.chainlit/config.toml` caps
+uploads at 20 MB, and `src/gsa/upload.py` caps them again. The first stops
+a browser sending the file; the second stops the server accepting it. Only
+the second is a guarantee.

src/gsa/chainlit_flow.py

@@ -0,0 +1,202 @@
+"""The Chainlit side of running a gene set analysis.
+
+Thin on purpose. Every decision -- what a usable reply is, what the person
+is told, what the model may see -- lives in `gsa.chat` and is tested there.
+What is left here is the part a browser has to verify: reading an attached
+file, asking a follow-up question, streaming progress, sending a download.
+
+**Why this is not a tool the agent calls.** The run takes minutes, which is
+longer than a chat turn, and the matrix is 1.2 MB, which must never enter
+the model's context. A tool call would put the model in the middle of both
+problems. So an attached file routes here *instead of* the graph, and the
+model is handed only the bounded result at the end, if the user asks for a
+summary at all.
+"""
+
+from pathlib import Path
+from typing import Any, Protocol
+
+from gsa import chat
+from gsa.client import GsaClient
+from gsa.job import AnalysisFailedError, await_result, submit_uploaded_matrix
+from gsa.upload import UploadRejectedError, discard, validate
+from util.logging import logging
+
+logger = logging.getLogger(__name__)
+
+#: Where result tables are written. Bounded by `prune_results` on each run.
+#:
+#: Not a bare path under a world-writable `/tmp`: these are a user's own
+#: analysis results, and on a shared host anyone could pre-create the
+#: directory or replace a file in it. `results_dir()` creates it 0700 and
+#: owned by this process, and `GSA_RESULTS_DIR` lets a deployment put it
+#: somewhere with a real quota.
+RESULTS_DIR_ENV = "GSA_RESULTS_DIR"
+
+
+def results_dir() -> Path:
+    """The results directory, created private to this user."""
+    import os
+    import tempfile
+
+    configured = os.environ.get(RESULTS_DIR_ENV)
+    base = (
+        Path(configured) if configured else Path(tempfile.gettempdir()) / "reactome-gsa"
+    )
+    base.mkdir(parents=True, exist_ok=True, mode=0o700)
+    return base
+
+
+#: Extensions we will try to read as a matrix. Anything else attached is
+#: almost certainly meant for a different conversation.
+MATRIX_SUFFIXES = {".tsv", ".csv", ".txt"}
+
+
+class Attachment(Protocol):
+    """The part of a Chainlit file element this needs."""
+
+    name: str
+    path: str
+
+
+def matrix_attachment(elements: list[Any] | None) -> Attachment | None:
+    """The first attachment that could be an expression matrix, if any.
+
+    Returns None for a message with no attachments, which is every ordinary
+    message -- so the caller can use this as the routing decision without
+    knowing anything about Chainlit.
+    """
+    for element in elements or []:
+        name = getattr(element, "name", "") or ""
+        path = getattr(element, "path", None)
+        if path and Path(name).suffix.lower() in MATRIX_SUFFIXES:
+            found: Attachment = element
+            return found
+    return None
+
+
+async def run_analysis(
+    attachment: Attachment,
+    *,
+    ask_for_grouping: Any,
+    send: Any,
+    update_progress: Any,
+    send_file: Any,
+    client: GsaClient | None = None,
+) -> None:
+    """Validate, ask for groups, submit, poll, deliver.
+
+    The callables are passed in rather than imported so this can be driven
+    without Chainlit. They are the four things a chat has to be able to do:
+    ask a question and wait, say something, revise what was said, and hand
+    over a file.
+    """
+    path = Path(attachment.path)
+    try:
+        await _run(
+            attachment, path, ask_for_grouping, send, update_progress, send_file, client
+        )
+    finally:
+        # The upload does not survive this function, whatever happened.
+        #
+        # Each path used to delete it for itself, which covered the ones I
+        # thought of. It did not cover a file Chainlit had not finished
+        # writing (`FileNotFoundError` straight out of `validate`), or an
+        # `AskUserMessage` that times out or is interrupted -- and an
+        # unhandled error is exactly when nobody is around to tidy up.
+        # `discard` is safe to call twice, so the paths that already delete
+        # it can keep doing so.
+        discard(path)
+
+
+async def _run(
+    attachment: Attachment,
+    path: Path,
+    ask_for_grouping: Any,
+    send: Any,
+    update_progress: Any,
+    send_file: Any,
+    client: GsaClient | None,
+) -> None:
+    try:
+        matrix = validate(path)
+    except UploadRejectedError as refusal:
+        await send(str(refusal))
+        return
+    except OSError:
+        # Unreadable, vanished, or not a file. The user did nothing wrong
+        # that they can act on, so do not describe it as their mistake.
+        logger.exception("could not read an uploaded file")
+        await send("I could not read that file — please try attaching it again.")
+        return
+
+    await send(chat.describe_matrix(matrix))
+
+    reply = await ask_for_grouping()
+    if not reply:
+        discard(path)
+        await send(
+            "No labels arrived, so I have not run anything. The file is deleted."
+        )
+        return
+
+    try:
+        grouping = chat.parse_grouping(reply, len(matrix.samples))
+    except chat.ReplyUnusableError as unusable:
+        await send(f"{unusable} Send the file again when you are ready.")
+        return
+
+    gsa = client or GsaClient()
+    try:
+        # `submit_uploaded_matrix` deletes the file itself, on every path.
+        analysis_id = await submit_uploaded_matrix(
+            gsa,
+            matrix=matrix,
+            dataset_type="rnaseq_counts",
+            analysis_group=grouping.labels,
+            group1=grouping.group1,
+            group2=grouping.group2,
+        )
+    except AnalysisFailedError as failure:
+        await send(f"I could not start the analysis: {failure}")
+        return
+    except Exception:
+        logger.exception("gsa submission failed")
+        await send("I could not reach the analysis service. Nothing was run.")
+        return
+
+    await send(
+        f"Started. This usually takes a few minutes — "
+        f"comparing **{grouping.group1}** with **{grouping.group2}**."
+    )
+
+    try:
+        finished = await await_result(
+            gsa,
+            analysis_id,
+            out_dir=results_dir(),
+            on_progress=lambda status: update_progress(chat.describe_progress(status)),
+        )
+    except AnalysisFailedError as failure:
+        # The service accepts and then fails, and says so only here.
+        await send(f"The analysis did not finish: {failure}")
+        return
+    except Exception:
+        logger.exception("gsa analysis failed", extra={"analysis": analysis_id})
+        await send("Something went wrong while waiting for the analysis.")
+        return
+
+    # `finished.for_model` is deliberately not used here.
+    #
+    # Nothing about this analysis reaches OpenAI. The result is described
+    # from the table and handed over as a file, both of which are the
+    # user's own data going back to the user. A model-written summary is
+    # spec 012's Story 3 the other way round -- opt-in, behind the existing
+    # warning -- and until that exists, the honest state is that the
+    # allow-listed view is computed and sent nowhere.
+    #
+    # It is computed rather than skipped so the disclosure rules stay
+    # exercised by the tests; if a summary is added later, the bounded view
+    # is what it must be given, not the result.
+    await send(chat.describe_result(finished))
+    await send_file(finished.table_path)