feat(assessment): Persist batch results and deliver them in callback metadata

backend/app/alembic/versions/084_assessment_submissions_and_result_files.py

@@ -0,0 +1,162 @@
+"""Assessment submissions table, submission/result-file pointers, provider error file id
+
+Revision ID: 084
+Revises: 083
+Create Date: 2026-09-09 00:00:00.000000
+
+Assessment submissions leave `evaluation_dataset`, whose type-agnostic name uniqueness
+let an eval dataset block an assessment one. Multi-MB payloads leave Postgres too:
+`submission_input` and `result_files` hold s3:// urls, and `provider_error_file_id`
+keeps OpenAI's error dump fetchable after the poll that surfaced it. `dataset_id` is
+dropped without a backfill: no assessment rows exist in any environment yet.
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+revision = "084"
+down_revision = "083"
+branch_labels = None
+depends_on = None
+
+RESULT_FILES_CHECK = "ck_assessment_result_files_is_object"
+
+
+def upgrade() -> None:
+    op.create_table(
+        "assessment_submission",
+        sa.Column(
+            "id",
+            postgresql.UUID(as_uuid=True),
+            primary_key=True,
+            comment="Unique identifier for the submission",
+        ),
+        sa.Column(
+            "name",
+            sa.String(),
+            nullable=False,
+            comment="Sanitized name; the object key is derived from it",
+        ),
+        sa.Column(
+            "description", sa.String(), nullable=True, comment="Optional description"
+        ),
+        sa.Column(
+            "object_store_url",
+            sa.String(),
+            nullable=False,
+            comment="Object-store url of the uploaded file; its suffix gives the format",
+        ),
+        sa.Column(
+            "total_items",
+            sa.Integer(),
+            nullable=False,
+            server_default="0",
+            comment="Row count, excluding the header",
+        ),
+        sa.Column(
+            "organization_id",
+            sa.Integer(),
+            sa.ForeignKey("organization.id", ondelete="CASCADE"),
+            nullable=False,
+        ),
+        sa.Column(
+            "project_id",
+            sa.Integer(),
+            sa.ForeignKey("project.id", ondelete="CASCADE"),
+            nullable=False,
+        ),
+        sa.Column("inserted_at", sa.DateTime(), nullable=False),
+        sa.Column("updated_at", sa.DateTime(), nullable=False),
+        sa.UniqueConstraint(
+            "name",
+            "organization_id",
+            "project_id",
+            name="uq_assessment_submission_name_org_project",
+        ),
+    )
+    op.create_index("ix_assessment_submission_name", "assessment_submission", ["name"])
+
+    op.add_column(
+        "assessment",
+        sa.Column(
+            "result_files",
+            postgresql.JSONB(astext_type=sa.Text()),
+            nullable=False,
+            server_default=sa.text("'{}'::jsonb"),
+            comment=(
+                "Result-file kind (results / errors / <stage>_results) to "
+                "{object_store_url} for every provider batch dump held; raw s3:// in the "
+                "column, presigned per delivery in the BATCH callback"
+            ),
+        ),
+    )
+    op.add_column(
+        "assessment",
+        sa.Column(
+            "submission_input",
+            sa.String(),
+            nullable=True,
+            comment=(
+                "Object-store url of the API-client BATCH submission rows "
+                "(submission.jsonl); the rows are never stored in this table"
+            ),
+        ),
+    )
+    op.drop_column("assessment", "dataset_id")
+    op.add_column(
+        "assessment",
+        sa.Column(
+            "submission_id",
+            postgresql.UUID(as_uuid=True),
+            sa.ForeignKey("assessment_submission.id", ondelete="SET NULL"),
+            nullable=True,
+            comment=(
+                "Uploaded submission the rows came from; set by RUN and by a BATCH "
+                "submitted with `submission_doc_id`. NULL when BATCH sent rows inline"
+            ),
+        ),
+    )
+    op.create_index("ix_assessment_submission_id", "assessment", ["submission_id"])
+
+    op.add_column(
+        "batch_job",
+        sa.Column(
+            "provider_error_file_id",
+            sa.String(),
+            nullable=True,
+            comment=(
+                "Provider's error file ID (OpenAI only; Anthropic and Gemini report "
+                "per-item errors inline)"
+            ),
+        ),
+    )
+    op.create_check_constraint(
+        RESULT_FILES_CHECK,
+        "assessment",
+        "jsonb_typeof(result_files) = 'object'",
+    )
+
+
+def downgrade() -> None:
+    op.drop_constraint(RESULT_FILES_CHECK, "assessment", type_="check")
+    op.drop_column("batch_job", "provider_error_file_id")
+
+    op.drop_index("ix_assessment_submission_id", table_name="assessment")
+    op.drop_column("assessment", "submission_id")
+    op.add_column(
+        "assessment",
+        sa.Column(
+            "dataset_id",
+            sa.Integer(),
+            sa.ForeignKey("evaluation_dataset.id", ondelete="SET NULL"),
+            nullable=True,
+            comment="External dataset (RUN); binding lives in `input`",
+        ),
+    )
+
+    op.drop_column("assessment", "submission_input")
+    op.drop_column("assessment", "result_files")
+
+    op.drop_index("ix_assessment_submission_name", table_name="assessment_submission")
+    op.drop_table("assessment_submission")

backend/app/api/docs/assessment/create.md

@@ -1,4 +1,4 @@
-Submit an assessment against a saved LLM configuration; results are delivered by webhook.
+Submit an assessment against a saved LLM configuration; poll for the result, or have it pushed to a webhook.
 
 An assessment grades one or more items with a config-defined LLM call, optionally gated by
 pre-filters (topic relevance / duplicate detection). The run mode is **inferred from the input
@@ -10,8 +10,8 @@ shape** — you do not pass a mode flag.
 * Pins to a saved config **version** (`config.id` + `config.version`); the config must be tagged
   `ASSESSMENT` (see the config-create docs for the `config_blob` assessment shape).
 * Optional pre-filters run before the grading call and can gate it per item.
-* **Webhook-only delivery** — the result is POSTed to the request's `callback_url` on completion.
-  There is no status or result poll endpoint.
+* **Two ways to get the result** — poll `GET /assessments/{assessment_id}`, and/or supply a
+  `callback_url` to have the finished result POSTed to you on completion.
 * `request_metadata` is echoed back unchanged in the callback for correlation.
 
 > **RESPONSE mode is not wired yet** — a single-object input currently returns `501 Not Implemented`.
@@ -53,12 +53,15 @@ shape** — you do not pass a mode flag.
   * **BATCH** — `{ "query": "<template>", "data": [ {<column>: <value>}, ... ] }`
     * `query` (required, non-empty) — a template with `{column}` placeholders substituted per row.
     * `data` (required, ≥ 1 row) — submission rows. Each row is a flat `column -> string` map. The
-      config's `assessment.params.input_schema` is **mandatory** and defines every column and its
-      `type` (`text` / `image` / `pdf`, with an attachment `format`). Every row is validated against
-      it: each declared column must be present, no undeclared columns are allowed, and `image`/`pdf`
-      columns must carry a URL. A row that does not match fails with `422` (see Errors).
+      config's `input_schema` is **mandatory** and defines every column: its `type` (`text` / `image` /
+      `pdf`, with an attachment `format`) and `strict` (default `false`). Every row is validated
+      against it: a `strict: true` column must be present and non-blank, any other column may be
+      omitted or left blank, an inline `data` row may not carry undeclared columns (a
+      `submission_doc_id` sheet's undeclared columns are ignored, so the schema selects which
+      columns take part), and a non-blank `image`/`pdf` value must be a URL. A row that does not
+      match fails with `422` (see Errors).
   * **RESPONSE** — `{ "query": "<text>", "attachments": [ ... ] }` *(deferred — returns 501)*.
-* `callback_url` (required) — the webhook the result is POSTed to on completion.
+* `callback_url` (optional) — the webhook the result is POSTed to on completion. Omit it to poll instead.
 * `request_metadata` (optional) — arbitrary object passed through unchanged in the callback.
 
 The two input shapes are strictly discriminated: a body carrying `data` is BATCH, one carrying
@@ -142,6 +145,6 @@ On completion the platform POSTs this payload to `callback_url`:
 
 * `501 Not Implemented` — RESPONSE-mode input (single object) is not wired yet; send a BATCH `data` list.
 * `422 Unprocessable Entity` — the body failed validation (e.g. `config.id`/`config.version` missing,
-  `callback_url` missing, or an input shape that carries both `data` and `attachments`), or a
-  submission row does not match the config's `input_schema` (a missing declared column, an
+  an input shape carrying both `data` and `submission_doc_id`, or neither), or a
+  submission row does not match the config's `input_schema` (a strict column absent or blank, an
   undeclared extra column, or a non-URL `image`/`pdf` value). The row index is named in the error.

backend/app/api/docs/assessment/export_assessment_results.md

@@ -1,4 +1,7 @@
-Export results for all child runs under an assessment.
+Export results for all child runs under a RUN assessment.
 
 For `json`, returns a flat list in the API response. For `csv`/`xlsx`,
 returns one file for a single run or a ZIP archive when multiple runs exist.
+
+Returns `422` for a BATCH assessment: its rows are served by
+`GET /assessments/{assessment_id}`.

backend/app/api/docs/assessment/get_assessment_detail.md

@@ -0,0 +1,21 @@
+Fetch a BATCH assessment's status and every row it has produced so far.
+
+Safe to poll while the run is in flight. `items` always holds exactly `total_items`
+entries, in submission order, including placeholders for rows the provider has not
+returned yet (`output.assessment` is `null`) and for rows a pre-filter gated out.
+
+Each row carries:
+
+- `row_index` — position in the original submission; the stable correlator.
+- `input` — the submitted row, echoed back only when `include_input=true`. That flag
+  re-reads the stored submission from object storage on every call, so leave it off in
+  a tight poll loop and set it once the run is terminal. `null` if the stored submission
+  could not be read.
+- `output` — identical in shape to the webhook payload's item, so one parser serves both.
+- `error` — the provider's error for that row, when it failed.
+
+Stop polling once `status` is `COMPLETED`, `COMPLETED_WITH_ERRORS` or `FAILED`. On a
+failed run, `error` carries the reason.
+
+Returns 404 when the assessment does not exist in this project, and 422 when it is not
+a BATCH assessment.

backend/app/api/docs/assessment/list_assessments.md

@@ -1,3 +1,14 @@
-List assessments runs for the current organization/project.
+List this project's BATCH assessments, newest first.
 
-Each record includes aggregate status counters across its child runs.
+Each row carries where the run is and what it was pinned to — status, stage, the config
+id and version, and the row count. It deliberately carries **no per-row counts or
+results**: those need the provider dump streamed back from object storage, which a list
+must not pay for. Fetch `GET /assessments/{assessment_id}` for a single run's rows.
+
+**Filtering**
+
+- `config_id` — return only the runs pinned to that config. Omit it for every run.
+- `version` — narrow further to one version of that config. Applied only with
+  `config_id`; omit it for every version.
+
+`limit` defaults to 50 (max 100) and `offset` pages through.

backend/app/api/docs/assessment/upload_dataset.md

@@ -2,3 +2,7 @@ Upload a CSV or Excel dataset for assessment workflows.
 
 The file is stored in object storage and indexed as an assessment dataset
 for the current organization and project.
+
+Excel files are cleaned on read: blank rows are dropped, then any column with no
+header or no values. `total_items`, the preview and the run all see the cleaned
+sheet, so a 1000-row sheet with 100 filled rows reports `total_items: 100`.

backend/app/api/docs/config/create.md

@@ -83,13 +83,17 @@ When `tag` is `"ASSESSMENT"`, `config_blob` uses the assessment shape instead of
 not rendered in the OpenAPI spec (so the default `config_blob` schema stays stable) — the
 shape is documented here:
 
+* `input_schema` (required, at the `config_blob` root) — **non-empty** mapping of each
+  submission column name to `{ type, format, strict }` (`type` is **required**: `text` |
+  `image` | `pdf`; `format`: `url` for attachment columns; `strict` defaults to `false`).
+  A `strict: true` column must be present and non-blank in every submission row; any other
+  column may be omitted or blank (see the submit docs for per-row validation). Unknown keys
+  in a column spec are rejected. Shared by the pre-filter and the assessment call.
 * `assessment` (required) — the grading call. `provider` is `openai` | `google` |
   `anthropic`, `type` is `"text"`. `params` carries the `model`, the `instructions`
-  (system prompt), an optional `json_output_schema` (structured-output JSON schema), and a
-  **mandatory, non-empty** `input_schema` mapping each column name to `{ type, format }`
-  (`type` is **required**: `text` | `image` | `pdf`; `format`: `url` for attachment
-  columns). Every declared column must be present in every submission row (see the submit
-  docs for per-row validation).
+  (system prompt), the **mandatory** per-row `submission` template (`{column}`
+  placeholders must name `input_schema` keys), and an optional `json_output_schema`
+  (structured-output JSON schema).
 * `pre_filters` (optional) — `topic_relevance` and/or `duplicate_detection`. Each runs its
   own llm call, so it carries `provider` (default `openai`) + its own `params`
   (a `TextLLMParams` object: `model`, `temperature`, …). Its criteria live in
@@ -101,6 +105,10 @@ shape is documented here:
 
 ```json
 "config_blob": {
+  "input_schema": {
+    "gcs_url": { "type": "image", "format": "url", "strict": true },
+    "rubric":  { "type": "text" }
+  },
   "pre_filters": {
     "topic_relevance": {
       "provider": "openai",
@@ -118,10 +126,7 @@ shape is documented here:
     "params": {
       "model": "gpt-4o",
       "instructions": "You are an AI Assessment Evaluator ...",
-      "input_schema": {
-        "gcs_url": { "type": "image", "format": "url" },
-        "rubric":  { "type": "text" }
-      },
+      "submission": "Grade the attached answer sheet against this rubric: {rubric}",
       "json_output_schema": {
         "type": "object",
         "properties": { "grade": { "type": "string" }, "feedback": { "type": "string" } },

backend/app/api/docs/config/create_version.md

@@ -6,13 +6,15 @@ create a new version under the same configuration with an incremented version nu
 Version numbers are automatically incremented sequentially (1, 2, 3, etc.)
 and cannot be manually set or skipped.
 
-## Examples
+The `config_blob` shape follows the parent config: the `completion` shape for a
+`default` config, the `assessment` shape for an `ASSESSMENT` config. How the body is
+applied differs between the two, so read the matching section below.
 
-Send only the fields you want to change. The `config_blob` shape follows the parent
-config: the `completion` shape for a `default` config, the `assessment` shape for an
-`ASSESSMENT` config.
+## default configs — partial update
+
+Send only the fields you want to change. They are merged onto the latest version, so
+anything you omit is carried forward.
 
-**When the parent config is `default` (completion shape):**
 ```json
 {
   "config_blob": {
@@ -24,24 +26,44 @@ config: the `completion` shape for a `default` config, the `assessment` shape fo
 }
 ```
 
-**When the parent config is `ASSESSMENT` (assessment shape):**
+## ASSESSMENT configs — full blob
+
+Send the **whole** `config_blob` every time. It replaces the previous version rather than
+merging onto it, so a column dropped from `input_schema`, a removed `json_output_schema`
+field, or an omitted `pre_filters` block is genuinely gone in the new version. A partial
+body is rejected with `422`, because `input_schema` and `assessment` are mandatory.
+
 ```json
 {
   "config_blob": {
+    "input_schema": {
+      "rubric": { "type": "text" },
+      "answer": { "type": "text", "strict": true }
+    },
     "pre_filters": {
       "topic_relevance": {
-        "params": { "model": "gpt-4o", "instructions": "Is this a Class 7 answer sheet?" }
+        "provider": "openai",
+        "params": { "model": "gpt-4o", "instructions": "Is this a Class 7 answer sheet?" },
+        "stop_on_fail": true
       }
     },
     "assessment": {
-      "params": { "model": "gpt-4o" }
+      "provider": "openai",
+      "type": "text",
+      "params": {
+        "model": "gpt-4o",
+        "instructions": "You are an AI Assessment Evaluator ...",
+        "submission": "Grade this answer against the rubric: {rubric}\n\nAnswer: {answer}"
+      }
     }
   },
-  "commit_message": "Switch grading model"
+  "commit_message": "Drop the unused columns"
 }
 ```
 
 ## Important
-- This endpoint accepts partial updates using dict[str, Any] for config_blob.
-- Only the fields that need to be updated should be provided.
-- The `type` field is inherited from the existing configuration and cannot be changed. Provider and model can change between versions.
+- Every field inside `config_blob` can change between versions, including provider and model.
+- `tag` belongs to the parent configuration and is never part of a version body.
+- `type` is inherited from the existing configuration and cannot be changed.
+- A run pins the `config_id` and `version` it was submitted with, so a new version never
+  alters a run that is already in flight or finished.