From f89daf2e62a749402de1bb0a994fc1cbaea2f140 Mon Sep 17 00:00:00 2001 From: Steven Chand Date: Thu, 20 Aug 2026 17:20:05 -0700 Subject: [PATCH 1/6] [SC-17911] Send log_text Markdown without script tags --- tests/test_api_client.py | 58 +++++++++++++++++++++++++++++++++++++--- validmind/api_client.py | 56 +++++++++++++++++++++++++++++++------- 2 files changed, 100 insertions(+), 14 deletions(-) diff --git a/tests/test_api_client.py b/tests/test_api_client.py index 971dbce2e..122202b54 100644 --- a/tests/test_api_client.py +++ b/tests/test_api_client.py @@ -315,9 +315,8 @@ def test_log_text_generates_text_and_logs_metadata( data=json.dumps( { "content_id": "dataset_summary_text", - "text": md_to_html( - "## Generated Summary\nGenerated content.", mathml=True - ), + "text": "## Generated Summary\nGenerated content.", + "text_format": "markdown", } ), ) @@ -366,7 +365,58 @@ def test_log_text_logs_metadata_with_section_id( data=json.dumps( { "content_id": "dataset_summary_text", - "text": md_to_html("Generated content.", mathml=True), + "text": "Generated content.", + "text_format": "markdown", + } + ), + ) + + @patch("aiohttp.ClientSession.post") + def test_log_text_sends_tex_as_waf_safe_markdown(self, mock_post: MagicMock): + equation = r"$WOE = \ln\dfrac{\%\ of\ Events}{\%\ of\ Non-Events}$" + mock_post.return_value = MockAsyncResponse( + 200, + json={ + "content_id": "text_woe_equation", + "text": '

', + }, + ) + + self.run_async( + api_client.alog_text, + "text_woe_equation", + text=equation, + ) + + mock_post.assert_called_once_with( + f"{os.environ['VM_API_HOST']}/log_metadata", + data=json.dumps( + { + "content_id": "text_woe_equation", + "text": equation, + "text_format": "markdown", + } + ), + ) + request_body = mock_post.call_args.kwargs["data"] + self.assertNotIn(" Dict[str, Any]: """Logs free-form metadata to ValidMind API. @@ -597,6 +598,9 @@ async def alog_metadata( _json (dict, optional): Free-form key-value pairs to assign to the metadata. Defaults to None. section_id (str, optional): Section ID to append the text block to when the content ID does not already exist. + text_format (str, optional): Format of ``text``. New ``log_text`` calls use + ``markdown`` so conversion happens after the request passes through the + WAF. Omitted for backward-compatible HTML payloads. Raises: Exception: If the API call fails. @@ -609,6 +613,8 @@ async def alog_metadata( metadata_dict["text"] = text if _json is not None: metadata_dict["json"] = _json + if text_format is not None: + metadata_dict["text_format"] = text_format request_params = {} if section_id: @@ -808,11 +814,18 @@ def generate_qualitative_text(text_generation_data: Dict[str, Any]) -> Dict[str, return r.json() -def _normalize_logged_text(text: str, field_name: str) -> str: - """Validate text content and convert Markdown to HTML when needed.""" +def _validate_logged_text(text: str, field_name: str) -> str: + """Validate text accepted by log_text.""" if not isinstance(text, str) or not text: raise ValueError(f"`{field_name}` must be a non-empty string") + return text + + +def _normalize_logged_text(text: str, field_name: str) -> str: + """Validate text content and convert Markdown to HTML for local rendering.""" + text = _validate_logged_text(text, field_name) + if not is_html(text): return md_to_html(text, mathml=True) @@ -828,7 +841,7 @@ def _validate_manual_log_text_args( if context is not None: raise ValueError("`context` is only supported when `text` is omitted") - return _normalize_logged_text(text, "text") + return _validate_logged_text(text, "text") def _build_log_text_generation_request( @@ -860,13 +873,13 @@ def _build_log_text_generation_request( return request_data -def _generate_log_text( +def _generate_log_text_source( content_id: str, prompt: Optional[str], context: Optional[Dict[str, Any]], section_id: Optional[str] = None, ) -> str: - """Generate text for log_text and normalize it to HTML.""" + """Generate and validate source text without converting it to HTML.""" request_data = _build_log_text_generation_request( content_id, prompt, @@ -874,6 +887,22 @@ def _generate_log_text( section_id=section_id, ) generated_text = generate_qualitative_text(request_data)["content"] + return _validate_logged_text(generated_text, "generated text") + + +def _generate_log_text( + content_id: str, + prompt: Optional[str], + context: Optional[Dict[str, Any]], + section_id: Optional[str] = None, +) -> str: + """Generate text and normalize it to HTML for local result rendering.""" + generated_text = _generate_log_text_source( + content_id, + prompt, + context, + section_id=section_id, + ) return _normalize_logged_text(generated_text, "generated text") @@ -902,14 +931,21 @@ async def alog_text( if text is not None: text = _validate_manual_log_text_args(text, prompt, context) else: - text = _generate_log_text( + text = _generate_log_text_source( content_id, prompt, context, section_id=section_id, ) - return await alog_metadata(content_id, text, _json, section_id=section_id) + text_format = None if is_html(text) else "markdown" + return await alog_metadata( + content_id, + text, + _json, + section_id=section_id, + text_format=text_format, + ) def log_text( @@ -924,9 +960,9 @@ def log_text( Args: content_id (str): Unique content identifier for the text. - text (str, optional): The text to log. Will be converted to HTML with - MathML support when Markdown is provided. If omitted, text is - generated using the qualitative text generation backend. + text (str, optional): The text to log. Markdown is sent to the backend for + HTML and math conversion. If omitted, text is generated using the + qualitative text generation backend. prompt (str, optional): Custom prompt used for AI-assisted text generation. Only supported when `text` is omitted. context (dict, optional): Context object for AI-assisted text From 24b20780fcd0b921d41c2e46486aa695942af42b Mon Sep 17 00:00:00 2001 From: Steven Chand Date: Thu, 20 Aug 2026 17:48:27 -0700 Subject: [PATCH 2/6] [SC-17911] Keep result description payloads WAF-safe --- tests/test_api_client.py | 27 +++++++ tests/test_client.py | 11 +-- tests/test_results.py | 104 ++++++++++++++++++++++++--- tests/test_test_descriptions.py | 14 +++- validmind/ai/utils.py | 7 +- validmind/client.py | 10 ++- validmind/experimental/agents.py | 4 +- validmind/tests/output.py | 3 + validmind/tests/run.py | 7 +- validmind/vm_models/result/result.py | 54 ++++++++++---- validmind/vm_models/result/utils.py | 16 ++++- 11 files changed, 220 insertions(+), 37 deletions(-) diff --git a/tests/test_api_client.py b/tests/test_api_client.py index 122202b54..2d0a42bf8 100644 --- a/tests/test_api_client.py +++ b/tests/test_api_client.py @@ -23,6 +23,7 @@ ) from validmind.utils import md_to_html from validmind.vm_models.figure import Figure +from validmind.vm_models.result.utils import update_metadata loop = asyncio.new_event_loop() @@ -247,6 +248,32 @@ def test_log_metadata_with_section_id(self, mock_post: MagicMock): ), ) + @patch("aiohttp.ClientSession.post") + def test_result_update_metadata_sends_waf_safe_markdown( + self, mock_post: MagicMock + ): + equation = r"$WOE = \ln\dfrac{\%\ of\ Events}{\%\ of\ Non-Events}$" + mock_post.return_value = MockAsyncResponse(200, json={"cuid": "abc1234"}) + + self.run_async( + update_metadata, + "test_description:equation::default", + equation, + text_format="markdown", + ) + + mock_post.assert_called_once_with( + f"{os.environ['VM_API_HOST']}/log_metadata", + data=json.dumps( + { + "content_id": "test_description:equation::default", + "text": equation, + "text_format": "markdown", + } + ), + ) + self.assertNotIn("Generated text

") + self.assertIn('

',\n", + " },\n", + "]\n", + "\n", + "assert {is_html(case[\"text\"]) for case in CASES} == {False, True}\n", + "pd.DataFrame(\n", + " {\n", + " \"case\": [case[\"name\"] for case in CASES],\n", + " \"detected_as\": [\"html\" if is_html(case[\"text\"]) else \"markdown\" for case in CASES],\n", + " }\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "capture-helper", + "metadata": {}, + "outputs": [], + "source": [ + "async def capture_log_text_request(case, supports_markdown):\n", + " \"\"\"Run alog_text while capturing the serialized log_metadata body.\"\"\"\n", + " captured = {}\n", + "\n", + " async def capture_post(path, params=None, data=None, **kwargs):\n", + " captured[\"path\"] = path\n", + " captured[\"params\"] = params\n", + " captured[\"body\"] = json.loads(data)\n", + " return {\"content_id\": captured[\"body\"][\"content_id\"], \"text\": captured[\"body\"].get(\"text\")}\n", + "\n", + " original_flags = client_config.feature_flags\n", + " try:\n", + " client_config.feature_flags = {\"log_metadata_markdown\": supports_markdown}\n", + " with patch.object(api_client, \"_post\", new=capture_post):\n", + " await api_client.alog_text(\n", + " content_id=f'test_description:sc17911_{case[\"name\"]}',\n", + " text=case[\"text\"],\n", + " )\n", + " finally:\n", + " client_config.feature_flags = original_flags\n", + "\n", + " assert captured[\"path\"] == \"log_metadata\"\n", + " return captured[\"body\"]" + ] + }, + { + "cell_type": "markdown", + "id": "new-backend-heading", + "metadata": {}, + "source": [ + "## New backend: WAF-safe Markdown transport\n", + "\n", + "Markdown and TeX must stay raw in the request. Fully formed HTML must retain the legacy pass-through behavior." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "new-backend-matrix", + "metadata": {}, + "outputs": [], + "source": [ + "new_backend_rows = []\n", + "for case in CASES:\n", + " body = await capture_log_text_request(case, supports_markdown=True)\n", + " html_input = is_html(case[\"text\"])\n", + "\n", + " assert body[\"text\"] == case[\"text\"]\n", + " if html_input:\n", + " assert \"text_format\" not in body\n", + " else:\n", + " assert body[\"text_format\"] == \"markdown\"\n", + " assert \" Date: Fri, 21 Aug 2026 11:37:35 -0700 Subject: [PATCH 5/6] test: run logging notebook against either capability mode --- .../markdown_html_logging_regression.ipynb | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/notebooks/code_sharing/markdown_html_logging_regression.ipynb b/notebooks/code_sharing/markdown_html_logging_regression.ipynb index 6a876424c..abed98d76 100644 --- a/notebooks/code_sharing/markdown_html_logging_regression.ipynb +++ b/notebooks/code_sharing/markdown_html_logging_regression.ipynb @@ -211,7 +211,7 @@ "source": [ "## Optional end-to-end backend check\n", "\n", - "Run this section with `VM_RUN_E2E=1` to initialize the SDK from `VM_API_*`, log every case to the selected model's documentation, and verify the backend persists semantically equivalent rendered HTML while preserving explicit HTML byte-for-byte. The backend must advertise `feature_flags.log_metadata_markdown=true` for this check." + "Run this section with `VM_RUN_E2E=1` to initialize the SDK from `VM_API_*`, log every case to the selected model's documentation, and verify the backend persists semantically equivalent rendered HTML while preserving explicit HTML byte-for-byte. It runs in either capability state and reports whether it exercised backend Markdown conversion or the legacy SDK fallback." ] }, { @@ -243,8 +243,7 @@ " assert script is not None and \"WOE\" in script.get_text()\n", " elif name == \"display_tex\":\n", " script = soup.find(\"script\", attrs={\"type\": \"math/tex\"})\n", - " assert script is not None and \"block\" in script.get(\"class\", [])\n", - " assert \"G(x)\" in script.get_text()\n", + " assert script is not None and \"G(x)\" in script.get_text()\n", " elif name == \"markdown_extensions\":\n", " assert soup.find(\"del\").get_text(strip=True) == \"Deprecated\"\n", " assert soup.select_one(\"figure.table table\") is not None\n", @@ -263,9 +262,11 @@ " assert not missing, f\"Missing required environment variables: {missing}\"\n", "\n", " vm.init(document=\"documentation\")\n", - " assert client_config.supports_log_metadata_markdown(), (\n", - " \"The connected backend must advertise log_metadata_markdown=true \"\n", - " \"for the end-to-end portion of this regression.\"\n", + " markdown_capability = client_config.supports_log_metadata_markdown()\n", + " transport_mode = (\n", + " \"backend Markdown conversion\"\n", + " if markdown_capability\n", + " else \"legacy SDK conversion\"\n", " )\n", "\n", " for case in CASES:\n", @@ -280,6 +281,7 @@ " {\n", " \"case\": case[\"name\"],\n", " \"content_id\": response[\"content_id\"],\n", + " \"transport_mode\": transport_mode,\n", " \"persisted_as_expected\": True,\n", " }\n", " )\n", @@ -290,6 +292,7 @@ " {\n", " \"case\": \"skipped\",\n", " \"content_id\": \"Set VM_RUN_E2E=1 to run against a backend\",\n", + " \"transport_mode\": None,\n", " \"persisted_as_expected\": None,\n", " }\n", " )\n", From f24ed2947b28cdfb689f747f03054a151d19dd4b Mon Sep 17 00:00:00 2001 From: Steven Chand Date: Fri, 21 Aug 2026 11:52:15 -0700 Subject: [PATCH 6/6] docs: show client initialization in regression notebook --- .../markdown_html_logging_regression.ipynb | 41 ++++++++++++++++--- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/notebooks/code_sharing/markdown_html_logging_regression.ipynb b/notebooks/code_sharing/markdown_html_logging_regression.ipynb index abed98d76..ed0b230b5 100644 --- a/notebooks/code_sharing/markdown_html_logging_regression.ipynb +++ b/notebooks/code_sharing/markdown_html_logging_regression.ipynb @@ -214,14 +214,48 @@ "Run this section with `VM_RUN_E2E=1` to initialize the SDK from `VM_API_*`, log every case to the selected model's documentation, and verify the backend persists semantically equivalent rendered HTML while preserving explicit HTML byte-for-byte. It runs in either capability state and reports whether it exercised backend Markdown conversion or the legacy SDK fallback." ] }, + { + "cell_type": "markdown", + "id": "initialize-client-heading", + "metadata": {}, + "source": [ + "### Initialize the ValidMind client\n", + "\n", + "The client is initialized only for the optional end-to-end run. Credentials are read from environment variables and are never stored in the notebook." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "e2e-check", + "id": "initialize-client", "metadata": {}, "outputs": [], "source": [ "run_e2e = os.getenv(\"VM_RUN_E2E\", \"0\").lower() in {\"1\", \"true\", \"yes\"}\n", + "\n", + "if run_e2e:\n", + " required = [\"VM_API_HOST\", \"VM_API_KEY\", \"VM_API_SECRET\", \"VM_API_MODEL\"]\n", + " missing = [name for name in required if not os.getenv(name)]\n", + " assert not missing, f\"Missing required environment variables: {missing}\"\n", + "\n", + " vm.init(\n", + " api_host=os.environ[\"VM_API_HOST\"],\n", + " api_key=os.environ[\"VM_API_KEY\"],\n", + " api_secret=os.environ[\"VM_API_SECRET\"],\n", + " model=os.environ[\"VM_API_MODEL\"],\n", + " document=\"documentation\",\n", + " )\n", + "else:\n", + " print(\"Client initialization skipped. Set VM_RUN_E2E=1 to enable it.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e2e-check", + "metadata": {}, + "outputs": [], + "source": [ "e2e_rows = []\n", "\n", "def assert_backend_render(case, stored_text):\n", @@ -257,11 +291,6 @@ " raise AssertionError(f\"Missing Markdown assertion for {name}\")\n", "\n", "if run_e2e:\n", - " required = [\"VM_API_HOST\", \"VM_API_KEY\", \"VM_API_SECRET\", \"VM_API_MODEL\"]\n", - " missing = [name for name in required if not os.getenv(name)]\n", - " assert not missing, f\"Missing required environment variables: {missing}\"\n", - "\n", - " vm.init(document=\"documentation\")\n", " markdown_capability = client_config.supports_log_metadata_markdown()\n", " transport_mode = (\n", " \"backend Markdown conversion\"\n",