From 171126e35e98ff9313f52b16e0a5c80e1e17100e Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 6 Aug 2026 20:08:35 -0400 Subject: [PATCH] Add a Sphinx extension to highlight custom keywords --- pep_sphinx_extensions/__init__.py | 2 + .../pep_processor/parsing/pep_soft_keyword.py | 88 +++++++++++++++++++ peps/pep-0842.rst | 22 ++--- 3 files changed, 101 insertions(+), 11 deletions(-) create mode 100644 pep_sphinx_extensions/pep_processor/parsing/pep_soft_keyword.py diff --git a/pep_sphinx_extensions/__init__.py b/pep_sphinx_extensions/__init__.py index 109c09d7890..29570d0e1db 100644 --- a/pep_sphinx_extensions/__init__.py +++ b/pep_sphinx_extensions/__init__.py @@ -105,6 +105,8 @@ def setup(app: Sphinx) -> dict[str, bool]: app.add_directive("superseded", pep_banner_directive.SupersededBanner) app.add_directive("withdrawn", pep_banner_directive.WithdrawnBanner) + app.setup_extension("pep_sphinx_extensions.pep_processor.parsing.pep_soft_keyword") + # Register event callbacks app.connect("builder-inited", _update_config_for_builder) # Update configuration values for builder used app.connect("env-before-read-docs", create_pep_zero) # PEP 0 hook diff --git a/pep_sphinx_extensions/pep_processor/parsing/pep_soft_keyword.py b/pep_sphinx_extensions/pep_processor/parsing/pep_soft_keyword.py new file mode 100644 index 00000000000..8fec6664616 --- /dev/null +++ b/pep_sphinx_extensions/pep_processor/parsing/pep_soft_keyword.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from docutils import nodes +from pygments.lexers.python import PythonLexer +from pygments.token import Keyword, Name +from sphinx import addnodes + +if TYPE_CHECKING: + from collections.abc import Iterator + + from sphinx.application import Sphinx + from sphinx.environment import BuildEnvironment + +_LANG_PREFIX = "python+soft-keywords:" + + +def _keywords_from_language(language: str) -> tuple[str, ...] | None: + if not language.startswith(_LANG_PREFIX): + return None + keywords = tuple(w for w in language[len(_LANG_PREFIX) :].split(",") if w) + return keywords or None + + +def _soft_keyword_lexer(keywords: tuple[str, ...]) -> type[PythonLexer]: + words = frozenset(keywords) + + class SoftKeywordPythonLexer(PythonLexer): + name = f"Python (+ {', '.join(keywords)})" + aliases: list[str] = [] + filenames: list[str] = [] # don't shadow the real Python lexer + mimetypes: list[str] = [] + url = "" + + def get_tokens_unprocessed( + self, text: str, stack: tuple[str, ...] = ("root",) + ) -> Iterator[tuple[int, object, str]]: + for index, token, value in super().get_tokens_unprocessed(text, stack): + if value in words and token in Name: + yield index, Keyword, value + else: + yield index, token, value + + return SoftKeywordPythonLexer + + +def _init_env(app: Sphinx, env: BuildEnvironment, docnames: list[str]) -> None: + if not hasattr(env, "pep_soft_keywords"): + env.pep_soft_keywords = {} + + +def _collect_languages(app: Sphinx, doctree: nodes.document) -> None: + found = set() + for node in doctree.findall(nodes.literal_block): + if keywords := _keywords_from_language(node.get("language", "")): + found.add(keywords) + for node in doctree.findall(addnodes.highlightlang): + if keywords := _keywords_from_language(node.get("lang", "")): + found.add(keywords) + + if found: + app.env.pep_soft_keywords[app.env.docname] = found + else: + app.env.pep_soft_keywords.pop(app.env.docname, None) + + +def _merge_info( + app: Sphinx, + env: BuildEnvironment, + docnames: list[str], + other: BuildEnvironment, +) -> None: + env.pep_soft_keywords.update(getattr(other, "pep_soft_keywords", {})) + + +def _register_lexers(app: Sphinx, env: BuildEnvironment) -> None: + for keyword_sets in env.pep_soft_keywords.values(): + for keywords in keyword_sets: + app.add_lexer(_LANG_PREFIX + ",".join(keywords), _soft_keyword_lexer(keywords)) + + +def setup(app: Sphinx) -> dict[str, bool]: + app.connect("env-before-read-docs", _init_env) + app.connect("doctree-read", _collect_languages) + app.connect("env-merge-info", _merge_info) + app.connect("env-updated", _register_lexers) + return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/peps/pep-0842.rst b/peps/pep-0842.rst index 6f8c55cfb4e..273d59cae5f 100644 --- a/peps/pep-0842.rst +++ b/peps/pep-0842.rst @@ -17,7 +17,7 @@ intent about the visibility of variables from outside the module. For example: -.. code-block:: python +.. code-block:: python+soft-keywords:export # spam.py from mypackage export name @@ -741,7 +741,7 @@ the name of the variable is passed as the first positional argument to the To visualize, the following code: -.. code-block:: python +.. code-block:: python+soft-keywords:export export NAME1, NAME2 @@ -777,13 +777,13 @@ and then ``export``\ ed. As an example, the following code: -.. code-block:: python +.. code-block:: python+soft-keywords:export export NAME1, NAME2 = VALUE1, VALUE2 is semantically equivalent to: -.. code-block:: python +.. code-block:: python+soft-keywords:export NAME1 = VALUE1 NAME2 = VALUE2 @@ -794,7 +794,7 @@ statements (``a = b``, ``a, b = c, d``, etc), and individual assignments that contain a type annotation (``a: type = b``; in contrast, a standalone ``export a: type`` is not valid). For example, each of the following are valid: -.. code-block:: python +.. code-block:: python+soft-keywords:export export hello = "world" export my, hovercraft = "full of", "eels" @@ -833,7 +833,7 @@ A function definition or class definition statement can be prefixed with To visualize, the following code: -.. code-block:: python +.. code-block:: python+soft-keywords:export export def NAME1(): ... @@ -843,7 +843,7 @@ To visualize, the following code: is semantically equivalent to: -.. code-block:: python +.. code-block:: python+soft-keywords:export def NAME1(): ... @@ -891,13 +891,13 @@ the imported names. For example, the following code: -.. code-block:: python +.. code-block:: python+soft-keywords:export from MODULE export NAME1, NAME2 is semantically equivalent to: -.. code-block:: python +.. code-block:: python+soft-keywords:export from MODULE import NAME1, NAME2 export NAME1, NAME2 @@ -908,7 +908,7 @@ using it elsewhere is a :class:`SyntaxError`. Lazy imports, as described by :pep:`810`, are also allowed to be used with ``from`` exports. For example: -.. code-block:: python +.. code-block:: python+soft-keywords:export lazy from foo export bar @@ -1107,7 +1107,7 @@ Add a ``private`` keyword for class bodies During discussion of this proposal, it was suggested to add a ``private`` keyword for use in classes. For example: -.. code-block:: python +.. code-block:: python+soft-keywords:private class Something: private def hello(self):