Infra: Add a Sphinx extension to highlight custom keywords

pep_sphinx_extensions/__init__.py

@@ -105,6 +105,8 @@ def setup(app: Sphinx) -> dict[str, bool]:
     app.add_directive("superseded", pep_banner_directive.SupersededBanner)
     app.add_directive("withdrawn", pep_banner_directive.WithdrawnBanner)
 
+    app.setup_extension("pep_sphinx_extensions.pep_processor.parsing.pep_soft_keyword")
+
     # Register event callbacks
     app.connect("builder-inited", _update_config_for_builder)  # Update configuration values for builder used
     app.connect("env-before-read-docs", create_pep_zero)  # PEP 0 hook

pep_sphinx_extensions/pep_processor/parsing/pep_soft_keyword.py

@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from docutils import nodes
+from pygments.lexers.python import PythonLexer
+from pygments.token import Keyword, Name
+from sphinx import addnodes
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+    from sphinx.application import Sphinx
+    from sphinx.environment import BuildEnvironment
+
+_LANG_PREFIX = "python+soft-keywords:"
+
+
+def _keywords_from_language(language: str) -> tuple[str, ...] | None:
+    if not language.startswith(_LANG_PREFIX):
+        return None
+    keywords = tuple(w for w in language[len(_LANG_PREFIX) :].split(",") if w)
+    return keywords or None
+
+
+def _soft_keyword_lexer(keywords: tuple[str, ...]) -> type[PythonLexer]:
+    words = frozenset(keywords)
+
+    class SoftKeywordPythonLexer(PythonLexer):
+        name = f"Python (+ {', '.join(keywords)})"
+        aliases: list[str] = []
+        filenames: list[str] = []  # don't shadow the real Python lexer
+        mimetypes: list[str] = []
+        url = ""
+
+        def get_tokens_unprocessed(
+            self, text: str, stack: tuple[str, ...] = ("root",)
+        ) -> Iterator[tuple[int, object, str]]:
+            for index, token, value in super().get_tokens_unprocessed(text, stack):
+                if value in words and token in Name:
+                    yield index, Keyword, value
+                else:
+                    yield index, token, value
+
+    return SoftKeywordPythonLexer
+
+
+def _init_env(app: Sphinx, env: BuildEnvironment, docnames: list[str]) -> None:
+    if not hasattr(env, "pep_soft_keywords"):
+        env.pep_soft_keywords = {}
+
+
+def _collect_languages(app: Sphinx, doctree: nodes.document) -> None:
+    found = set()
+    for node in doctree.findall(nodes.literal_block):
+        if keywords := _keywords_from_language(node.get("language", "")):
+            found.add(keywords)
+    for node in doctree.findall(addnodes.highlightlang):
+        if keywords := _keywords_from_language(node.get("lang", "")):
+            found.add(keywords)
+
+    if found:
+        app.env.pep_soft_keywords[app.env.docname] = found
+    else:
+        app.env.pep_soft_keywords.pop(app.env.docname, None)
+
+
+def _merge_info(
+    app: Sphinx,
+    env: BuildEnvironment,
+    docnames: list[str],
+    other: BuildEnvironment,
+) -> None:
+    env.pep_soft_keywords.update(getattr(other, "pep_soft_keywords", {}))
+
+
+def _register_lexers(app: Sphinx, env: BuildEnvironment) -> None:
+    for keyword_sets in env.pep_soft_keywords.values():
+        for keywords in keyword_sets:
+            app.add_lexer(_LANG_PREFIX + ",".join(keywords), _soft_keyword_lexer(keywords))
+
+
+def setup(app: Sphinx) -> dict[str, bool]:
+    app.connect("env-before-read-docs", _init_env)
+    app.connect("doctree-read", _collect_languages)
+    app.connect("env-merge-info", _merge_info)
+    app.connect("env-updated", _register_lexers)
+    return {"parallel_read_safe": True, "parallel_write_safe": True}

peps/pep-0842.rst

@@ -17,7 +17,7 @@ intent about the visibility of variables from outside the module.
 
 For example:
 
-.. code-block:: python
+.. code-block:: python+soft-keywords:export
 
    # spam.py
    from mypackage export name
@@ -741,7 +741,7 @@ the name of the variable is passed as the first positional argument to the
 
 To visualize, the following code:
 
-.. code-block:: python
+.. code-block:: python+soft-keywords:export
 
    export NAME1, NAME2
 
@@ -777,13 +777,13 @@ and then ``export``\ ed.
 
 As an example, the following code:
 
-.. code-block:: python
+.. code-block:: python+soft-keywords:export
 
    export NAME1, NAME2 = VALUE1, VALUE2
 
 is semantically equivalent to:
 
-.. code-block:: python
+.. code-block:: python+soft-keywords:export
 
    NAME1 = VALUE1
    NAME2 = VALUE2
@@ -794,7 +794,7 @@ statements (``a = b``, ``a, b = c, d``, etc), and individual assignments
 that contain a type annotation (``a: type = b``; in contrast, a standalone
 ``export a: type`` is not valid). For example, each of the following are valid:
 
-.. code-block:: python
+.. code-block:: python+soft-keywords:export
 
    export hello = "world"
    export my, hovercraft = "full of", "eels"
@@ -833,7 +833,7 @@ A function definition or class definition statement can be prefixed with
 
 To visualize, the following code:
 
-.. code-block:: python
+.. code-block:: python+soft-keywords:export
 
    export def NAME1():
       ...
@@ -843,7 +843,7 @@ To visualize, the following code:
 
 is semantically equivalent to:
 
-.. code-block:: python
+.. code-block:: python+soft-keywords:export
 
    def NAME1():
       ...
@@ -891,13 +891,13 @@ the imported names.
 
 For example, the following code:
 
-.. code-block:: python
+.. code-block:: python+soft-keywords:export
 
    from MODULE export NAME1, NAME2
 
 is semantically equivalent to:
 
-.. code-block:: python
+.. code-block:: python+soft-keywords:export
 
    from MODULE import NAME1, NAME2
    export NAME1, NAME2
@@ -908,7 +908,7 @@ using it elsewhere is a :class:`SyntaxError`.
 Lazy imports, as described by :pep:`810`, are also allowed to be used with ``from``
 exports. For example:
 
-.. code-block:: python
+.. code-block:: python+soft-keywords:export
 
    lazy from foo export bar
 
@@ -1107,7 +1107,7 @@ Add a ``private`` keyword for class bodies
 During discussion of this proposal, it was suggested to add a ``private``
 keyword for use in classes. For example:
 
-.. code-block:: python
+.. code-block:: python+soft-keywords:private
 
    class Something:
       private def hello(self):