WIP: ISARIC data schema integration

.github/workflows/build.yml

@@ -26,7 +26,7 @@ jobs:
       - name: Install build dependencies
         run: |
           python3 -m pip install uv
-          uv sync --verbose --active --group build --no-install-project --no-cache --refresh
+          uv sync --verbose --active --group build --no-install-project --no-cache --refresh --no-managed-python
 
       - name: Build package artifacts and install wheel
         run: |

.github/workflows/test.yml

@@ -34,6 +34,10 @@ jobs:
         run: |
           make test
 
+      - name: Run doctests (example snippets in docstrings)
+        run: |
+            uv run python3 -m doctest -v src/isaricanalytics/*.py
+
       - name: Upload coverage.xml artifact
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
         with:

Makefile

@@ -131,3 +131,7 @@ test: clean
 	                               --tb=native \
 	                               --verbosity=3 \
 	                               tests/unit
+
+doctest: clean
+	@echo "$(PACKAGE_NAME)[$(BRANCH)@$(HEAD)]: Running Python doctests"
+	PYTHONPATH="isaricanalytics" python3 -m doctest -v isaricanalytics/*.py

docs/index.rst

@@ -4,8 +4,9 @@ ISARIC Analytics
 
 The `ISARIC Analytics <https://github.com/ISARICResearch/IsaricAnalytics/>`_ library (:code:`isaricanalytics`) is for users requiring or interested in:
 
-- REDCap data extraction
-- Data analytics and/or visualisation
+- Raw data extraction and cleaning from `REDCap <https://projectredcap.org>`_ project databases
+- Transforming custom raw clinical datasets into the `ISARIC data schema <https://isaric-arc.readthedocs.io/en/latest/sources/isaric-data-schema.html>`_
+- Data analytics & visualisation with `Pandas <pandas.pydata.org>`_ and `Plotly <plotly.com/python>`_
 
 in the setting of clinical epidemiology.
 
@@ -40,12 +41,12 @@ The documentation is currently limited mainly to an :ref:`API reference <api-ref
 
 See the linked pages below for more information on how to use the libraries or to contribute to their development. Some pages may still be under development.
 
-
 .. toctree::
    :maxdepth: 1
    :caption: Contents:
 
    sources/redcap-data
+   sources/isaric-data-schema
    sources/analytics
    sources/visualisation/index
    sources/citing-isaric-analytics

docs/sources/api-reference.rst

@@ -11,7 +11,9 @@ API Reference
    :name: api_reference_toc
    :caption: Contents:
 
-   isaricanalytics/redcap_data
+   isaricanalytics/redcap-data
+   isaricanalytics/isaric-transformations
+   isaricanalytics/isaric-data-schema
    isaricanalytics/analytics
    isaricanalytics/visualisation
    isaricanalytics/utils

docs/sources/isaricanalytics/isaric-data-schema.rst

@@ -0,0 +1,7 @@
+======================================
+``isaricanalytics.isaric_data_schema``
+======================================
+
+.. automodule:: isaricanalytics.isaric_data_schema
+    :members:
+    :special-members:

docs/sources/isaricanalytics/isaric-transformations.rst

@@ -0,0 +1,7 @@
+==========================================
+``isaricanalytics.isaric_transformations``
+==========================================
+
+.. automodule:: isaricanalytics.isaric_transformations
+    :members:
+    :special-members:

pyproject.toml

@@ -49,10 +49,12 @@
 
   requires-python = ">=3.12"
   dependencies = [
+      "adtl>=0.13.3",
+      "isaric-bridge@git+https://github.com/ISARICResearch/BRIDGE@v1.2",
       "kaleido>=1.0.0",
       "lifelines==0.30.0",
-      "numpy==1.26.4",
-      "pandas==2.2.1",
+      "numpy==2.2.6",
+      "pandas==2.3.3",
       "plotly@git+https://github.com/plotly/plotly.py",
       "requests>=2.32.4",
       "scikit-learn==1.5.2",
@@ -72,18 +74,18 @@
   build-backend = "hatchling.build"
 
 [tool.hatch.version]
-  path = "isaricanalytics/__init__.py"
+  path = "src/isaricanalytics/__init__.py"
 
 [tool.hatch.metadata]
 allow-direct-references = true
 
 [tool.hatch.build.targets.sdist]
-  packages = ["isaricanalytics", "docs/sources/plot-gallery/",]
+  packages = ["src/isaricanalytics", "docs/sources/plot-gallery/",]
 
 [tool.hatch.build.targets.sdist.force-include]
 
 [tool.hatch.build.targets.wheel]
-  packages = ["isaricanalytics", "docs/sources/plot-gallery",]
+  packages = ["src/isaricanalytics", "docs/sources/plot-gallery",]
 
 [tool.hatch.build.targets.wheel.force-include]
 

src/isaricanalytics/isaric_data_schema.py

@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+__all__ = [
+    "transform_to_isaric_data_schema",
+]
+
+
+# -- IMPORTS --
+
+# -- Standard libraries --
+import pathlib
+import typing
+import warnings as warnings_
+
+# -- 3rd party libraries --
+import adtl
+import pandas as pd
+
+# -- Internal ISARIC libraries --
+from bridge.arc.arc_api import ArcApiClient, ArcApiClientError
+from bridge.arc.arc_core import get_arc
+
+import isaricanalytics.isaric_transformations as tf
+
+
+class IsaricDataSchemaTransformationException(Exception): ...
+
+
+def _warn_on_non_arc_columns(
+    column_set: typing.Iterable, column_type: str, arc_version: str, stacklevel=2
+) -> None:
+    """Warns on the transform function detecting non-ARC columns in the core or long schema tables."""  # noqa: E501
+    columns_str = ", ".join(map(lambda s: f'"{s}"', sorted(column_set)))
+    msg = (
+        f"\n\nThe following {column_type} table columns are not in ARC "
+        f"{arc_version}: {columns_str}\n\nPlease check these columns."
+    )
+    warnings_.warn(msg, DeprecationWarning, stacklevel=stacklevel)
+
+
+def transform_to_isaric_data_schema(
+    parser_file: str | pathlib.Path,
+    data_file: str | pathlib.Path,
+    arc_version: str | None = None,
+) -> dict[str, pd.DataFrame]:
+    """:py:class:`dict` : A dict of ISARIC schema-compliant short/core- and long-format datasets as Pandas dataframes.
+
+    Parameters
+    ----------
+    parser_file : str or pathlib.Path
+        The parser TOML file path.
+
+    data_file : str or pathlib.Path
+        The data file path.
+
+    arc_version : str or None, default=None
+        Optional ARC version string used to fetch the ARC data dictionary
+        associated with the ARC version; defaults to ``None``.
+
+    Returns
+    -------
+    dict
+        A dict of two Pandas dataframes representing short- and long-format
+        ISARIC schema-compliant transforms of the original dataset.
+
+    Raises
+    ------
+    IsaricDataSchemaTransformationException
+        In case of an ARC API client exception.
+    """  # noqa : E501
+    # Call ADTL to parse the data and retrieve the core and long-format tables.
+    ids_tables = adtl.parse(parser_file, data_file, include_transform=tf.__file__)
+
+    # An ARC version is required to fetch the ARC data dictionary. If no ARC
+    # version is provided by the caller, fetch the latest directly from ARC @
+    # GitHub - in case of an exception raise it.
+    if not arc_version:
+        try:
+            arc_version = ArcApiClient().get_arc_version_list()[0]
+        except (ArcApiClientError, IndexError) as e:
+            raise IsaricDataSchemaTransformationException(
+                f'Exception fetching ARC version "{arc_version}": '
+                f"{e}.\n\n Please check that you have provided a valid "
+                "ARC version. If you have then there may be a network- "
+                "related error, so please retry after some time."
+            )
+
+    # Fetch the ARC data dictionary associated with the ARC version - in case
+    # of an exception raise it.
+    try:
+        arc_dd = get_arc(arc_version)[0]
+    except (ArcApiClientError, IndexError) as e:
+        raise IsaricDataSchemaTransformationException(
+            "Exception fetching ARC data dictionary for ARC version "
+            f'"{arc_version}": {e}.\n\n Please check that you have provided a '
+            "valid ARC version. If you have, then please retry after some "
+            "time."
+        )
+
+    # Checking transformed columns in the core and long tables against ARC, and
+    # issuing warnings about any non-ARC columns in either.
+    non_arc_core_columns = set(ids_tables["core"].columns).difference(
+        arc_dd["Variable"]
+    )
+    if non_arc_core_columns:
+        _warn_on_non_arc_columns(
+            non_arc_core_columns, "core", arc_version, stacklevel=2
+        )
+
+    non_arc_long_columns = set(ids_tables["long"].columns).difference(
+        arc_dd["Variable"]
+    )
+    if non_arc_long_columns:
+        _warn_on_non_arc_columns(
+            non_arc_long_columns, "long", arc_version, stacklevel=2
+        )
+
+    return ids_tables

src/isaricanalytics/isaric_transformations.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+__all__ = [
+    "MISSING_ATTRIBUTE_STATUS",
+    "attribute_status_fill",
+    "values_strip_missing",
+]
+
+
+# -- IMPORTS --
+
+# -- Standard libraries --
+from enum import Enum
+
+# -- 3rd party libraries --
+
+# -- Internal libraries --
+
+
+class MISSING_ATTRIBUTE_STATUS(Enum):
+    # Unknown
+    UNKNOWN = "UNK"
+
+    # No information
+    NO_INFORMATION = "NI"
+
+    # Not asked
+    NOT_ASKED = "NASK"
+
+    # Not applicable
+    NOT_APPLICABLE = "NA"
+
+
+def attribute_status_fill(field: str) -> str | None:
+    """:py:class:`str` or :py:class:`NoneType` : Infer attribute status if it is missing.
+
+    Parameters
+    ----------
+    field : str
+        Attribute status field.
+
+    Returns
+    -------
+    str or None
+        The original status field if it is one of the values:
+
+        * ``'UNK'`` - unknown
+        * ``'NI'`` - no information
+        * ``'NASK'`` - not asked
+        * ``'NA'`` - not applicable
+
+        or ``'VAL'`` if it is non-null but different from the above, otherwise
+        ``None``.
+
+    Examples
+    --------
+    >>> attribute_status_fill("UNK")
+    'UNK'
+    >>> attribute_status_fill("NI")
+    'NI'
+    >>> attribute_status_fill("NASK")
+    'NASK'
+    >>> attribute_status_fill("NA")
+    'NA'
+    >>> attribute_status_fill("XYZ")
+    'VAL'
+    >>> attribute_status_fill(None)
+    >>>
+    """  # noqa : E501
+    if field is None:
+        return None
+
+    match field:
+        case (
+            MISSING_ATTRIBUTE_STATUS.UNKNOWN.value
+            | MISSING_ATTRIBUTE_STATUS.NO_INFORMATION.value
+            | MISSING_ATTRIBUTE_STATUS.NOT_ASKED.value
+            | MISSING_ATTRIBUTE_STATUS.NOT_APPLICABLE.value
+        ):
+            return field
+        case _:
+            return "VAL"
+
+
+def values_strip_missing(field: str) -> str | None:
+    """:py:class:`str` or :py:class:`NoneType` : Strip missing attribute status field.
+
+    Parameters
+    ----------
+    field : str
+        Attribute status field.
+
+    Returns
+    -------
+    str or None
+        Null if it is one of the values:
+
+        * ``'UNK'`` - unknown
+        * ``'NI'`` - no information
+        * ``'NASK'`` - not asked
+        * ``'NA'`` - not applicable
+
+        and ``None`` otherwise.
+
+    Examples
+    --------
+    >>> values_strip_missing("UNK")
+    >>>
+    >>> values_strip_missing("NI")
+    >>>
+    >>> values_strip_missing("NASK")
+    >>>
+    >>> values_strip_missing("NA")
+    >>>
+    >>> values_strip_missing("XYZ")
+    'XYZ'
+    >>> values_strip_missing(None)
+    >>>
+    """  # noqa : E501
+
+    match field:
+        case (
+            MISSING_ATTRIBUTE_STATUS.UNKNOWN.value
+            | MISSING_ATTRIBUTE_STATUS.NO_INFORMATION.value
+            | MISSING_ATTRIBUTE_STATUS.NOT_ASKED.value
+            | MISSING_ATTRIBUTE_STATUS.NOT_APPLICABLE.value
+        ):
+            return None
+        case _:
+            return field
+
+
+if __name__ == "__main__":  # pragma: no cover
+    # Doctest the module from the project root using
+    #
+    #     python3 -m doctest -v src/isaricanalytics/isaric_transformations.py  # noqa : E501
+    #
+    import doctest
+
+    doctest.testmod()