diff --git a/.gitignore b/.gitignore index b7de905d092b..5790e4736541 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ build_cython cython_debug *.egg-info +# Airspeed Velocity (asv) benchmark environments, results and html +benchmarks/.asv/ + # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 113b0060fdfd..f769cfa48623 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ This release is compatible with NumPy 2.5. * Bumped the default minimum required DPC++ compiler version to `2026.1.1` and migrated to the OpenCL ICD loader from the conda-forge `ocl-icd-system` (Linux) and `khronos-opencl-icd-loader` (Windows) packages [#2905](https://github.com/IntelPython/dpnp/pull/2905) * Linked the `dpnp_backend_c` library against only the MKL SYCL domains it uses (`BLAS`, `RNG`, `VM`) [#3012](https://github.com/IntelPython/dpnp/pull/3012) * `dpnp` uses pybind11 3.1.0 [#3015](https://github.com/IntelPython/dpnp/pull/3015) +* Reworked the ASV benchmarks and added end-to-end workload benchmarks derived from dpBench [#2996](https://github.com/IntelPython/dpnp/pull/2996) ### Deprecated diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000000..3927abb70a7f --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,267 @@ +# dpnp ASV Benchmarks + +Performance benchmarks for [dpnp](https://github.com/IntelPython/dpnp) using +[Airspeed Velocity (ASV)](https://asv.readthedocs.io/en/stable/). + +## Coverage + +| File | API | Benchmarks | Params | Sizes | +|------|-----|------------|--------|-------| +| `bench_dpbench.py` | `dpnp` (end-to-end workloads) | `BlackScholes`, `L2Norm`, `PairwiseDistance`, `Rambo`, `Gpairs` | `preset`, `precision` | dpBench presets `S`, `M16Gb`, `M`, `L` | +| `bench_elementwise.py` | `dpnp` vs `numpy` | `Elementwise` (26 unary math functions) | `executor`, `size`, `dtype` | 2^16, 2^20, 2^24 | +| `bench_linalg.py` | `dpnp` vs `numpy` (`dot`, `matmul`, `inner`, `einsum`) | `MatMul` | `executor`, `order`, `dtype` | 16 to 1024 square | +| `bench_random.py` | `dpnp.random` vs `numpy.random` | `Sample` (`rand`, `randn`, `random_sample`) | `executor`, `size` | 2^16, 2^20, 2^24 | + +### dpBench workloads + +`bench_dpbench.py` runs a set of dpnp workloads derived from +[dpBench](https://github.com/IntelPython/dpbench), which live in +`benchmarks/benchmarks/dpbench/workloads`. They measure the end-to-end time of a +whole workload rather than of an individual API call. + +| Workload | Domain | +| ------------------- | ------------------ | +| `black_scholes` | Finance | +| `l2_norm` | Distance Compute | +| `pairwise_distance` | Distance Compute | +| `rambo` | Particle Physics | +| `gpairs` | Astrophysics | + +Host input data is generated and copied to the device the way dpBench does, and +each kernel ends with `dpnp.synchronize_array_data`, so a single call blocks +until the device work has finished. dpBench is not a dependency. See +[`benchmarks/dpbench/README.md`](benchmarks/dpbench/README.md) for the +source-to-module mapping and the intended differences. + +## Device and precision + +dpnp allocates on the default SYCL device. Use `ONEAPI_DEVICE_SELECTOR` to +target a specific one: + +```bash +ONEAPI_DEVICE_SELECTOR=level_zero:gpu asv run \ + --python=same \ + --launch-method spawn \ + --quick +``` + +**The parameter matrix is the same on every machine.** All four dpBench presets +are declared statically, so a given benchmark has the same parameter set +everywhere and results are comparable across devices and across the CI pool. +What varies per device is which of those points *run*: `setup` calls +`_dpbench_runner.preset_fits` and raises `SkipNotImplemented` for any preset +whose estimated peak element count (the workload's `peak_elements`, taken at the +wider of the two precisions) exceeds **0.25** of the device's `global_mem_size`. +So a large discrete GPU exercises the bigger problem sizes automatically while a +small iGPU reports `S` and skips the rest, and a skipped point stays visible as a +skip rather than vanishing from the matrix. + +The cheapest preset always runs. If even that does not fit, it is attempted +anyway so the failure is a loud allocation error rather than silence. + +Note that dpBench's preset names are not ordered by size: `M16Gb` is *smaller* +than `M` for every workload except `rambo`, where it is larger and equal to `L`. +Anything that needs the cheapest preset sorts explicitly rather than relying on +declaration order. + +**Both precisions are benchmarked.** Devices without fp64 support (common on +iGPUs) skip the `double` points via `SkipNotImplemented` rather than failing, so +such a device still produces `single` results. The `float64` points of +`bench_elementwise.py` and `bench_linalg.py` skip the same way for the `dpnp` +executor; the `numpy` executor is unaffected. dpBench's own configs request +`double` throughout, and that value is kept in each workload's `PRECISION` for +reference. + +No benchmark module opens a SYCL queue at import time, so benchmark discovery +and `asv check` work on a machine with no usable device; only `setup` needs one. + +One caveat on comparability: ASV keys results by machine, commit and +environment, not by device. Benchmarking two devices on the same host therefore +overwrites one set of results with the other. Give each device its own machine +name when you do that: + +```bash +ONEAPI_DEVICE_SELECTOR=level_zero:gpu asv run --python=same \ + --launch-method spawn --machine "$(hostname)-gpu" +``` + +## Notes on Measurement + +### Process launch method + +**Always pass `--launch-method spawn`.** ASV defaults to a forkserver, which +`fork()`s a process that has already initialized a SYCL runtime; the SYCL +runtime is multi-threaded and not fork-safe, so benchmarks may hang until +`default_benchmark_timeout` expires (reported as `failed`) or fail with +`USM Allocation` errors on `level_zero` devices. `spawn` starts a fresh +interpreter per benchmark and avoids this entirely. + +### Asynchronous execution + +**Every timed body that runs dpnp work must block on it.** dpnp enqueues to a +SYCL queue and returns before the kernel has run, so a body that does not block +measures submission rather than execution. Unsynchronized, a 1024x1024 float32 +`dot` measured **0.4 ms** against **36 ms** synchronized on a CPU device -- which +would have reported dpnp as an order of magnitude faster than NumPy on work +where it is in fact slightly slower. + +The dpBench workloads each end with `dpnp.synchronize_array_data`, and the +comparison suites obtain a synchronizer from `_utils.make_synchronizer` in +`setup` and pass every result through it (`self.sync(...)`). For the `numpy` +executor the synchronizer does nothing. + +### First-call costs + +The first call on a fresh queue pays SYCL kernel/JIT and allocator warmup. +`WorkloadRunner.setup` therefore runs each workload once before ASV starts +timing it, so the dpBench suite is warmed explicitly. The `bench_elementwise.py`, +`bench_linalg.py` and `bench_random.py` suites do **not** warm up and rely on +ASV's default `warmup_time`. + +### Validation + +Each workload ships the NumPy `reference` implementation from dpBench. On the +cheapest preset, `setup` compares the dpnp results for all `OUTPUT_ARGS` +against it (mirroring dpBench's `infrastructure/benchmark_validation.py`, same +`1e-05` relative-error tolerance). A numerically wrong kernel therefore fails +the benchmark instead of being silently timed. Validation runs outside the +timed region and does not affect the reported numbers. + +Only the cheapest preset is validated: the reference runs on the host and at the +larger presets costs far more than the benchmark it guards -- tens of seconds +for `pairwise_distance` at `M16Gb` -- while checking numerics that do not depend +on the problem size. + +### Noise at small presets + +The smallest sizes are dominated by per-call dispatch overhead and are +noticeably noisier. On a CPU device the run-to-run spread of the median at `S` +was measured between **2%** and **25%** across workloads, against the **20%** +`regressions_thresholds` in `asv.conf.json`, whereas the larger presets settled +to a few percent. Treat `S` as a smoke-test size only and do not use it for +regression gating; prefer the largest preset the device fits. + +## Running Benchmarks + +ASV cannot build dpnp -- it is a SYCL/DPC++ extension that requires the Intel +oneAPI compiler and a lengthy build -- so the benchmarks always run against an +**existing environment** that already has dpnp installed. A bare `asv run` is +not supported; always pass `--python=same` or `--environment existing:`. + +Create an environment +[following these instructions](https://intelpython.github.io/dpnp/quick_start_guide.html), +then install the benchmarking tooling into it: + +```bash +conda install -c conda-forge asv scipy +``` + +`scipy` is needed because `scipy.special.erf` is used by the NumPy reference +that the `black_scholes` benchmark validates its dpnp results against. + +Do **not** use `pip install ".[benchmark]"` for an environment that already has +dpnp: dpnp is a scikit-build project, so pip reinstalls the `dpnp` package +itself and triggers a full oneAPI/DPC++ rebuild of the backend just to pull in +two pure-Python dependencies. The `benchmark` extra in `pyproject.toml` records +those two dependencies for the case where dpnp is being built from source +anyway; note that the usual editable-install invocation passes `--no-deps`, so +it does *not* install them: + +```bash +pip install --no-build-isolation --no-deps -e . +conda install -c conda-forge asv scipy +``` + +All commands below are run from the `benchmarks/` directory, where +`asv.conf.json` lives. + +Register the machine once. Without this a non-interactive or CI run aborts with +`No information stored about machine`: + +```bash +asv machine --yes +``` + +Validate the whole suite without running it. This is cheap and catches broken +signatures and import errors; it accepts no `--bench`, so it is all-or-nothing: + +```bash +asv check --python=same +``` + +Smoke-run the benchmarks, optionally scoped with `--bench`: + +```bash +asv run --python=same --launch-method spawn --quick --bench bench_dpbench +``` + +This only *prints* results. Without `--set-commit-hash` ASV discards them, so +`asv compare` and `asv publish` will see nothing. + +To record results, assert which revision the installed dpnp corresponds to: + +```bash +asv run --python=same --launch-method spawn --set-commit-hash HEAD +``` + +ASV does not verify that claim -- it is your assertion -- and the +`For dpnp commit ...` progress line prints the branch head rather than the +value passed, so trust the result filename or `asv show`. + +Pointing ASV at an interpreter explicitly works too: + +```bash +asv run --environment existing:/full/conda/path/envs/dpnp_env/bin/python \ + --launch-method spawn +``` + +`asv.conf.json` sets `branches` to `HEAD` rather than to named branches, so that +results recorded on a feature branch are picked up. With named branches +`asv publish` reports `Couldn't find in branches (...)` and silently +drops them. + +### Comparing two revisions + +`asv continuous` and any `` range spec cannot be used here: ASV refuses +a range spec when it cannot install the project into the environment. Compare +two recorded runs instead. Rebuild dpnp in the same environment between them, +and omit `--quick` so the statistics path engages: + +```bash +# against the old build +asv run --python=same --launch-method spawn --set-commit-hash +# rebuild/reinstall dpnp, then +asv run --python=same --launch-method spawn --set-commit-hash +asv compare +``` + +View recorded results in a browser: + +```bash +asv publish +asv preview +``` + +## Writing new benchmarks + +Read ASV's guidelines for writing benchmarks +[here](https://asv.readthedocs.io/en/stable/writing_benchmarks.html). + +Parameter axes shared by two or more `bench_*` modules live in `_utils.py`; +single-use axes stay in the module that needs them. Two rules keep results +usable: + +* Keep parameter values plain strings, numbers or tuples. A live module or dtype + object renders as `` in the result tables and embeds + a local path in the result identity. +* Keep `params` static. Deriving an axis from the machine makes rows + incomparable between devices; decide per-device behaviour in `setup` instead, + by raising `SkipNotImplemented` (see `bench_dpbench._Workload.setup`). +* Block on dpnp work inside the timed body, or you are timing submission -- see + [Asynchronous execution](#asynchronous-execution). + +To add another dpBench workload, follow +[`benchmarks/dpbench/README.md`](benchmarks/dpbench/README.md), then add a +benchmark class for it to `bench_dpbench.py`. Copy an existing one: it is a +banner, a docstring, a `WORKLOAD` attribute and a one-line `time_*` method -- +the parameter axes are inherited from `_Workload`. diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json index 3d0e7f88d55f..0036108a6250 100644 --- a/benchmarks/asv.conf.json +++ b/benchmarks/asv.conf.json @@ -1,89 +1,24 @@ { - // The version of the config file format. Do not change, unless - // you know what you are doing. "version": 1, - - // The name of the project being benchmarked "project": "dpnp", - - // The project's homepage - "project_url": "", - - // The URL or local path of the source code repository for the - // project being benchmarked + "project_url": "https://github.com/IntelPython/dpnp", + "show_commit_url": "https://github.com/IntelPython/dpnp/commit/", "repo": "..", - - // List of branches to benchmark. If not provided, defaults to "master" - // (for git) or "tip" (for mercurial). "branches": [ "HEAD" ], - - // The DVCS being used. If not set, it will be automatically - // determined from "repo" by looking at the protocol in the URL - // (if remote), or by looking for special directories, such as - // ".git" (if local). - "dvcs": "git", - - // The tool to use to create environments. May be "conda", - // "virtualenv" or other value depending on the plugins in use. - // If missing or the empty string, the tool will be automatically - // determined by looking for tools on the PATH environment - // variable. - "environment_type": "virtualenv", - - // the base URL to show a commit for the project. - "show_commit_url": "", - - // The Pythons you'd like to test against. If not provided, defaults - // to the current version of Python used to run `asv`. - "pythons": [ - "3.7" + "environment_type": "conda", + "conda_channels": [ + "https://software.repos.intel.com/python/conda/", + "conda-forge" ], - - // The matrix of dependencies to test. Each key is the name of a - // package (in PyPI) and the values are version numbers. An empty - // list indicates to just test against the default (latest) - // version. - "matrix": { - "Cython": [], - }, - - // The directory (relative to the current directory) that benchmarks are - // stored in. If not provided, defaults to "benchmarks" "benchmark_dir": "benchmarks", - - // The directory (relative to the current directory) to cache the Python - // environments in. If not provided, defaults to "env" - "env_dir": "env", - - // The directory (relative to the current directory) that raw benchmark - // results are stored in. If not provided, defaults to "results". - "results_dir": "results", - - // The directory (relative to the current directory) that the html tree - // should be written to. If not provided, defaults to "html". - "html_dir": "html", - - // The number of characters to retain in the commit hashes. - // "hash_length": 8, - - // `asv` will cache wheels of the recent builds in each - // environment, making them faster to install next time. This is - // number of builds to keep, per environment. - "build_cache_size": 8, - - // The commits after which the regression search in `asv publish` - // should start looking for regressions. Dictionary whose keys are - // regexps matching to benchmark names, and values corresponding to - // the commit (exclusive) after which to start looking for - // regressions. The default is to start from the first commit - // with results. If the commit is `null`, regression detection is - // skipped for the matching benchmark. - // - // "regressions_first_commits": { - // "some_benchmark": "352cdf", // Consider regressions only after this - // commit - // "another_benchmark": null, // Skip regression detection altogether - // } + "env_dir": ".asv/env", + "results_dir": ".asv/results", + "html_dir": ".asv/html", + "build_cache_size": 2, + "default_benchmark_timeout": 500, + "regressions_thresholds": { + ".*": 0.2 + } } diff --git a/benchmarks/benchmarks/__init__.py b/benchmarks/benchmarks/__init__.py index 75e277849b30..450f408d07fa 100644 --- a/benchmarks/benchmarks/__init__.py +++ b/benchmarks/benchmarks/__init__.py @@ -26,4 +26,4 @@ # THE POSSIBILITY OF SUCH DAMAGE. # ***************************************************************************** -from . import common +"""ASV benchmarks for dpnp.""" diff --git a/benchmarks/benchmarks/_utils.py b/benchmarks/benchmarks/_utils.py new file mode 100644 index 000000000000..8beb7192124c --- /dev/null +++ b/benchmarks/benchmarks/_utils.py @@ -0,0 +1,81 @@ +# ***************************************************************************** +# Copyright (c) 2026, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""Shared helpers and parameter axes for the dpnp ASV benchmarks.""" + +import dpctl +import numpy +from asv_runner.benchmarks.mark import SkipNotImplemented + +import dpnp + +# executor axis, keyed by name so ASV's tables stay readable +_EXECUTORS = {"dpnp": dpnp, "numpy": numpy} +_EXECUTOR_NAMES = list(_EXECUTORS) + +# axes shared across multiple files +_SIZES_1D = [2**16, 2**20, 2**24] +_DTYPES = ["float64", "float32", "int64", "int32"] + +_DEFAULT_QUEUE = None + + +def default_queue(): + """Return a queue on dpnp's default device, created on first use. + + Deferring creation keeps benchmark discovery free of a device requirement. + """ + global _DEFAULT_QUEUE + + if _DEFAULT_QUEUE is None: + _DEFAULT_QUEUE = dpctl.SyclQueue() + return _DEFAULT_QUEUE + + +def make_synchronizer(executor): + """Return a callable blocking until ``executor``'s work has finished. + + dpnp enqueues asynchronously, so a timed body that does not block measures + submission rather than execution. NumPy is synchronous. + """ + if executor == "dpnp": + return dpnp.synchronize_array_data + return lambda result: None + + +def skip_unsupported_dtype(q, dtype): + """Skip the benchmark if the device does not support the given dtype.""" + dtype = dpnp.dtype(dtype) + device = q.sycl_device + if ( + dtype in (dpnp.float64, dpnp.complex128) and not device.has_aspect_fp64 + ) or (dtype == dpnp.float16 and not device.has_aspect_fp16): + raise SkipNotImplemented( + f"Skipping benchmark for {dtype.name} on this device" + + " as it is not supported." + ) diff --git a/benchmarks/benchmarks/bench_dpbench.py b/benchmarks/benchmarks/bench_dpbench.py new file mode 100644 index 000000000000..915288d47a4e --- /dev/null +++ b/benchmarks/benchmarks/bench_dpbench.py @@ -0,0 +1,155 @@ +# ***************************************************************************** +# Copyright (c) 2026, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""Benchmarks for whole dpnp workloads derived from dpBench. + +One class per workload, parametrized by data-size preset and floating-point +precision. See ``dpbench/README.md`` for where the workloads come from. +""" + +from asv_runner.benchmarks.mark import SkipNotImplemented + +from ._utils import default_queue, skip_unsupported_dtype +from .dpbench import _dpbench_runner as runner +from .dpbench.workloads import ( + black_scholes, + gpairs, + l2_norm, + pairwise_distance, + rambo, +) + +# Static axes, so the parameter matrix is the same on every machine. What a +# device cannot run is skipped in setup instead. +_PRESETS = ["S", "M16Gb", "M", "L"] +_PRECISIONS = list(runner.PRECISIONS) + + +class _Workload: + """Shared setup for one dpBench-derived workload. + + Subclasses declare ``WORKLOAD`` and a single ``time_*`` method. Defines no + ``time_*`` itself, so ASV does not discover it as a benchmark. + """ + + WORKLOAD = None + params = [_PRESETS, _PRECISIONS] + param_names = ["preset", "precision"] + + def setup(self, preset, precision): + queue = default_queue() + skip_unsupported_dtype(queue, runner.float_dtype(precision)) + + if preset not in self.WORKLOAD.PRESETS: + raise SkipNotImplemented( + f"{self.WORKLOAD.NAME} has no {preset} preset." + ) + + if not runner.preset_fits(self.WORKLOAD, preset, queue.sycl_device): + raise SkipNotImplemented( + f"Skipping the {preset} preset as its estimated peak footprint" + " does not fit this device's memory." + ) + + self._runner = runner.WorkloadRunner(self.WORKLOAD, preset, precision) + self._runner.setup() + + # Validating the larger presets costs far more than the benchmark it + # guards, and the numerics do not depend on the problem size. + if preset == runner.presets_by_size(self.WORKLOAD)[0]: + self._runner.validate() + + +# --------------------------------------------------------------------------- +# Black-Scholes formula (finance) +# --------------------------------------------------------------------------- + + +class BlackScholes(_Workload): + """European option pricing over an array of options.""" + + WORKLOAD = black_scholes + + def time_black_scholes(self, preset, precision): + self._runner.run() + + +# --------------------------------------------------------------------------- +# L2 norm (distance compute) +# --------------------------------------------------------------------------- + + +class L2Norm(_Workload): + """Row-wise Euclidean norm of an (npoints, dims) point cloud.""" + + WORKLOAD = l2_norm + + def time_l2_norm(self, preset, precision): + self._runner.run() + + +# --------------------------------------------------------------------------- +# Pairwise distance (distance compute) +# --------------------------------------------------------------------------- + + +class PairwiseDistance(_Workload): + """Full (npoints, npoints) Euclidean distance matrix via GEMM.""" + + WORKLOAD = pairwise_distance + + def time_pairwise_distance(self, preset, precision): + self._runner.run() + + +# --------------------------------------------------------------------------- +# Rambo (particle physics) +# --------------------------------------------------------------------------- + + +class Rambo(_Workload): + """Phase-space four-momenta generation for collision events.""" + + WORKLOAD = rambo + + def time_rambo(self, preset, precision): + self._runner.run() + + +# --------------------------------------------------------------------------- +# Galaxy pairs (astrophysics) +# --------------------------------------------------------------------------- + + +class Gpairs(_Workload): + """Weighted galaxy-pair counts binned by separation radius.""" + + WORKLOAD = gpairs + + def time_gpairs(self, preset, precision): + self._runner.run() diff --git a/benchmarks/benchmarks/bench_elementwise.py b/benchmarks/benchmarks/bench_elementwise.py index 10cd0aea8397..3be1dda3f536 100644 --- a/benchmarks/benchmarks/bench_elementwise.py +++ b/benchmarks/benchmarks/bench_elementwise.py @@ -26,105 +26,107 @@ # THE POSSIBILITY OF SUCH DAMAGE. # ***************************************************************************** -import numpy +"""Benchmarks for unary elementwise math functions, dpnp against NumPy.""" -import dpnp +from ._utils import ( + _DTYPES, + _EXECUTOR_NAMES, + _EXECUTORS, + _SIZES_1D, + default_queue, + make_synchronizer, + skip_unsupported_dtype, +) -from .common import Benchmark +class Elementwise: + """Unary elementwise ufuncs, dpnp against NumPy.""" -# asv run --python=python --bench Elementwise -# --quick option will run every case once -# but looks like first execution has additional overheads -# (need to be investigated) -class Elementwise(Benchmark): - executors = {"dpnp": dpnp, "numpy": numpy} - params = [ - ["dpnp", "numpy"], - [2**16, 2**20, 2**24], - ["float64", "float32", "int64", "int32"], - ] + params = [_EXECUTOR_NAMES, _SIZES_1D, _DTYPES] param_names = ["executor", "size", "dtype"] def setup(self, executor, size, dtype): - self.np = self.executors[executor] + self.np = _EXECUTORS[executor] + if executor == "dpnp": + skip_unsupported_dtype(default_queue(), dtype) + self.sync = make_synchronizer(executor) dt = getattr(self.np, dtype) self.a = self.np.arange(size, dtype=dt) def time_arccos(self, *args): - self.np.arccos(self.a) + self.sync(self.np.arccos(self.a)) def time_arccosh(self, *args): - self.np.arccosh(self.a) + self.sync(self.np.arccosh(self.a)) def time_arcsin(self, *args): - self.np.arcsin(self.a) + self.sync(self.np.arcsin(self.a)) def time_arcsinh(self, *args): - self.np.arcsinh(self.a) + self.sync(self.np.arcsinh(self.a)) def time_arctan(self, *args): - self.np.arctan(self.a) + self.sync(self.np.arctan(self.a)) def time_arctanh(self, *args): - self.np.arctanh(self.a) + self.sync(self.np.arctanh(self.a)) def time_cbrt(self, *args): - self.np.cbrt(self.a) + self.sync(self.np.cbrt(self.a)) def time_cos(self, *args): - self.np.cos(self.a) + self.sync(self.np.cos(self.a)) def time_cosh(self, *args): - self.np.cosh(self.a) + self.sync(self.np.cosh(self.a)) def time_degrees(self, *args): - self.np.degrees(self.a) + self.sync(self.np.degrees(self.a)) def time_exp(self, *args): - self.np.exp(self.a) + self.sync(self.np.exp(self.a)) def time_exp2(self, *args): - self.np.exp2(self.a) + self.sync(self.np.exp2(self.a)) def time_expm1(self, *args): - self.np.expm1(self.a) + self.sync(self.np.expm1(self.a)) def time_log(self, *args): - self.np.log(self.a) + self.sync(self.np.log(self.a)) def time_log10(self, *args): - self.np.log10(self.a) + self.sync(self.np.log10(self.a)) def time_log1p(self, *args): - self.np.log1p(self.a) + self.sync(self.np.log1p(self.a)) def time_log2(self, *args): - self.np.log2(self.a) + self.sync(self.np.log2(self.a)) def time_rad2deg(self, *args): - self.np.rad2deg(self.a) + self.sync(self.np.rad2deg(self.a)) def time_radians(self, *args): - self.np.radians(self.a) + self.sync(self.np.radians(self.a)) def time_reciprocal(self, *args): - self.np.reciprocal(self.a) + self.sync(self.np.reciprocal(self.a)) def time_sin(self, *args): - self.np.sin(self.a) + self.sync(self.np.sin(self.a)) def time_sinh(self, *args): - self.np.sinh(self.a) + self.sync(self.np.sinh(self.a)) def time_sqrt(self, *args): - self.np.sqrt(self.a) + self.sync(self.np.sqrt(self.a)) def time_square(self, *args): - self.np.square(self.a) + self.sync(self.np.square(self.a)) def time_tan(self, *args): - self.np.tan(self.a) + self.sync(self.np.tan(self.a)) def time_tanh(self, *args): - self.np.tanh(self.a) + self.sync(self.np.tanh(self.a)) diff --git a/benchmarks/benchmarks/bench_linalg.py b/benchmarks/benchmarks/bench_linalg.py index 9d8c08a5e587..e532195652eb 100644 --- a/benchmarks/benchmarks/bench_linalg.py +++ b/benchmarks/benchmarks/bench_linalg.py @@ -26,153 +26,49 @@ # THE POSSIBILITY OF SUCH DAMAGE. # ***************************************************************************** -import numpy +"""Benchmarks for matrix products, dpnp against NumPy.""" -import dpnp +from ._utils import ( + _DTYPES, + _EXECUTOR_NAMES, + _EXECUTORS, + default_queue, + make_synchronizer, + skip_unsupported_dtype, +) -from .common import TYPES1, Benchmark, get_indexes_rand, get_squares_ +# square matrix orders -- local to this suite +_ORDERS = [16, 32, 64, 128, 256, 512, 1024] -class Eindot(Benchmark): - params = [ - [dpnp, numpy], - [16, 32, 64, 128, 256, 512, 1024], - ["float64", "float32", "int64", "int32"], - ] - param_names = ["executor", "size", "dtype"] +# --------------------------------------------------------------------------- +# Square matrix products +# --------------------------------------------------------------------------- - def setup(self, np, size, dtype): - dt = getattr(np, dtype) - # self.a = np.arange(60000.0).reshape(150, 400) - self.a = np.arange(size * size, dtype=dt).reshape((size, size)) - # self.ac = self.a.copy() - # self.at = self.a.T - # self.atc = self.a.T.copy() - # self.b = np.arange(240000.0).reshape(400, 600) - self.b = np.arange(size * size, dtype=dt).reshape((size, size)) - # self.c = np.arange(600) - # self.d = np.arange(400) +class MatMul: + """Products of two square matrices -- dot, matmul, inner and einsum.""" - # self.a3 = np.arange(480000.).reshape(60, 80, 100) - # self.b3 = np.arange(192000.).reshape(80, 60, 40) + params = [_EXECUTOR_NAMES, _ORDERS, _DTYPES] + param_names = ["executor", "order", "dtype"] - def time_dot_a_b(self, np): - np.dot(self.a, self.b) + def setup(self, executor, order, dtype): + self.np = _EXECUTORS[executor] + self.sync = make_synchronizer(executor) + if executor == "dpnp": + skip_unsupported_dtype(default_queue(), dtype) + dt = getattr(self.np, dtype) + self.a = self.np.arange(order * order, dtype=dt).reshape((order, order)) + self.b = self.np.arange(order * order, dtype=dt).reshape((order, order)) - def time_dot_d_dot_b_c(self, np, *args): - np.dot(self.d, np.dot(self.b, self.c)) + def time_dot(self, executor, order, dtype): + self.sync(self.np.dot(self.a, self.b)) - def time_dot_trans_a_at(self, np, *args): - np.dot(self.a, self.at) + def time_matmul(self, executor, order, dtype): + self.sync(self.np.matmul(self.a, self.b)) - def time_dot_trans_a_atc(self, np, *args): - np.dot(self.a, self.atc) + def time_inner(self, executor, order, dtype): + self.sync(self.np.inner(self.a, self.b)) - def time_dot_trans_at_a(self, np, *args): - np.dot(self.at, self.a) - - def time_dot_trans_atc_a(self, np, *args): - np.dot(self.atc, self.a) - - def time_einsum_i_ij_j(self, np, *args): - np.einsum("i,ij,j", self.d, self.b, self.c) - - def time_einsum_ij_jk_a_b(self, np, *args): - np.einsum("ij,jk", self.a, self.b) - - def time_einsum_ijk_jil_kl(self, np, *args): - np.einsum("ijk,jil->kl", self.a3, self.b3) - - def time_inner_trans_a_a(self, np, *args): - np.inner(self.a, self.a) - - def time_inner_trans_a_ac(self, np, *args): - np.inner(self.a, self.ac) - - def time_matmul_a_b(self, np, *args): - np.matmul(self.a, self.b) - - def time_matmul_d_matmul_b_c(self, np, *args): - np.matmul(self.d, np.matmul(self.b, self.c)) - - def time_matmul_trans_a_at(self, np, *args): - np.matmul(self.a, self.at) - - def time_matmul_trans_a_atc(self, np, *args): - np.matmul(self.a, self.atc) - - def time_matmul_trans_at_a(self, np, *args): - np.matmul(self.at, self.a) - - def time_matmul_trans_atc_a(self, np, *args): - np.matmul(self.atc, self.a) - - def time_tensordot_a_b_axes_1_0_0_1(self, np, *args): - np.tensordot(self.a3, self.b3, axes=([1, 0], [0, 1])) - - -class Linalg(Benchmark): - params = [[dpnp, numpy], ["svd", "pinv", "det", "norm"], TYPES1] - param_names = ["executor", "op", "type"] - - def setup(self, np, op, typename): - np.seterr(all="ignore") - - self.func = getattr(np.linalg, op) - - if op == "cholesky": - # we need a positive definite - self.a = np.dot( - get_squares_()[typename], get_squares_()[typename].T - ) - else: - self.a = get_squares_()[typename] - - # check that dtype is supported at all - try: - self.func(self.a[:2, :2]) - except TypeError: - raise NotImplementedError() - - def time_op(self, np, op, typename): - self.func(self.a) - - -class Lstsq(Benchmark): - params = [dpnp, numpy] - param_names = ["executor"] - - def setup(self, np): - self.a = get_squares_()["float64"] - self.b = get_indexes_rand()[:100].astype(np.float64) - - def time_numpy_linalg_lstsq_a__b_float64(self, np): - np.linalg.lstsq(self.a, self.b, rcond=-1) - - -# class Einsum(Benchmark): -# param_names = ['dtype'] -# params = [[np.float64]] -# def setup(self, dtype): -# self.a = np.arange(2900, dtype=dtype) -# self.b = np.arange(3000, dtype=dtype) -# self.c = np.arange(24000, dtype=dtype).reshape(20, 30, 40) -# self.c1 = np.arange(1200, dtype=dtype).reshape(30, 40) -# self.d = np.arange(10000, dtype=dtype).reshape(10,100,10) - -# #outer(a,b): trigger sum_of_products_contig_stride0_outcontig_two -# def time_einsum_outer(self, dtype): -# np.einsum("i,j", self.a, self.b, optimize=True) - -# # multiply(a, b):trigger sum_of_products_contig_two -# def time_einsum_multiply(self, dtype): -# np.einsum("..., ...", self.c1, self.c , optimize=True) - -# # sum and multiply:trigger sum_of_products_contig_stride0_outstride0_two -# def time_einsum_sum_mul(self, dtype): -# np.einsum(",i...->", 300, self.d, optimize=True) - -# # sum and multiply:trigger sum_of_products_stride0_contig_outstride0_two -# def time_einsum_sum_mul2(self, dtype): -# np.einsum("i...,->", self.d, 300, optimize=True) + def time_einsum_ij_jk(self, executor, order, dtype): + self.sync(self.np.einsum("ij,jk", self.a, self.b)) diff --git a/benchmarks/benchmarks/bench_random.py b/benchmarks/benchmarks/bench_random.py index 191569842371..29ac0ee5e82d 100644 --- a/benchmarks/benchmarks/bench_random.py +++ b/benchmarks/benchmarks/bench_random.py @@ -26,30 +26,34 @@ # THE POSSIBILITY OF SUCH DAMAGE. # ***************************************************************************** -import numpy +"""Benchmarks for random sampling, dpnp.random against numpy.random.""" -import dpnp +from ._utils import ( + _EXECUTOR_NAMES, + _EXECUTORS, + _SIZES_1D, + make_synchronizer, +) -from .common import Benchmark +class Sample: + """Random sampling, dpnp against NumPy.""" -# asv run --python=python --quick --bench Sample -class Sample(Benchmark): - executors = {"dpnp": dpnp, "numpy": numpy} - params = [["dpnp", "numpy"], [2**16, 2**20, 2**24]] + params = [_EXECUTOR_NAMES, _SIZES_1D] param_names = ["executor", "size"] def setup(self, executor, size): - self.executor = self.executors[executor] + self.executor = _EXECUTORS[executor] + self.sync = make_synchronizer(executor) def time_rand(self, executor, size): np = self.executor - np.random.rand(size) + self.sync(np.random.rand(size)) def time_randn(self, executor, size): np = self.executor - np.random.randn(size) + self.sync(np.random.randn(size)) def time_random_sample(self, executor, size): np = self.executor - np.random.random_sample((size,)) + self.sync(np.random.random_sample((size,))) diff --git a/benchmarks/benchmarks/common.py b/benchmarks/benchmarks/common.py deleted file mode 100644 index ce0956cec5d6..000000000000 --- a/benchmarks/benchmarks/common.py +++ /dev/null @@ -1,156 +0,0 @@ -# ***************************************************************************** -# Copyright (c) 2020, Intel Corporation -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# - Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# - Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# - Neither the name of the copyright holder nor the names of its contributors -# may be used to endorse or promote products derived from this software -# without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF -# THE POSSIBILITY OF SUCH DAMAGE. -# ***************************************************************************** - -import random - -import numpy - -# Various pre-crafted datasets/variables for testing -# !!! Must not be changed -- only appended !!! -# while testing numpy we better not rely on numpy to produce random -# sequences -random.seed(1) -# but will seed it nevertheless -numpy.random.seed(1) - -nx, ny = 1000, 1000 -# reduced squares based on indexes_rand, primarily for testing more -# time-consuming functions (ufunc, linalg, etc) -nxs, nys = 100, 100 - -# a set of interesting types to test -TYPES1 = [ - "int16", - "float16", - "int32", - "float32", - "int64", - "float64", - "complex64", - "longfloat", - "complex128", -] -if "complex256" in numpy.typeDict: - TYPES1.append("complex256") - - -def memoize(func): - result = [] - - def wrapper(): - if not result: - result.append(func()) - return result[0] - - return wrapper - - -# values which will be used to construct our sample data matrices -# replicate 10 times to speed up initial imports of this helper -# and generate some redundancy - - -@memoize -def get_values(): - rnd = numpy.random.RandomState(1) - values = numpy.tile(rnd.uniform(0, 100, size=nx * ny // 10), 10) - return values - - -@memoize -def get_squares(): - values = get_values() - squares = { - t: numpy.array(values, dtype=getattr(numpy, t)).reshape((nx, ny)) - for t in TYPES1 - } - - # adjust complex ones to have non-degenerated imagery part -- use - # original data transposed for that - for t, v in squares.items(): - if t.startswith("complex"): - v += v.T * 1j - return squares - - -@memoize -def get_squares_(): - # smaller squares - squares_ = {t: s[:nxs, :nys] for t, s in get_squares().items()} - return squares_ - - -@memoize -def get_vectors(): - # vectors - vectors = {t: s[0] for t, s in get_squares().items()} - return vectors - - -@memoize -def get_indexes(): - indexes = list(range(nx)) - # so we do not have all items - indexes.pop(5) - indexes.pop(95) - - indexes = numpy.array(indexes) - return indexes - - -@memoize -def get_indexes_rand(): - rnd = random.Random(1) - - indexes_rand = get_indexes().tolist() # copy - rnd.shuffle(indexes_rand) # in-place shuffle - indexes_rand = numpy.array(indexes_rand) - return indexes_rand - - -@memoize -def get_indexes_(): - # smaller versions - indexes = get_indexes() - indexes_ = indexes[indexes < nxs] - return indexes_ - - -@memoize -def get_indexes_rand_(): - indexes_rand = get_indexes_rand() - indexes_rand_ = indexes_rand[indexes_rand < nxs] - return indexes_rand_ - - -class Benchmark: - # warmup_time = 0 - # number = 3 # test repeats for one setup - # repeat = 1 - # rounds = 1 - pass diff --git a/benchmarks/benchmarks/dpbench/README.md b/benchmarks/benchmarks/dpbench/README.md new file mode 100644 index 000000000000..ca9afef6717f --- /dev/null +++ b/benchmarks/benchmarks/dpbench/README.md @@ -0,0 +1,50 @@ +## dpBench-derived workloads + +The modules under `workloads/` reproduce benchmarks from +[dpBench](https://github.com/IntelPython/dpbench), so that dpnp is measured on +the same quantity: the end-to-end time of a whole workload rather than of a +single API call. dpBench is not a dependency; `_dpbench_runner.py` re-implements +the parts ASV needs (data initialization, host-to-device transfer, execution and +reference validation). + +Reference version: dpBench `0.2.0+79.g4501644`. + +Per workload, three modules from `dpbench/benchmarks/default//` and one +config from `dpbench/configs/bench_info/` map onto one module here: + +| dpnp module | dpBench sources | +| -------------------------------- | ----------------------------------------------------------------------------------- | +| `workloads/black_scholes.py` | `black_scholes_{dpnp,numpy,initialize}.py`, `black_scholes.toml` | +| `workloads/l2_norm.py` | `l2_norm_{dpnp,numpy,initialize}.py`, `l2_norm.toml` | +| `workloads/pairwise_distance.py` | `pairwise_distance_{dpnp,numpy,initialize}.py`, `pairwise_distance.toml` | +| `workloads/rambo.py` | `rambo_{dpnp,numpy,initialize}.py`, `rambo.toml` | +| `workloads/gpairs.py` | `gpairs_{dpnp,numpy,initialize}.py`, `gpairs.toml` | + +`_dpnp.py` became `()`, `_numpy.py` became `reference()` and +`_initialize.py` became `initialize()`. From the TOML, `[benchmark]` gives +`INPUT_ARGS` / `ARRAY_ARGS` / `OUTPUT_ARGS`, `[benchmark.init]` gives +`INIT_INPUT_ARGS` / `INIT_OUTPUT_ARGS` / `PRECISION`, and +`[benchmark.parameters.*]` gives `PRESETS`. + +### Intended differences + +1. `black_scholes` calls `dpnp.scipy.special.erf`, where dpnp now keeps `erf`. +2. Every kernel ends with `dpnp.synchronize_array_data()`. ASV times the + `time_*` method directly, so the kernel has to block or only host-side + dispatch is measured. +3. `rambo.initialize` draws its random block in one `numpy.random.rand` call + rather than element by element. This consumes the same RNG stream in the same + order, so the data is bit-identical, but it is far faster -- which matters + because ASV re-runs `setup` for every round. +4. `peak_elements(params)` is new: it estimates a preset's peak element count so + `_dpbench_runner.preset_fits` can skip presets too large for the device. + +### Adding a workload + +Add a module under `workloads/` exposing the same interface as the existing ones, +translate its `bench_info` TOML into the metadata constants, add a +`peak_elements` estimate, and record it in the table above. + +Then add a benchmark class to `bench_dpbench.py` -- that is what puts the +workload into the suite. `WORKLOADS` in `workloads/__init__.py` is only a +registry; adding to it alone has no effect. diff --git a/benchmarks/benchmarks/dpbench/__init__.py b/benchmarks/benchmarks/dpbench/__init__.py new file mode 100644 index 000000000000..def3c0ad47bb --- /dev/null +++ b/benchmarks/benchmarks/dpbench/__init__.py @@ -0,0 +1,34 @@ +# ***************************************************************************** +# Copyright (c) 2026, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""dpBench-derived ASV benchmarks for dpnp. + +This sub-package vendors a handful of dpnp workloads from dpBench +(https://github.com/IntelPython/dpbench) and exposes them as Airspeed Velocity +benchmarks. See ``benchmarks/README.md`` for details. +""" diff --git a/benchmarks/benchmarks/dpbench/_dpbench_runner.py b/benchmarks/benchmarks/dpbench/_dpbench_runner.py new file mode 100644 index 000000000000..c3dfed4f9e57 --- /dev/null +++ b/benchmarks/benchmarks/dpbench/_dpbench_runner.py @@ -0,0 +1,288 @@ +# ***************************************************************************** +# Copyright (c) 2026, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""Minimal re-implementation of dpBench's benchmark execution model for ASV. + +dpBench (https://github.com/IntelPython/dpbench) drives its benchmarks through +a fairly heavy runner that spawns a sub-process per framework, resolves TOML +configuration, validates results against a reference and persists timings to a +database. None of that machinery is importable in a lightweight ASV +environment (it pulls in ``numba_dpex``, ``sqlalchemy``, ``alembic`` and more), +so this module re-implements just the parts that matter for benchmarking: + +* data initialization -- the host (NumPy) input data is produced exactly the + way dpBench produces it, using each workload's ``initialize`` function and a + precision-driven ``types_dict`` (see ``dpbench.infrastructure.benchmark``); +* host-to-device transfer -- array arguments are copied to the device with the + same ``dpnp.asarray`` logic dpBench's ``DpnpFramework.copy_to_func`` uses; +* execution -- the dpnp implementation is invoked and blocks on device + completion (each vendored kernel ends with ``dpnp.synchronize_array_data``), + matching how dpBench itself times the workload; +* validation -- the dpnp results are compared against the workload's NumPy + reference implementation, mirroring + ``dpbench.infrastructure.benchmark_validation``. +""" + +import numpy + +import dpnp + +# Precision -> dtype mapping, copied from dpBench's +# ``dpbench/configs/precision_dtypes.toml``. +PRECISION_DTYPES = { + "int": {"single": "i4", "double": "i8"}, + "float": {"single": "f4", "double": "f8"}, +} + +# Precisions ASV benchmarks each workload at. dpBench's configs request +# ``double`` throughout, but not every device supports fp64 (many iGPUs do +# not), so ``single`` is benchmarked as well and the unsupported one is skipped +# per device -- that way an fp64-less device still produces results instead of +# reporting nothing. +PRECISIONS = ["single", "double"] + +# Fraction of the device's global memory a benchmark's estimated peak +# footprint is allowed to occupy. Kept well below 1.0 because the estimates +# below only count the obvious buffers, the device is usually shared with a +# display server, and dpnp's own allocator caches freed blocks. +_MEMORY_BUDGET_FRACTION = 0.25 + + +def build_types_dict(precision): + """Build the ``types_dict`` passed to a workload's ``initialize``. + + Mirrors ``Benchmark._get_types_dict`` in dpBench. + """ + return { + kind: numpy.dtype(precision_strings[precision]) + for kind, precision_strings in PRECISION_DTYPES.items() + } + + +def float_dtype(precision): + """Return the floating-point dtype used at ``precision``.""" + return build_types_dict(precision)["float"] + + +def preset_fits(workload, preset, device, precision="double"): + """Whether ``preset``'s estimated peak footprint fits ``device``'s memory. + + ``precision`` is the *widest* precision benchmarked, so the verdict is the + same for every precision parameter. + """ + # The cheapest preset always runs, so that an undersized device fails loudly + # on allocation rather than reporting nothing. + if preset == presets_by_size(workload)[0]: + return True + + itemsize = float_dtype(precision).itemsize + budget = _MEMORY_BUDGET_FRACTION * device.global_mem_size + peak = workload.peak_elements(workload.PRESETS[preset]) + return peak * itemsize <= budget + + +def presets_by_size(workload): + """Return the workload's preset names ordered cheapest-first. + + dpBench's preset names are not ordered by size (``M16Gb`` is smaller than + ``M``), so sort explicitly rather than relying on the declaration order. + """ + return sorted( + workload.PRESETS, + key=lambda name: workload.peak_elements(workload.PRESETS[name]), + ) + + +def initialize_host_data(workload, preset, precision): + """Produce the host (NumPy) input data for ``workload`` at ``preset``. + + Mirrors ``Benchmark.initialize_input_data`` / + ``_initialize_input_data_from_init`` in dpBench. + """ + if preset not in workload.PRESETS: + raise NotImplementedError( + f"{workload.NAME} doesn't have a {preset} preset." + ) + + # Preset parameters (scalars such as ``nopt``, ``seed``, ``nbins``, ...). + data = dict(workload.PRESETS[preset]) + + # The precision-driven types dictionary, if the workload's ``initialize`` + # consumes one. + if "types_dict" in workload.INIT_INPUT_ARGS: + data["types_dict"] = build_types_dict(precision) + + # Call ``initialize`` and store its outputs under the configured names. + init_kwargs = {arg: data[arg] for arg in workload.INIT_INPUT_ARGS} + initialized = workload.initialize(**init_kwargs) + + if isinstance(initialized, tuple): + for name, value in zip(workload.INIT_OUTPUT_ARGS, initialized): + data[name] = value + elif len(workload.INIT_OUTPUT_ARGS) == 1: + data[workload.INIT_OUTPUT_ARGS[0]] = initialized + else: + raise ValueError("Unsupported initialize output") + + return data + + +def _copy_to_device(ref_array): + """Copy a host array to the (default) device. + + Mirrors ``DpnpFramework.copy_to_func`` in dpBench. + """ + if ref_array.flags["C_CONTIGUOUS"]: + order = "C" + elif ref_array.flags["F_CONTIGUOUS"]: + order = "F" + else: + order = "K" + return dpnp.asarray( + ref_array, + dtype=ref_array.dtype, + order=order, + ) + + +def set_input_args(workload, host_data): + """Build the kernel keyword arguments, copying array args to the device. + + Mirrors ``_set_input_args`` in dpBench. + """ + inputs = {} + for arg in workload.INPUT_ARGS: + if arg in workload.ARRAY_ARGS: + inputs[arg] = _copy_to_device(host_data[arg]) + else: + inputs[arg] = host_data[arg] + return inputs + + +def relative_error(ref, val): + """Relative error between a reference and a measured array. + + Copied from ``dpbench.infrastructure.benchmark_validation``. + """ + ref_norm = numpy.linalg.norm(ref) + if ref_norm == 0: + val_norm = numpy.linalg.norm(val) + if val_norm == 0: + return 0.0 + ref_norm = val_norm + + return numpy.linalg.norm(ref - val) / ref_norm + + +def validate(expected, actual, rel_error=1e-05): + """Check that ``actual`` matches ``expected`` closely enough. + + Mirrors ``dpbench.infrastructure.benchmark_validation.validate``: a + mismatch is tolerated only while the relative error stays below + ``rel_error``. Raises :exc:`ValueError` naming the offending argument + instead of returning a bool, so a wrong result fails the benchmark rather + than being silently timed. + """ + for name, ref in expected.items(): + val = actual[name] + if numpy.allclose(ref, val): + continue + error = relative_error(ref, val) + if error >= rel_error: + raise ValueError( + f"Validation failed for {name!r}: relative error {error:.3e} " + f"exceeds the {rel_error:.0e} tolerance." + ) + + +class WorkloadRunner: + """Sets up and runs a single dpBench workload for one preset. + + Each vendored kernel ends with ``dpnp.synchronize_array_data`` on its + output, so a single :meth:`run` call blocks until the device work has + completed. ASV wall-clock-times the ``time_*`` method that calls + :meth:`run`, and thus captures the end-to-end (host dispatch + device) + execution time of the workload -- the same quantity dpBench measures. + """ + + def __init__(self, workload, preset, precision="double"): + self.workload = workload + self.preset = preset + self.precision = precision + + self.fn = getattr(workload, workload.NAME) + self.kwargs = None + + def setup(self): + """Initialize host data, transfer it to the device and warm up.""" + # The host data is deliberately not retained: once the array arguments + # have been copied to the device it would just pin a second, host-side + # copy of the whole problem (several GiB at the larger presets). + host_data = initialize_host_data( + self.workload, self.preset, self.precision + ) + inputs = set_input_args(self.workload, host_data) + self.kwargs = {arg: inputs[arg] for arg in self.workload.INPUT_ARGS} + + # Warmup (equivalent to dpBench's warmup step in ``_exec``). + self.run() + + def run(self): + """Execute the kernel once, blocking on device completion.""" + self.fn(**self.kwargs) + + def validate(self): + """Compare the dpnp results against the NumPy reference. + + Runs the workload's ``reference`` implementation on a fresh copy of the + same host data and compares every ``OUTPUT_ARGS`` entry, mirroring + dpBench's post-run validation step. Called from the benchmark's + ``setup``, so a numerically wrong kernel fails the benchmark instead of + being timed. + """ + expected = { + arg: value + for arg, value in self._reference_outputs().items() + if arg in self.workload.OUTPUT_ARGS + } + actual = { + arg: dpnp.asnumpy(self.kwargs[arg]) + for arg in self.workload.OUTPUT_ARGS + } + validate(expected, actual) + + def _reference_outputs(self): + """Run the NumPy reference on freshly initialized host data.""" + # A fresh initialization is required: the kernel writes into its output + # arrays, so ``self._host_data`` no longer holds their initial values. + host_data = initialize_host_data( + self.workload, self.preset, self.precision + ) + kwargs = {arg: host_data[arg] for arg in self.workload.INPUT_ARGS} + self.workload.reference(**kwargs) + return kwargs diff --git a/benchmarks/benchmarks/dpbench/workloads/__init__.py b/benchmarks/benchmarks/dpbench/workloads/__init__.py new file mode 100644 index 000000000000..f6823188e294 --- /dev/null +++ b/benchmarks/benchmarks/dpbench/workloads/__init__.py @@ -0,0 +1,63 @@ +# ***************************************************************************** +# Copyright (c) 2026, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""dpnp workloads vendored from dpBench. + +Each module exposes a uniform interface consumed by ``_dpbench_runner``: + +* ``NAME`` -- workload name; also the name of the kernel function; +* ``PRECISION`` -- the precision dpBench's config requests for this workload; +* ``INPUT_ARGS`` / ``ARRAY_ARGS`` / ``OUTPUT_ARGS`` -- kernel argument metadata; +* ``INIT_INPUT_ARGS`` / ``INIT_OUTPUT_ARGS`` -- ``initialize`` argument metadata; +* ``PRESETS`` -- all dpBench data-size presets (S, M16Gb, M, L); +* ``peak_elements(params)`` -- estimated peak device element count for a preset, + used to pick the presets that fit into the device's memory; +* ``initialize(...)`` -- host data generator; +* ``(...)`` -- the dpnp kernel; +* ``reference(...)`` -- the NumPy kernel the dpnp results are validated against. +""" + +from . import black_scholes, gpairs, l2_norm, pairwise_distance, rambo + +# All vendored workloads, in a stable order. +WORKLOADS = [ + black_scholes, + l2_norm, + pairwise_distance, + rambo, + gpairs, +] + +__all__ = [ + "WORKLOADS", + "black_scholes", + "l2_norm", + "pairwise_distance", + "rambo", + "gpairs", +] diff --git a/benchmarks/benchmarks/dpbench/workloads/black_scholes.py b/benchmarks/benchmarks/dpbench/workloads/black_scholes.py new file mode 100644 index 000000000000..dd4aa6d6de2f --- /dev/null +++ b/benchmarks/benchmarks/dpbench/workloads/black_scholes.py @@ -0,0 +1,176 @@ +# ***************************************************************************** +# Copyright (c) 2026, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""Black-Scholes formula workload. + +The dpnp implementation, the NumPy reference and the data initialization are +copied verbatim from dpBench (https://github.com/IntelPython/dpbench), and the +metadata below mirrors ``dpbench/configs/bench_info/black_scholes.toml``. +""" + +import dpnp as np + +# --- dpBench benchmark metadata (see black_scholes.toml) -------------------- + +NAME = "black_scholes" +# Precision requested by the dpBench config. ASV benchmarks every precision in +# ``_dpbench_runner.PRECISIONS`` that the device supports, so this is only the +# documented dpBench default. +PRECISION = "double" + +# Arguments passed to the kernel, in order. +INPUT_ARGS = [ + "nopt", + "price", + "strike", + "t", + "rate", + "volatility", + "call", + "put", +] +# Arguments that are arrays and therefore copied to the device. +ARRAY_ARGS = ["price", "strike", "t", "call", "put"] +# Arguments that the kernel writes into. +OUTPUT_ARGS = ["call", "put"] + +# Arguments passed to ``initialize`` and the values it returns, in order. +INIT_INPUT_ARGS = ["nopt", "seed", "types_dict"] +INIT_OUTPUT_ARGS = [ + "price", + "strike", + "t", + "rate", + "volatility", + "call", + "put", +] + +# Data-size presets, copied verbatim from dpBench. +PRESETS = { + "S": {"nopt": 524288, "seed": 777777}, + "M16Gb": {"nopt": 67108864, "seed": 777777}, + "M": {"nopt": 134217728, "seed": 777777}, + "L": {"nopt": 268435456, "seed": 777777}, +} + + +def peak_elements(params): + """Estimated peak number of float elements held on the device. + + 5 input/output arrays of ``nopt`` elements, plus the ~8 temporaries the + kernel below materializes (``a``, ``b``, ``z``, ``c``, ``y``, ``w1``, + ``w2``, ``Se``, ...). + """ + return 13 * params["nopt"] + + +def initialize(nopt, seed, types_dict): + import numpy + import numpy.random as default_rng + + dtype: numpy.dtype = types_dict["float"] + S0L = dtype.type(10.0) + S0H = dtype.type(50.0) + XL = dtype.type(10.0) + XH = dtype.type(50.0) + TL = dtype.type(1.0) + TH = dtype.type(2.0) + RISK_FREE = dtype.type(0.1) + VOLATILITY = dtype.type(0.2) + + default_rng.seed(seed) + price = default_rng.uniform(S0L, S0H, nopt).astype(dtype) + strike = default_rng.uniform(XL, XH, nopt).astype(dtype) + t = default_rng.uniform(TL, TH, nopt).astype(dtype) + rate = RISK_FREE + volatility = VOLATILITY + call = numpy.zeros(nopt, dtype=dtype) + put = -numpy.ones(nopt, dtype=dtype) + + return (price, strike, t, rate, volatility, call, put) + + +def black_scholes(nopt, price, strike, t, rate, volatility, call, put): + mr = -rate + sig_sig_two = volatility * volatility * 2 + + P = price + S = strike + T = t + + a = np.log(P / S) + b = T * mr + + z = T * sig_sig_two + c = 0.25 * z + y = np.true_divide(1.0, np.sqrt(z)) + + w1 = (a - b + c) * y + w2 = (a - b - c) * y + + d1 = 0.5 + 0.5 * np.scipy.special.erf(w1) + d2 = 0.5 + 0.5 * np.scipy.special.erf(w2) + + Se = np.exp(b) * S + + call[:] = P * d1 - Se * d2 + put[:] = call - P + Se + + np.synchronize_array_data(put) + + +def reference(nopt, price, strike, t, rate, volatility, call, put): + """NumPy reference, copied from dpBench's ``black_scholes_numpy.py``.""" + import numpy + from scipy.special import erf + + mr = -rate + sig_sig_two = volatility * volatility * 2 + + P = price + S = strike + T = t + + a = numpy.log(P / S) + b = T * mr + + z = T * sig_sig_two + c = 0.25 * z + y = numpy.true_divide(1.0, numpy.sqrt(z)) + + w1 = (a - b + c) * y + w2 = (a - b - c) * y + + d1 = 0.5 + 0.5 * erf(w1) + d2 = 0.5 + 0.5 * erf(w2) + + Se = numpy.exp(b) * S + + call[:] = P * d1 - Se * d2 + put[:] = call - P + Se diff --git a/benchmarks/benchmarks/dpbench/workloads/gpairs.py b/benchmarks/benchmarks/dpbench/workloads/gpairs.py new file mode 100644 index 000000000000..27509f85739f --- /dev/null +++ b/benchmarks/benchmarks/dpbench/workloads/gpairs.py @@ -0,0 +1,183 @@ +# ***************************************************************************** +# Copyright (c) 2026, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""GPairs (galaxy pair counting) workload. + +The dpnp implementation, the NumPy reference and the data initialization are +copied verbatim from dpBench (https://github.com/IntelPython/dpbench), and the +metadata below mirrors ``dpbench/configs/bench_info/gpairs.toml``. +""" + +import numpy + +import dpnp as np + +# --- dpBench benchmark metadata (see gpairs.toml) --------------------------- + +NAME = "gpairs" +# See the note on ``PRECISION`` in ``black_scholes.py``. +PRECISION = "double" + +INPUT_ARGS = [ + "nopt", + "nbins", + "x1", + "y1", + "z1", + "w1", + "x2", + "y2", + "z2", + "w2", + "rbins", + "results", +] +ARRAY_ARGS = [ + "x1", + "y1", + "z1", + "w1", + "x2", + "y2", + "z2", + "w2", + "rbins", + "results", +] +OUTPUT_ARGS = ["results"] + +INIT_INPUT_ARGS = ["nopt", "seed", "nbins", "rmax", "rmin", "types_dict"] +INIT_OUTPUT_ARGS = [ + "x1", + "y1", + "z1", + "w1", + "x2", + "y2", + "z2", + "w2", + "rbins", + "results", +] + +PRESETS = { + "S": {"nopt": 128, "seed": 1234, "nbins": 20, "rmax": 50, "rmin": 0.1}, + "M16Gb": { + "nopt": 4096, + "seed": 1234, + "nbins": 20, + "rmax": 50, + "rmin": 0.1, + }, + "M": {"nopt": 8192, "seed": 1234, "nbins": 20, "rmax": 50, "rmin": 0.1}, + "L": { + "nopt": 524288, + "seed": 1234, + "nbins": 20, + "rmax": 50, + "rmin": 0.1, + }, +} + + +def peak_elements(params): + """Estimated peak number of float elements held on the device. + + Dominated by the ``(nopt, nopt)`` distance matrix ``dm``; the kernel also + materializes a same-shaped ``outer(w1, w2)`` and a boolean mask of the same + extent once per bin, hence the factor of 3. + """ + nopt = params["nopt"] + return 3 * nopt * nopt + 8 * nopt + + +def _generate_rbins(dtype, nbins, rmax, rmin): + rbins = numpy.logspace(numpy.log10(rmin), numpy.log10(rmax), nbins).astype( + dtype + ) + + return (rbins**2).astype(dtype) + + +def initialize(nopt, seed, nbins, rmax, rmin, types_dict): + import numpy.random as default_rng + + default_rng.seed(seed) + dtype = types_dict["float"] + x1 = numpy.random.randn(nopt).astype(dtype) + y1 = numpy.random.randn(nopt).astype(dtype) + z1 = numpy.random.randn(nopt).astype(dtype) + w1 = numpy.random.rand(nopt).astype(dtype) + w1 = w1 / numpy.sum(w1) + + x2 = numpy.random.randn(nopt).astype(dtype) + y2 = numpy.random.randn(nopt).astype(dtype) + z2 = numpy.random.randn(nopt).astype(dtype) + w2 = numpy.random.rand(nopt).astype(dtype) + w2 = w2 / numpy.sum(w2) + + rbins = _generate_rbins(dtype=dtype, rmin=rmin, rmax=rmax, nbins=nbins) + results = numpy.zeros_like(rbins).astype(dtype) + return (x1, y1, z1, w1, x2, y2, z2, w2, rbins, results) + + +def _gpairs_impl(x1, y1, z1, w1, x2, y2, z2, w2, rbins): + dm = ( + np.square(x2 - x1[:, None]) + + np.square(y2 - y1[:, None]) + + np.square(z2 - z1[:, None]) + ) + return np.array( + [ + np.outer(w1, w2)[dm <= rbins[k]].sum(dtype=np.result_type(w1, w2)) + for k in range(len(rbins)) + ], + device=x1.device, + ) + + +def gpairs(nopt, nbins, x1, y1, z1, w1, x2, y2, z2, w2, rbins, results): + results[:] = _gpairs_impl(x1, y1, z1, w1, x2, y2, z2, w2, rbins) + + np.synchronize_array_data(results) + + +def _gpairs_reference_impl(x1, y1, z1, w1, x2, y2, z2, w2, rbins): + dm = ( + numpy.square(x2 - x1[:, None]) + + numpy.square(y2 - y1[:, None]) + + numpy.square(z2 - z1[:, None]) + ) + return numpy.array( + [numpy.outer(w1, w2)[dm <= rbins[k]].sum() for k in range(len(rbins))] + ) + + +def reference(nopt, nbins, x1, y1, z1, w1, x2, y2, z2, w2, rbins, results): + """NumPy reference, copied from dpBench's ``gpairs_numpy.py``.""" + results[:] = _gpairs_reference_impl(x1, y1, z1, w1, x2, y2, z2, w2, rbins) diff --git a/benchmarks/benchmarks/dpbench/workloads/l2_norm.py b/benchmarks/benchmarks/dpbench/workloads/l2_norm.py new file mode 100644 index 000000000000..077c87e55d6c --- /dev/null +++ b/benchmarks/benchmarks/dpbench/workloads/l2_norm.py @@ -0,0 +1,96 @@ +# ***************************************************************************** +# Copyright (c) 2026, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""L2-norm workload. + +The dpnp implementation, the NumPy reference and the data initialization are +copied verbatim from dpBench (https://github.com/IntelPython/dpbench), and the +metadata below mirrors ``dpbench/configs/bench_info/l2_norm.toml``. +""" + +import dpnp as np + +# --- dpBench benchmark metadata (see l2_norm.toml) -------------------------- + +NAME = "l2_norm" +# See the note on ``PRECISION`` in ``black_scholes.py``. +PRECISION = "double" + +INPUT_ARGS = ["a", "d"] +ARRAY_ARGS = ["a", "d"] +OUTPUT_ARGS = ["d"] + +INIT_INPUT_ARGS = ["npoints", "dims", "seed", "types_dict"] +INIT_OUTPUT_ARGS = ["a", "d"] + +PRESETS = { + "S": {"npoints": 32768, "dims": 3, "seed": 777777}, + "M16Gb": {"npoints": 134217728, "dims": 3, "seed": 777777}, + "M": {"npoints": 268435456, "dims": 3, "seed": 777777}, + "L": {"npoints": 536870912, "dims": 3, "seed": 777777}, +} + + +def peak_elements(params): + """Estimated peak number of float elements held on the device. + + The ``(npoints, dims)`` input plus the same-shaped ``sq`` temporary, and + two ``npoints``-sized vectors (``d`` and the ``sum`` reduction). + """ + return 2 * params["npoints"] * params["dims"] + 2 * params["npoints"] + + +def initialize(npoints, dims, seed, types_dict): + import numpy + import numpy.random as default_rng + + dtype = types_dict["float"] + + default_rng.seed(seed) + + return ( + default_rng.random((npoints, dims)).astype(dtype), + numpy.zeros(npoints).astype(dtype), + ) + + +def l2_norm(a, d): + sq = np.square(a) + sum = sq.sum(axis=1, dtype=sq.dtype) + d[:] = np.sqrt(sum) + + np.synchronize_array_data(d) + + +def reference(a, d): + """NumPy reference, copied from dpBench's ``l2_norm_numpy.py``.""" + import numpy + + sq = numpy.square(a) + sum = sq.sum(axis=1) + d[:] = numpy.sqrt(sum) diff --git a/benchmarks/benchmarks/dpbench/workloads/pairwise_distance.py b/benchmarks/benchmarks/dpbench/workloads/pairwise_distance.py new file mode 100644 index 000000000000..4067764bd0ed --- /dev/null +++ b/benchmarks/benchmarks/dpbench/workloads/pairwise_distance.py @@ -0,0 +1,108 @@ +# ***************************************************************************** +# Copyright (c) 2026, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""Pairwise-distance workload. + +The dpnp implementation, the NumPy reference and the data initialization are +copied verbatim from dpBench (https://github.com/IntelPython/dpbench), and the +metadata below mirrors ``dpbench/configs/bench_info/pairwise_distance.toml``. +""" + +import dpnp as np + +# --- dpBench benchmark metadata (see pairwise_distance.toml) ---------------- + +NAME = "pairwise_distance" +# See the note on ``PRECISION`` in ``black_scholes.py``. +PRECISION = "double" + +INPUT_ARGS = ["X1", "X2", "D"] +ARRAY_ARGS = ["X1", "X2", "D"] +OUTPUT_ARGS = ["D"] + +INIT_INPUT_ARGS = ["npoints", "dims", "seed", "types_dict"] +INIT_OUTPUT_ARGS = ["X1", "X2", "D"] + +PRESETS = { + "S": {"npoints": 1024, "dims": 3, "seed": 7777777}, + "M16Gb": {"npoints": 21846, "dims": 3, "seed": 7777777}, + "M": {"npoints": 32768, "dims": 3, "seed": 7777777}, + "L": {"npoints": 44032, "dims": 3, "seed": 7777777}, +} + + +def peak_elements(params): + """Estimated peak number of float elements held on the device. + + Dominated by the ``(npoints, npoints)`` distance matrix ``D``; the two + ``(npoints, dims)`` inputs are negligible in comparison but counted anyway. + """ + npoints = params["npoints"] + return npoints * npoints + 2 * npoints * params["dims"] + + +def initialize(npoints, dims, seed, types_dict): + import numpy + import numpy.random as default_rng + + dtype = types_dict["float"] + + default_rng.seed(seed) + + return ( + default_rng.random((npoints, dims)).astype(dtype), + default_rng.random((npoints, dims)).astype(dtype), + numpy.empty((npoints, npoints), dtype), + ) + + +def pairwise_distance(X1, X2, D): + x1 = np.sum(np.square(X1), axis=1, dtype=X1.dtype) + x2 = np.sum(np.square(X2), axis=1, dtype=X2.dtype) + np.dot(X1, X2.T, D) + D *= -2 + x3 = x1.reshape(x1.size, 1) + np.add(D, x3, D) + np.add(D, x2, D) + np.sqrt(D, D) + + np.synchronize_array_data(D) + + +def reference(X1, X2, D): + """NumPy reference, copied from dpBench's ``pairwise_distance_numpy.py``.""" + import numpy + + x1 = numpy.sum(numpy.square(X1), axis=1) + x2 = numpy.sum(numpy.square(X2), axis=1) + numpy.dot(X1, X2.T, D) + D *= -2 + x3 = x1.reshape(x1.size, 1) + numpy.add(D, x3, D) + numpy.add(D, x2, D) + numpy.sqrt(D, D) diff --git a/benchmarks/benchmarks/dpbench/workloads/rambo.py b/benchmarks/benchmarks/dpbench/workloads/rambo.py new file mode 100644 index 000000000000..87572c51bfad --- /dev/null +++ b/benchmarks/benchmarks/dpbench/workloads/rambo.py @@ -0,0 +1,114 @@ +# ***************************************************************************** +# Copyright (c) 2026, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""Rambo workload. + +The dpnp implementation, the NumPy reference and the data initialization are +copied from dpBench (https://github.com/IntelPython/dpbench), and the metadata +below mirrors ``dpbench/configs/bench_info/rambo.toml``. +""" + +import dpnp as np + +# --- dpBench benchmark metadata (see rambo.toml) ---------------------------- + +NAME = "rambo" +# See the note on ``PRECISION`` in ``black_scholes.py``. +PRECISION = "double" + +INPUT_ARGS = ["nevts", "nout", "C1", "F1", "Q1", "output"] +ARRAY_ARGS = ["C1", "F1", "Q1", "output"] +OUTPUT_ARGS = ["output"] + +INIT_INPUT_ARGS = ["nevts", "nout", "types_dict"] +INIT_OUTPUT_ARGS = ["C1", "F1", "Q1", "output"] + +PRESETS = { + "S": {"nevts": 32768, "nout": 4}, + "M16Gb": {"nevts": 16777216, "nout": 4}, + "M": {"nevts": 8388608, "nout": 4}, + "L": {"nevts": 16777216, "nout": 4}, +} + + +def peak_elements(params): + """Estimated peak number of float elements held on the device. + + The ``(nevts, nout, 4)`` output, the three ``(nevts, nout)`` inputs and the + ~6 same-shaped temporaries the kernel materializes (``C``, ``S``, ``F``, + ``Q``, and the ``sin``/``cos`` results). + """ + return 13 * params["nevts"] * params["nout"] + + +def initialize(nevts, nout, types_dict): + import numpy + + dtype = types_dict["float"] + + # dpBench draws these element-by-element in a Python loop; drawing the + # whole block at once consumes the same RNG stream in the same order (so + # the data is bit-identical) but is orders of magnitude faster, which + # matters because ASV re-runs ``setup`` for every benchmark round. + numpy.random.seed(777) + draws = numpy.random.rand(nevts, nout, 4) + + C1 = draws[..., 0].astype(dtype) + F1 = draws[..., 1].astype(dtype) + Q1 = (draws[..., 2] * draws[..., 3]).astype(dtype) + + return (C1, F1, Q1, numpy.empty((nevts, nout, 4), dtype)) + + +def rambo(nevts, nout, C1, F1, Q1, output): + C = 2.0 * C1 - 1.0 + S = np.sqrt(1 - np.square(C)) + F = 2.0 * np.pi * F1 + Q = -np.log(Q1) + + output[:, :, 0] = Q + output[:, :, 1] = Q * S * np.sin(F) + output[:, :, 2] = Q * S * np.cos(F) + output[:, :, 3] = Q * C + + np.synchronize_array_data(output) + + +def reference(nevts, nout, C1, F1, Q1, output): + """NumPy reference, copied from dpBench's ``rambo_numpy.py``.""" + import numpy + + C = 2.0 * C1 - 1.0 + S = numpy.sqrt(1 - numpy.square(C)) + F = 2.0 * numpy.pi * F1 + Q = -numpy.log(Q1) + + output[:, :, 0] = Q + output[:, :, 1] = Q * S * numpy.sin(F) + output[:, :, 2] = Q * S * numpy.cos(F) + output[:, :, 3] = Q * C diff --git a/benchmarks/pytest_benchmark/README.md b/benchmarks/pytest_benchmark/README.md deleted file mode 100644 index 77015a089ef9..000000000000 --- a/benchmarks/pytest_benchmark/README.md +++ /dev/null @@ -1,38 +0,0 @@ -# dpnp/benchmarks/pytest_benchmark/ - -## Prerequisites -* pytest >= 6.1.1 -* pytest-benchmark >= 3.4.1 - - -## Running benchmark tests -```bash -pytest benchmarks/ --benchmark-json=results.json -``` -Running tests and saving the current run into `STORAGE`, see [1] -```bash -pytest benchmarks/ --benchmark-json=results.json --benchmark-autosave -``` - -## Creating `.csv` report -```bash -pytest-benchmark compare results.json --csv=results.csv --group-by='name' -``` - -## Optional: creating histogram -Note: make sure that `pytest-benchmark[histogram]` installed -```bash -# example -pip install pytest-benchmark[histogram] -pytest -vv benchmarks/ --benchmark-autosave --benchmark-histogram -pytest-benchmark compare .benchmarks/Linux-CPython-3.7-64bit/* --histogram -``` - -## Advanced running example -``` -pytest benchmarks/ --benchmark-columns='min, max, mean, stddev, median, rounds, iterations' --benchmark-json=results.json --benchmark-autosave -pytest-benchmark compare results.json --csv=results.csv --group-by='name' -``` - - -[1] https://pytest-benchmark.readthedocs.io/en/latest/usage.html diff --git a/benchmarks/pytest_benchmark/test_random.py b/benchmarks/pytest_benchmark/test_random.py deleted file mode 100644 index 5c91894b2480..000000000000 --- a/benchmarks/pytest_benchmark/test_random.py +++ /dev/null @@ -1,119 +0,0 @@ -# cython: language_level=3 -# ***************************************************************************** -# Copyright (c) 2016, Intel Corporation -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# - Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# - Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# - Neither the name of the copyright holder nor the names of its contributors -# may be used to endorse or promote products derived from this software -# without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF -# THE POSSIBILITY OF SUCH DAMAGE. -# ***************************************************************************** - -import numpy as np -import pytest - -import dpnp - -ROUNDS = 30 -ITERATIONS = 4 - -NNUMBERS = 2**26 - - -@pytest.mark.parametrize( - "function", [dpnp.random.beta, np.random.beta], ids=["dpnp", "numpy"] -) -def test_beta(benchmark, function): - result = benchmark.pedantic( - target=function, - args=( - 4.0, - 5.0, - NNUMBERS, - ), - rounds=ROUNDS, - iterations=ITERATIONS, - ) - - -@pytest.mark.parametrize( - "function", - [dpnp.random.exponential, np.random.exponential], - ids=["dpnp", "numpy"], -) -def test_exponential(benchmark, function): - result = benchmark.pedantic( - target=function, - args=( - 4.0, - NNUMBERS, - ), - rounds=ROUNDS, - iterations=ITERATIONS, - ) - - -@pytest.mark.parametrize( - "function", [dpnp.random.gamma, np.random.gamma], ids=["dpnp", "numpy"] -) -def test_gamma(benchmark, function): - result = benchmark.pedantic( - target=function, - args=( - 2.0, - 4.0, - NNUMBERS, - ), - rounds=ROUNDS, - iterations=ITERATIONS, - ) - - -@pytest.mark.parametrize( - "function", [dpnp.random.normal, np.random.normal], ids=["dpnp", "numpy"] -) -def test_normal(benchmark, function): - result = benchmark.pedantic( - target=function, - args=( - 0.0, - 1.0, - NNUMBERS, - ), - rounds=ROUNDS, - iterations=ITERATIONS, - ) - - -@pytest.mark.parametrize( - "function", [dpnp.random.uniform, np.random.uniform], ids=["dpnp", "numpy"] -) -def test_uniform(benchmark, function): - result = benchmark.pedantic( - target=function, - args=( - 0.0, - 1.0, - NNUMBERS, - ), - rounds=ROUNDS, - iterations=ITERATIONS, - ) diff --git a/pyproject.toml b/pyproject.toml index 2e9662978a3d..0cf3e9721872 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,6 +75,7 @@ readme = {file = "README.md", content-type = "text/markdown"} requires-python = ">=3.10,<3.15" [project.optional-dependencies] +benchmark = ["asv>=0.6", "scipy"] coverage = [ "coverage", "Cython",