diff --git a/CMakeLists.txt b/CMakeLists.txt index 62377bef9..fcd429786 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -135,6 +135,17 @@ install( DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" FILES_MATCHING PATTERN "*.h" ) + +# Install the generated dispatch-surface header over the committed one. They are +# identical for the default declaration; if the surface was overridden, the +# installed headers must describe what the library was actually built with. This +# must be declared after the directory install above so that it wins. +if(DEFINED SVS_GENERATED_DISPATCH_HEADER) + install( + FILES "${SVS_GENERATED_DISPATCH_HEADER}" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/svs/core/distance" + ) +endif() install( EXPORT svs-targets NAMESPACE "svs::" diff --git a/cmake/AGENTS.md b/cmake/AGENTS.md index f6f81aecb..8b6da30b9 100644 --- a/cmake/AGENTS.md +++ b/cmake/AGENTS.md @@ -10,6 +10,7 @@ Build modules, dependency wiring, and feature toggles. ## Intel-specific modules - **`cmake/mkl.cmake`:** MKL linkage (static vs dynamic threading). Do not hardcode MKL versions. When changing linkage mode, validate threading behavior in tests. - **`cmake/multi-arch.cmake`:** AVX-512 / SIMD ISA dispatch. Do not hardcode `-march` or ISA flags outside this file. Changes must align with `include/svs/multi-arch/` runtime dispatch code. +- **`cmake/dispatch-surface.cmake`:** The declared x86 dispatch surface — the fixed extents and the ISA levels. Edit it, never the generated `include/svs/core/distance/dispatch_surface.h`, which every configure overwrites. `cmake/dispatch-checks/` holds the ctest checkers that hold the built binary to this declaration. - **`cmake/numa.cmake`:** NUMA-aware memory allocation. Respect NUMA topology assumptions in performance-critical code. - **`cmake/openmp.cmake`:** Threading model. Do not assume specific OpenMP version or runtime without checking source-of-truth. diff --git a/cmake/dispatch-checks/README.md b/cmake/dispatch-checks/README.md new file mode 100644 index 000000000..331e913f6 --- /dev/null +++ b/cmake/dispatch-checks/README.md @@ -0,0 +1,66 @@ + + +# Dispatch surface checks + +These scripts are run by ctest to hold the built binary to the surface +declared in `cmake/dispatch-surface.cmake`. They are not part of building +the library. See `cmake/dispatch-surface.cmake` for documentation on the +declaration format, the current extent list and ISA levels, and how to add +a new level. + +These tests are only added to the build when the x86 object libraries +exist. Run `ctest` from `/tests`, not from the build root. + +## check-dispatch-linkage.cmake + +**Test:** `dispatch_surface_linkage` + +Verifies that every kernel declared in the surface is defined in the +archive, that the archive defines no undeclared kernels, and that a test +probe naming every kernel defines none of its own. Failure means a +declared kernel is missing or an extra kernel was built but not declared. + +## check-dispatch-declaration.cmake + +**Test:** `dispatch_surface_declaration` + +Verifies that the symbol count derived from the surface declaration matches +the symbol count in the archive and in a consumer object that names every +entry point. Failure means the declaration's extent list, ISA levels, type +pairs, or distance enumerator order disagrees with the built kernels. This +catches generator bugs that would pass linkage checks by being wrong +consistently in both the archive and the probe. + +## check-dispatch-instructions.cmake + +**Test:** `dispatch_instructions_` (one test per ISA level) + +Verifies that each level's object file stays within its instruction budget, +emitting only instructions the level's runtime predicate guarantees the host +supports. Failure means an object file contains instructions not guaranteed +by its level's predicate, causing illegal-instruction faults on hosts the +dispatcher routes to that level. + +## check-dispatch-execution.cmake + +**Test:** `dispatch_surface_execution` + +Verifies that a call through the entry points on the test host enters the +ISA level the host satisfies, observed by breaking in gdb on each level's +kernel and reporting which one runs. Failure means runtime dispatch routes +to the wrong level, or a specialization disappeared behind a preprocessor +guard while still linking and counting in the symbol table. diff --git a/cmake/dispatch-checks/check-dispatch-declaration.cmake b/cmake/dispatch-checks/check-dispatch-declaration.cmake new file mode 100644 index 000000000..4e50d83f7 --- /dev/null +++ b/cmake/dispatch-checks/check-dispatch-declaration.cmake @@ -0,0 +1,346 @@ +# Copyright 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +##### +##### Checks the built kernels against the declaration, not against the generated +##### header. +##### +##### Run in script mode: +##### +##### cmake -DSVS_SURFACE_FILE=cmake/dispatch-surface.cmake \ +##### -DSVS_TYPE_PAIR_HEADER=include/svs/multi-arch/x86/preprocessor.h \ +##### -DSVS_ENUM_HEADER=include/svs/core/distance/distance_core.h \ +##### -DSVS_ARCHIVE= \ +##### -DSVS_CONSUMER_OBJECT= \ +##### -DSVS_NM= \ +##### -P cmake/dispatch-checks/check-dispatch-declaration.cmake +##### +##### cmake/dispatch-checks/check-dispatch-linkage.cmake compares the archive +##### against a probe, and both are generated from the same header: a generator +##### that dropped an extent would drop it from both and still agree. The +##### expectation here is derived from the three hand-written sources instead -- +##### the extent list and levels, the type pairs, and the enumerator order -- so +##### the generated header is not consulted at all and a generator bug has nowhere +##### to hide. +##### + +# Without it, CMP0057 is unset in script mode and the `IN_LIST` tests below are +# rejected as unknown arguments. +cmake_minimum_required(VERSION 3.21) + +foreach(required + SVS_SURFACE_FILE SVS_TYPE_PAIR_HEADER SVS_ENUM_HEADER + SVS_ARCHIVE SVS_CONSUMER_OBJECT SVS_NM +) + if(NOT ${required}) + message(FATAL_ERROR "${required} is not set.") + endif() +endforeach() +foreach(required + SVS_SURFACE_FILE SVS_TYPE_PAIR_HEADER SVS_ENUM_HEADER + SVS_ARCHIVE SVS_CONSUMER_OBJECT +) + if(NOT EXISTS "${${required}}") + message(FATAL_ERROR "${required} does not exist: ${${required}}") + endif() +endforeach() + +##### +##### Ground truth 1: the extents and the ISA levels +##### + +include("${SVS_SURFACE_FILE}") + +if(NOT SVS_SUPPORTED_DIMS OR NOT SVS_ISA_LEVELS) + message(FATAL_ERROR + "${SVS_SURFACE_FILE} declares no extents or no ISA levels; it is not a " + "dispatch-surface declaration." + ) +endif() + +# svs::Dynamic mangles as its numeric value: the extent is a size_t template +# argument, and Dynamic is SIZE_MAX. +set(svs_expected_extents ${SVS_SUPPORTED_DIMS} "18446744073709551615") + +set(svs_expected_levels) +foreach(level_spec IN LISTS SVS_ISA_LEVELS) + string(REPLACE "|" ";" fields "${level_spec}") + list(GET fields 0 level) + list(APPEND svs_expected_levels "${level}") +endforeach() + +##### +##### Ground truth 2: the enumerator order, which fixes the mangled digit +##### + +file(READ "${SVS_ENUM_HEADER}" svs_enum_text) +if(NOT svs_enum_text MATCHES "enum class AVX_AVAILABILITY[ \t\r\n]*{([^}]*)}") + message(FATAL_ERROR + "No `enum class AVX_AVAILABILITY` found in ${SVS_ENUM_HEADER}. Its " + "enumerator order is what maps an ISA level onto the digit in a mangled " + "name, so it cannot be inferred." + ) +endif() +string(REPLACE "," ";" svs_enumerators "${CMAKE_MATCH_1}") +set(svs_enum_index 0) +foreach(enumerator IN LISTS svs_enumerators) + string(STRIP "${enumerator}" enumerator) + if(enumerator) + set(svs_digit_of_${enumerator} ${svs_enum_index}) + math(EXPR svs_enum_index "${svs_enum_index} + 1") + endif() +endforeach() + +set(svs_expected_digits) +foreach(level IN LISTS svs_expected_levels) + if(NOT DEFINED svs_digit_of_${level}) + message(FATAL_ERROR + "ISA level '${level}' is declared in ${SVS_SURFACE_FILE} but is not an " + "AVX_AVAILABILITY enumerator." + ) + endif() + list(APPEND svs_expected_digits ${svs_digit_of_${level}}) +endforeach() + +##### +##### Ground truth 3: how many type pairs each level has kernels for +##### + +file(READ "${SVS_TYPE_PAIR_HEADER}" svs_pairs_text) +# Fold line continuations away so each #define is one line to match against. +string(REGEX REPLACE "\\\\[ \t]*\r?\n" " " svs_pairs_text "${svs_pairs_text}") + +# Returns the number of M(...) invocations in a type-pair list macro, following one +# level of aliasing -- `#define SVS_TYPE_PAIRS_AVX2 SVS_FOR_EACH_TYPE_PAIR`. +function(svs_type_pair_count out_var macro_name) + if(NOT svs_pairs_text MATCHES "#define ${macro_name}(\\([^)]*\\))?[ \t]+([^\n]*)") + message(FATAL_ERROR + "No `#define ${macro_name}` in ${SVS_TYPE_PAIR_HEADER}. Every ISA level " + "needs one; without it the level instantiates nothing." + ) + endif() + set(body "${CMAKE_MATCH_2}") + string(STRIP "${body}" body) + if(body MATCHES "^[A-Za-z_][A-Za-z0-9_]*$") + svs_type_pair_count(count "${body}") + set(${out_var} ${count} PARENT_SCOPE) + return() + endif() + string(REGEX MATCHALL "M\\(" hits "${body}") + list(LENGTH hits count) + if(count EQUAL 0) + message(FATAL_ERROR "${macro_name} in ${SVS_TYPE_PAIR_HEADER} invokes M zero times.") + endif() + set(${out_var} ${count} PARENT_SCOPE) +endfunction() + +##### +##### Ground truth 4: the distances +##### + +# Adding a distance means adding its Impl class here; otherwise its kernels are +# counted by nothing and a level could ship without them. +set(svs_distance_classes "L2Impl" "IPImpl" "CosineSimilarityImpl") + +set(svs_mangled_classes) +foreach(class IN LISTS svs_distance_classes) + string(LENGTH "${class}" len) + list(APPEND svs_mangled_classes "${len}${class}") +endforeach() + +##### +##### The expectation +##### + +# Keyed __; the count is the number of type pairs, +# one symbol each. +set(svs_expected_keys) +set(svs_expected_total 0) +foreach(level IN LISTS svs_expected_levels) + svs_type_pair_count(pairs "SVS_TYPE_PAIRS_${level}") + set(svs_pairs_of_${level} ${pairs}) + foreach(class IN LISTS svs_mangled_classes) + foreach(extent IN LISTS svs_expected_extents) + set(key "${class}_${extent}_${svs_digit_of_${level}}") + list(APPEND svs_expected_keys "${key}") + set(svs_want_${key} ${pairs}) + math(EXPR svs_expected_total "${svs_expected_total} + ${pairs}") + endforeach() + endforeach() +endforeach() + +##### +##### What was built +##### + +# Mangled names throughout: demangled ones carry `[clone .isra.0]` suffixes that +# differ between a local instantiation and an explicit one. +set(svs_kernel_regex "^_ZN3svs8distance([0-9]+[A-Za-z0-9_]+)ILm([0-9]+)E.*AVX_AVAILABILITYE([0-9]+)E") + +function(svs_symbols out_var file nm_arg) + execute_process( + COMMAND "${SVS_NM}" ${nm_arg} "${file}" + OUTPUT_VARIABLE raw + ERROR_VARIABLE err + RESULT_VARIABLE status + ) + if(NOT status EQUAL 0) + message(FATAL_ERROR "${SVS_NM} failed on ${file}: ${err}") + endif() + set(symbols) + string(REPLACE "\n" ";" lines "${raw}") + foreach(line IN LISTS lines) + if(line MATCHES "_ZN3svs8distance.*AVX_AVAILABILITY") + # The mangled name is the last whitespace-separated field. + string(REGEX MATCH "[^ \t]+$" symbol "${line}") + list(APPEND symbols "${symbol}") + endif() + endforeach() + list(REMOVE_DUPLICATES symbols) + set(${out_var} "${symbols}" PARENT_SCOPE) +endfunction() + +# Sets _KEYS, _FOREIGN and _HAVE_ in the caller. +function(svs_tally prefix symbols) + set(keys) + set(foreign) + foreach(symbol IN LISTS symbols) + if(NOT symbol MATCHES "${svs_kernel_regex}") + continue() + endif() + set(key "${CMAKE_MATCH_1}_${CMAKE_MATCH_2}_${CMAKE_MATCH_3}") + if(NOT key IN_LIST svs_expected_keys) + list(APPEND foreign "${symbol}") + continue() + endif() + if(NOT key IN_LIST keys) + list(APPEND keys "${key}") + set(${prefix}_HAVE_${key} 0) + endif() + math(EXPR next "${${prefix}_HAVE_${key}} + 1") + set(${prefix}_HAVE_${key} ${next}) + set(${prefix}_HAVE_${key} ${next} PARENT_SCOPE) + endforeach() + set(${prefix}_KEYS "${keys}" PARENT_SCOPE) + set(${prefix}_FOREIGN "${foreign}" PARENT_SCOPE) +endfunction() + +svs_symbols(archive_defines "${SVS_ARCHIVE}" --defined-only) +svs_symbols(consumer_defines "${SVS_CONSUMER_OBJECT}" --defined-only) +svs_symbols(consumer_references "${SVS_CONSUMER_OBJECT}" --undefined-only) + +svs_tally(ARCHIVE "${archive_defines}") +svs_tally(CONSUMER "${consumer_references}") + +##### +##### Compare +##### + +set(errors 0) + +function(svs_report_first list_var limit) + set(shown ${${list_var}}) + list(LENGTH shown count) + if(count GREATER limit) + list(SUBLIST shown 0 ${limit} shown) + endif() + foreach(entry IN LISTS shown) + message(" ${entry}") + endforeach() + if(count GREATER limit) + math(EXPR rest "${count} - ${limit}") + message(" ... and ${rest} more") + endif() +endfunction() + +# Sets _ERRORS in the caller. `what` names the thing being compared, for +# the failure message. +function(svs_compare prefix what) + set(wrong) + foreach(key IN LISTS svs_expected_keys) + set(have 0) + if(DEFINED ${prefix}_HAVE_${key}) + set(have ${${prefix}_HAVE_${key}}) + endif() + if(NOT have EQUAL ${svs_want_${key}}) + list(APPEND wrong "${key}: expected ${svs_want_${key}}, got ${have}") + endif() + endforeach() + list(LENGTH wrong n_wrong) + set(${prefix}_ERRORS 0 PARENT_SCOPE) + if(NOT n_wrong EQUAL 0) + message("${n_wrong} of ${what} disagree with ${SVS_SURFACE_FILE}.") + message("Each entry is __. A count of 0 means") + message("the declaration asks for kernels that were never built; a count") + message("below the number of type pairs means the level is missing some.") + message(" Mismatched:") + svs_report_first(wrong 10) + set(${prefix}_ERRORS 1 PARENT_SCOPE) + endif() +endfunction() + +svs_compare(ARCHIVE "the archive's kernel groups") +svs_compare(CONSUMER "the kernel groups a consumer reaches through the entry points") +math(EXPR errors "${ARCHIVE_ERRORS} + ${CONSUMER_ERRORS}") + +if(ARCHIVE_FOREIGN) + list(LENGTH ARCHIVE_FOREIGN n) + message("${n} kernels in the archive are outside the declared surface.") + message("Nothing in ${SVS_SURFACE_FILE} asks for them, so no consumer reaches") + message("them and they only add to the size of the library.") + message(" Outside the surface:") + svs_report_first(ARCHIVE_FOREIGN 10) + math(EXPR errors "${errors} + 1") +endif() + +# The whole point of the extern declarations. A kernel the consumer defines at a +# declared level came from the generic primary template at the consumer's own +# -march instead of from the library. +set(consumer_leaks) +foreach(symbol IN LISTS consumer_defines) + if(symbol MATCHES "${svs_kernel_regex}") + if(CMAKE_MATCH_3 IN_LIST svs_expected_digits) + list(APPEND consumer_leaks "${symbol}") + endif() + endif() +endforeach() +list(LENGTH consumer_leaks n_leaks) +if(NOT n_leaks EQUAL 0) + message("${n_leaks} kernels at a declared ISA level are instantiated by the consumer.") + message("Each is missing its `extern template` declaration, so every consumer of") + message("the headers builds it locally, from the generic scalar template, at") + message("whatever -march that consumer uses.") + message(" Instantiated locally:") + svs_report_first(consumer_leaks 10) + math(EXPR errors "${errors} + 1") +endif() + +if(NOT errors EQUAL 0) + message(FATAL_ERROR "dispatch declaration check failed") +endif() + +list(LENGTH svs_expected_levels n_levels) +list(LENGTH svs_expected_extents n_extents) +list(LENGTH svs_distance_classes n_distances) +set(pair_display) +foreach(level IN LISTS svs_expected_levels) + list(APPEND pair_display "${level}: ${svs_pairs_of_${level}}") +endforeach() +string(REPLACE ";" ", " pair_display "${pair_display}") +message( + "dispatch declaration: ${svs_expected_total} kernels required by " + "${SVS_SURFACE_FILE} (${n_levels} levels x ${n_extents} extents x " + "${n_distances} distances x type pairs per level {${pair_display}}); the " + "archive defines exactly those and the entry points reach all of them" +) diff --git a/cmake/dispatch-checks/check-dispatch-execution.cmake b/cmake/dispatch-checks/check-dispatch-execution.cmake new file mode 100644 index 000000000..e15eb9f2a --- /dev/null +++ b/cmake/dispatch-checks/check-dispatch-execution.cmake @@ -0,0 +1,155 @@ +# Copyright 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +##### +##### Checks which ISA level a call through the entry points actually enters. +##### +##### Run in script mode: +##### +##### cmake -DSVS_PROBE= -DSVS_NM= -DSVS_GDB= \ +##### -P cmake/dispatch-checks/check-dispatch-execution.cmake +##### +##### Everything else about the surface is a property of the symbol table, which a +##### specialization can satisfy while never running: one that disappears behind an +##### `#if` still links and still counts. This breaks on every level's kernel for one +##### extent and reports the one the host's run reaches. +##### +##### Only the level this host satisfies is checked. The weaker levels are checked by +##### hosts that satisfy only those, which is what the CI matrix is for. +##### + +foreach(required SVS_PROBE SVS_NM SVS_GDB) + if(NOT ${required}) + message(FATAL_ERROR "${required} is not set.") + endif() +endforeach() +if(NOT EXISTS "${SVS_PROBE}") + message(FATAL_ERROR "SVS_PROBE does not exist: ${SVS_PROBE}") +endif() + +# The probe owns the runtime predicates, so it -- not this script -- decides which +# level the entry points are obliged to choose here. +execute_process( + COMMAND "${SVS_PROBE}" --report + OUTPUT_VARIABLE report + ERROR_VARIABLE err + RESULT_VARIABLE status +) +if(NOT status EQUAL 0) + message(FATAL_ERROR "${SVS_PROBE} --report failed: ${err}") +endif() +if(NOT report MATCHES "expect-level ([0-9]+)") + message(FATAL_ERROR "${SVS_PROBE} --report printed no expected level:\n${report}") +endif() +set(expected_digit "${CMAKE_MATCH_1}") +if(NOT report MATCHES "probe-extent ([0-9]+)") + message(FATAL_ERROR "${SVS_PROBE} --report printed no probe extent:\n${report}") +endif() +set(extent "${CMAKE_MATCH_1}") + +if(expected_digit EQUAL 0) + message( + "dispatch execution: this host satisfies no ISA level in the surface, so the " + "entry points can only reach AVX_AVAILABILITY::NONE; nothing to check" + ) + return() +endif() + +##### +##### The candidate symbols +##### + +# L2 at float/float (`ff`) for the extent the probe reports: one symbol per level, +# including any AVX_AVAILABILITY::NONE fallback the consumer instantiated itself. +execute_process( + COMMAND "${SVS_NM}" --defined-only "${SVS_PROBE}" + OUTPUT_VARIABLE raw + ERROR_VARIABLE err + RESULT_VARIABLE status +) +if(NOT status EQUAL 0) + message(FATAL_ERROR "${SVS_NM} failed on ${SVS_PROBE}: ${err}") +endif() + +set(candidates) +string(REPLACE "\n" ";" lines "${raw}") +foreach(line IN LISTS lines) + # The mangled name is the last whitespace-separated field. + string(REGEX MATCH "[^ \t]+$" symbol "${line}") + if(NOT symbol MATCHES "^_ZN3svs8distance6L2ImplILm${extent}Eff.*7computeE") + continue() + endif() + # Skip GCC's `.isra` clones: gdb reads a dot in a linespec as a file name. + if(symbol MATCHES "\\.") + continue() + endif() + list(APPEND candidates "${symbol}") +endforeach() +list(REMOVE_DUPLICATES candidates) + +list(LENGTH candidates n_candidates) +if(n_candidates EQUAL 0) + message(FATAL_ERROR + "No L2 kernel symbol for extent ${extent} at float/float is defined in " + "${SVS_PROBE}. Either the entry points inlined every kernel, in which case " + "the `extern template` declarations are not in effect, or the surface no " + "longer covers that extent." + ) +endif() + +##### +##### Run under the debugger +##### + +set(gdb_args -q -batch) +foreach(symbol IN LISTS candidates) + list(APPEND gdb_args -ex "break ${symbol}") +endforeach() +list(APPEND gdb_args -ex "run --report" -ex "info symbol $pc") + +execute_process( + COMMAND "${SVS_GDB}" ${gdb_args} "${SVS_PROBE}" + OUTPUT_VARIABLE gdb_out + ERROR_VARIABLE gdb_err + RESULT_VARIABLE status +) +if(NOT status EQUAL 0) + message("${gdb_out}") + message(FATAL_ERROR "${SVS_GDB} failed on ${SVS_PROBE}: ${gdb_err}") +endif() + +# gdb demangles for `info symbol`, so the level appears as the enum's underlying +# value: `(svs::distance::AVX_AVAILABILITY)2`. +if(NOT gdb_out MATCHES "AVX_AVAILABILITY\\)([0-9]+)") + message("${gdb_out}") + message(FATAL_ERROR + "The run never entered any of the ${n_candidates} kernels breakpointed for " + "extent ${extent}. A call through the entry points reached none of them, so " + "the dispatch is not routing to the surface." + ) +endif() +set(actual_digit "${CMAKE_MATCH_1}") + +if(NOT actual_digit EQUAL expected_digit) + message("The entry points routed a call to AVX_AVAILABILITY level ${actual_digit},") + message("but this host satisfies level ${expected_digit}. Either the runtime") + message("predicates in the entry points disagree with the ones the probe uses, or") + message("the enumerator order in the surface no longer matches the dispatch order.") + message(FATAL_ERROR "dispatch execution check failed") +endif() + +message( + "dispatch execution: a call to L2 at extent ${extent} entered " + "AVX_AVAILABILITY level ${actual_digit}, the highest this host satisfies" +) diff --git a/cmake/dispatch-checks/check-dispatch-instructions.cmake b/cmake/dispatch-checks/check-dispatch-instructions.cmake new file mode 100644 index 000000000..fafd97d2d --- /dev/null +++ b/cmake/dispatch-checks/check-dispatch-instructions.cmake @@ -0,0 +1,148 @@ +# Copyright 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +##### +##### Checks that an ISA level's object file stays inside its instruction budget. +##### +##### Run in script mode: +##### +##### cmake -DSVS_OBJECT= \ +##### -DSVS_LEVEL=AVX2 -DSVS_ARCH=haswell \ +##### -DSVS_OBJDUMP= \ +##### -P cmake/dispatch-checks/check-dispatch-instructions.cmake +##### +##### A level promises the host satisfies its runtime predicate and nothing more, +##### so an instruction the predicate does not guarantee is an illegal-instruction +##### fault on a host the dispatcher considers supported. +##### + +# Without it, CMP0007 is unset in script mode and the empty "forbids nothing" +# field of a budget row below is silently dropped rather than read as empty. +cmake_minimum_required(VERSION 3.21) + +foreach(required SVS_OBJECT SVS_LEVEL SVS_ARCH SVS_OBJDUMP) + if(NOT ${required}) + message(FATAL_ERROR "${required} is not set.") + endif() +endforeach() +if(NOT EXISTS "${SVS_OBJECT}") + message(FATAL_ERROR "SVS_OBJECT does not exist: ${SVS_OBJECT}") +endif() + +# What each instruction class looks like in AT&T disassembly. Register classes are +# matched with their `%` sigil so that a mangled name can never look like one. +set(svs_class_ymm "%ymm") +set(svs_class_zmm "%zmm") +set(svs_class_mask "%k[1-7]") +set(svs_class_vnni "vpdpwssd|vpdpbusd|vpdpwssds|vpdpbusds") + +# "<-march>||", one row per instruction budget +# any ISA level declares. A budget with no row here is a hard error: an +# unrecognized -march silently permitting everything is how a level ends up +# emitting instructions its runtime predicate does not guarantee. +set(svs_budget_table + "x86-64||ymm zmm mask vnni" + "haswell|ymm|zmm mask vnni" + "skylake-avx512|zmm|vnni" + "cascadelake|zmm|" +) + +set(svs_budget_found FALSE) +foreach(row IN LISTS svs_budget_table) + string(REPLACE "|" ";" fields "${row}") + list(GET fields 0 arch) + if(arch STREQUAL SVS_ARCH) + list(GET fields 1 svs_required) + list(GET fields 2 svs_forbidden) + set(svs_budget_found TRUE) + break() + endif() +endforeach() +if(NOT svs_budget_found) + message(FATAL_ERROR + "No instruction budget is known for -march=${SVS_ARCH} (ISA level " + "${SVS_LEVEL}). Add a row to svs_budget_table in this file stating what " + "that budget requires and forbids." + ) +endif() +string(REPLACE " " ";" svs_required "${svs_required}") +string(REPLACE " " ";" svs_forbidden "${svs_forbidden}") + +execute_process( + COMMAND "${SVS_OBJDUMP}" -d --no-show-raw-insn "${SVS_OBJECT}" + OUTPUT_VARIABLE svs_disassembly + ERROR_VARIABLE err + RESULT_VARIABLE status +) +if(NOT status EQUAL 0) + message(FATAL_ERROR "${SVS_OBJDUMP} failed on ${SVS_OBJECT}: ${err}") +endif() + +# Drop the `:` heading lines: they are the only place a mangled name +# appears, and a mangled name must not be mistaken for an instruction. +string(REGEX REPLACE "\n[0-9a-f]+ <[^\n]*>:" "\n" svs_disassembly "${svs_disassembly}") + +function(svs_count_class out_var class) + if(NOT DEFINED svs_class_${class}) + message(FATAL_ERROR "Unknown instruction class '${class}' in the budget table.") + endif() + string(REGEX MATCHALL "${svs_class_${class}}" hits "${svs_disassembly}") + list(LENGTH hits count) + set(${out_var} ${count} PARENT_SCOPE) +endfunction() + +set(errors 0) +set(summary) + +foreach(class IN LISTS svs_required) + svs_count_class(count "${class}") + if(count EQUAL 0) + message("${SVS_OBJECT} contains no ${class} instructions.") + message("ISA level ${SVS_LEVEL} is compiled at -march=${SVS_ARCH}, which is") + message("chosen precisely so the kernels use them. An object without any is") + message("a level built at the wrong budget, or one whose kernels fell back") + message("to the generic scalar template.") + math(EXPR errors "${errors} + 1") + else() + list(APPEND summary "${count} ${class}") + endif() +endforeach() + +foreach(class IN LISTS svs_forbidden) + svs_count_class(count "${class}") + if(NOT count EQUAL 0) + message("${SVS_OBJECT} contains ${count} ${class} instructions.") + message("ISA level ${SVS_LEVEL} guarantees only what its runtime predicate") + message("tests, so a host the dispatcher routes here faults on them. Either") + message("lower -march=${SVS_ARCH} in cmake/dispatch-surface.cmake, or give") + message("these kernels their own level with a predicate that covers them.") + math(EXPR errors "${errors} + 1") + endif() +endforeach() + +if(NOT errors EQUAL 0) + message(FATAL_ERROR "dispatch instruction check failed for level ${SVS_LEVEL}") +endif() + +string(REPLACE ";" ", " summary_display "${summary}") +if(svs_forbidden STREQUAL "") + set(forbidden_display "nothing forbidden") +else() + string(REPLACE ";" ", " forbidden_display "${svs_forbidden}") + set(forbidden_display "no ${forbidden_display}") +endif() +message( + "dispatch instructions: level ${SVS_LEVEL} at -march=${SVS_ARCH} has " + "${summary_display}, and ${forbidden_display}" +) diff --git a/cmake/dispatch-checks/check-dispatch-linkage.cmake b/cmake/dispatch-checks/check-dispatch-linkage.cmake new file mode 100644 index 000000000..6ec22b711 --- /dev/null +++ b/cmake/dispatch-checks/check-dispatch-linkage.cmake @@ -0,0 +1,163 @@ +# Copyright 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +##### +##### Checks the dispatch surface against the symbols that were actually built. +##### +##### Run in script mode: +##### +##### cmake -DSVS_PROBE_OBJECT= \ +##### -DSVS_ARCHIVE= \ +##### -DSVS_NM= \ +##### -P cmake/dispatch-checks/check-dispatch-linkage.cmake +##### +##### The probe object names every kernel the surface declares and nothing else +##### (see tests/multi-arch/x86/link_probe.cpp), so the kernels it *references* +##### are exactly the kernels the archive must *define* -- and it must define no +##### others. Both directions are checked, plus that the probe defines none of +##### its own. +##### + +foreach(required SVS_PROBE_OBJECT SVS_ARCHIVE SVS_NM) + if(NOT ${required}) + message(FATAL_ERROR "${required} is not set.") + endif() +endforeach() +foreach(required SVS_PROBE_OBJECT SVS_ARCHIVE) + if(NOT EXISTS "${${required}}") + message(FATAL_ERROR "${required} does not exist: ${${required}}") + endif() +endforeach() + +# Distance kernels, and nothing else in the archive. Mangled names are used +# throughout: demangled ones carry `[clone .isra.0]` suffixes that differ between +# a local instantiation and an explicit one. +set(svs_kernel_regex "_ZN3svs8distance.*Impl") + +# Returns the mangled names of the matching symbols, one per list element. +function(svs_symbols out_var) + cmake_parse_arguments(arg "" "FILE" "NM_ARGS" ${ARGN}) + execute_process( + COMMAND "${SVS_NM}" ${arg_NM_ARGS} "${arg_FILE}" + OUTPUT_VARIABLE raw + ERROR_VARIABLE err + RESULT_VARIABLE status + ) + if(NOT status EQUAL 0) + message(FATAL_ERROR "${SVS_NM} failed on ${arg_FILE}: ${err}") + endif() + + set(symbols) + string(REPLACE "\n" ";" lines "${raw}") + foreach(line IN LISTS lines) + if(line MATCHES "${svs_kernel_regex}") + # The mangled name is the last whitespace-separated field. + string(REGEX MATCH "[^ \t]+$" symbol "${line}") + list(APPEND symbols "${symbol}") + endif() + endforeach() + list(REMOVE_DUPLICATES symbols) + list(SORT symbols) + set(${out_var} "${symbols}" PARENT_SCOPE) +endfunction() + +# Shows up to `limit` entries of a list. The names stay mangled -- pipe them +# through c++filt to read them. +function(svs_report_symbols symbols limit) + list(LENGTH symbols count) + set(shown ${symbols}) + if(count GREATER limit) + list(SUBLIST shown 0 ${limit} shown) + endif() + foreach(symbol IN LISTS shown) + message(" ${symbol}") + endforeach() + if(count GREATER limit) + math(EXPR rest "${count} - ${limit}") + message(" ... and ${rest} more") + endif() +endfunction() + +svs_symbols(probe_defines FILE "${SVS_PROBE_OBJECT}" NM_ARGS --defined-only) +svs_symbols(probe_references FILE "${SVS_PROBE_OBJECT}" NM_ARGS --undefined-only) +svs_symbols(archive_defines FILE "${SVS_ARCHIVE}" NM_ARGS --defined-only) + +list(LENGTH probe_defines n_probe_defines) +list(LENGTH probe_references n_probe_references) +list(LENGTH archive_defines n_archive_defines) + +set(errors 0) + +# A probe that names nothing is not a passing probe. This is what an LTO build +# looks like here, since the object holds IR rather than symbols. +if(n_probe_references EQUAL 0 AND n_probe_defines EQUAL 0) + message("${SVS_PROBE_OBJECT} names no distance kernels at all.") + message("Nothing can be concluded from it. If this is a link-time-optimized") + message("build, nm cannot see the symbols and this check does not apply.") + message(FATAL_ERROR "dispatch linkage check found no symbols to check") +endif() + +# The probe compiles at -march=x86-64 and guarantees nothing about the host, so a +# kernel it defines itself is a kernel some other consumer would also define +# itself -- at whatever -march that consumer happens to use. +if(NOT n_probe_defines EQUAL 0) + message("${n_probe_defines} kernels are instantiated by the probe itself.") + message("Each is missing its `extern template` declaration, so every consumer") + message("of the headers instantiates it locally, from the generic primary") + message("template, at the consumer's own -march. Declare them: the extern") + message("blocks in the distance headers must cover the whole surface.") + message(" Instantiated locally:") + svs_report_symbols("${probe_defines}" 10) + math(EXPR errors "${errors} + 1") +endif() + +# Declared but never instantiated. The link should already have failed, so this +# only fires when the object is inspected without being linked. +set(missing ${probe_references}) +if(archive_defines) + list(REMOVE_ITEM missing ${archive_defines}) +endif() +list(LENGTH missing n_missing) +if(NOT n_missing EQUAL 0) + message("${n_missing} kernels are declared but never instantiated.") + message("They are declared `extern template` in the distance headers but no") + message("translation unit defines them, so linking against the library fails.") + message(" Undefined:") + svs_report_symbols("${missing}" 10) + math(EXPR errors "${errors} + 1") +endif() + +# Instantiated but unreachable through the surface: dead weight in the archive. +set(unreachable ${archive_defines}) +if(probe_references) + list(REMOVE_ITEM unreachable ${probe_references}) +endif() +list(LENGTH unreachable n_unreachable) +if(NOT n_unreachable EQUAL 0) + message("${n_unreachable} kernels are instantiated but are not part of the") + message("dispatch surface. Nothing declares them, so no consumer reaches") + message("them; they only add to the size of the library.") + message(" Unreachable:") + svs_report_symbols("${unreachable}" 10) + math(EXPR errors "${errors} + 1") +endif() + +if(NOT errors EQUAL 0) + message(FATAL_ERROR "dispatch linkage check failed") +endif() + +message( + "dispatch linkage: ${n_archive_defines} kernels declared, instantiated and " + "reachable; none instantiated by the consumer" +) diff --git a/cmake/dispatch-surface.cmake b/cmake/dispatch-surface.cmake new file mode 100644 index 000000000..971b1785f --- /dev/null +++ b/cmake/dispatch-surface.cmake @@ -0,0 +1,101 @@ +# Copyright 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +##### +##### The dispatch surface for the x86 distance kernels. +##### +##### This file is the single place the extent list and the ISA levels are +##### written down. Everything derived from them is generated: +##### +##### - include/svs/core/distance/dispatch_surface.h, which drives every +##### `extern template` and explicit instantiation, and `supported_dim_list` +##### - the object library each ISA level's translation unit is compiled into, +##### and the instruction budget it is compiled at +##### +##### Edit this file. Do not edit the generated header; it is regenerated on +##### every configure and your changes there will be overwritten. +##### +##### A build may point somewhere else with -DSVS_DISPATCH_SURFACE_FILE=, +##### in which case the committed header is left alone and only the build tree +##### describes that surface. +##### +##### Bookkeeping +##### +##### To add or remove a fixed extent: +##### Edit `SVS_SUPPORTED_DIMS`. The generated header, every `extern +##### template`, and `supported_dim_list` follow automatically. +##### +##### To add an ISA level: +##### 1. Add a row to `SVS_ISA_LEVELS`. +##### 2. Add a `SVS_TYPE_PAIRS_` list in +##### include/svs/multi-arch/x86/preprocessor.h, listing the element-type +##### pairs that level has kernels for. +##### 3. Add the level's translation unit at +##### include/svs/multi-arch/x86/.cpp. +##### The object library and its compile flags follow from the row here. A +##### level without a type-pair list is a compile error, not an empty +##### instantiation set. +##### +##### To add or remove an element-type pair for a level: +##### Edit that level's `SVS_TYPE_PAIRS_` list in +##### include/svs/multi-arch/x86/preprocessor.h. Not configured here: a type +##### pair exists because an implementation exists for it. +##### +##### To change a level's instruction budget: +##### Edit the middle field of its row in `SVS_ISA_LEVELS`, observing the +##### constraint recorded there. +##### + +# Extents that get their own fixed-extent kernel. +# +# This list is a *performance* choice, not a compatibility one. Any +# dimensionality not listed here still works and is fully supported: it +# dispatches to the `svs::Dynamic` kernel, which takes the length at run time. +# Listing an extent buys a fully unrolled kernel for it, at the cost of one more +# set of instantiations across every ISA level and type pair. +# +# `svs::Dynamic` is required and is appended automatically -- do not list it. +set(SVS_SUPPORTED_DIMS 64 96 100 128 160 200 512 768) + +# Runtime ISA levels. Each has one translation unit, which instantiates every +# extent above at that level. +# +# Only levels with their own translation unit appear here, which is why +# `AVX_AVAILABILITY::NONE` has no row: nothing instantiates its kernels ahead +# of time, so each translation unit that uses them instantiates them itself, at +# its own `-march` -- generic `x86-64` for this project's own build. A row here +# would name a translation unit and an object library that do not exist. +# +# `AVX_AVAILABILITY` mangles positionally, so `NONE` must keep its enumerator +# value despite having no row; renumbering the enumerators is an ABI break. +# +# || +# +# enumerator a value of `svs::distance::AVX_AVAILABILITY` +# instruction budget the -march this level's kernels are compiled to; it must +# match what the level guarantees about the host, because +# anything the compiler is allowed to emit here will run on +# any host that satisfies the level +# TU infix names both the level's translation unit, +# include/svs/multi-arch/x86/.cpp, and the object +# library it is compiled into +# +# Adding a level here also requires a `SVS_TYPE_PAIRS_` list in +# include/svs/multi-arch/x86/preprocessor.h, saying which element-type pairs +# that level has kernels for. That is deliberately not configured here: a type +# pair exists because an implementation exists for it. +set(SVS_ISA_LEVELS + "AVX2|haswell|avx2" + "AVX512|cascadelake|avx512" +) diff --git a/cmake/generate-dispatch-surface.cmake b/cmake/generate-dispatch-surface.cmake new file mode 100644 index 000000000..65aaa7c43 --- /dev/null +++ b/cmake/generate-dispatch-surface.cmake @@ -0,0 +1,250 @@ +# Copyright 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +##### +##### Derives the dispatch surface declared in cmake/dispatch-surface.cmake. +##### +##### Produces: +##### include/svs/core/distance/dispatch_surface.h (source tree, committed) +##### SVS_DISPATCH_TU_SPECS -- "|||", one per ISA level +##### + +include_guard(GLOBAL) + +set(SVS_DEFAULT_DISPATCH_SURFACE_FILE "${CMAKE_CURRENT_LIST_DIR}/dispatch-surface.cmake") +set(SVS_DISPATCH_SURFACE_FILE "${SVS_DEFAULT_DISPATCH_SURFACE_FILE}" + CACHE FILEPATH + "Declaration of the ahead-of-time distance-kernel dispatch surface" +) +if(NOT EXISTS "${SVS_DISPATCH_SURFACE_FILE}") + message(FATAL_ERROR + "SVS_DISPATCH_SURFACE_FILE does not exist: ${SVS_DISPATCH_SURFACE_FILE}" + ) +endif() +include("${SVS_DISPATCH_SURFACE_FILE}") + +file(REAL_PATH "${SVS_DISPATCH_SURFACE_FILE}" svs_surface_real) +file(REAL_PATH "${SVS_DEFAULT_DISPATCH_SURFACE_FILE}" svs_default_surface_real) +if(svs_surface_real STREQUAL svs_default_surface_real) + set(svs_surface_is_default TRUE) +else() + set(svs_surface_is_default FALSE) + message(STATUS + "Dispatch surface overridden by ${SVS_DISPATCH_SURFACE_FILE}; the " + "committed header will not be refreshed" + ) +endif() + +# Re-run configure when the declaration changes, so the generated header and the +# translation units cannot go stale. +set_property( + DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS + "${SVS_DISPATCH_SURFACE_FILE}" +) + +##### +##### Validate the extent list +##### + +if(NOT SVS_SUPPORTED_DIMS) + message(FATAL_ERROR + "SVS_SUPPORTED_DIMS is empty in ${SVS_DISPATCH_SURFACE_FILE}. At least " + "one fixed extent is required." + ) +endif() + +foreach(dim IN LISTS SVS_SUPPORTED_DIMS) + if(NOT dim MATCHES "^[1-9][0-9]*$") + message(FATAL_ERROR + "SVS_SUPPORTED_DIMS contains '${dim}', which is not a positive " + "integer. svs::Dynamic is required and is appended automatically, " + "so it must not be listed." + ) + endif() +endforeach() + +set(svs_dims_sorted ${SVS_SUPPORTED_DIMS}) +list(REMOVE_DUPLICATES svs_dims_sorted) +list(LENGTH SVS_SUPPORTED_DIMS svs_dims_given) +list(LENGTH svs_dims_sorted svs_dims_unique) +if(NOT svs_dims_given EQUAL svs_dims_unique) + message(FATAL_ERROR + "SVS_SUPPORTED_DIMS contains duplicate extents. Every extent must " + "appear exactly once." + ) +endif() + +# svs::Dynamic is mandatory: it is what serves every dimensionality without a +# fixed-extent kernel, and the library is incorrect without it. +set(svs_dim_list ${SVS_SUPPORTED_DIMS} "svs::Dynamic") +list(LENGTH svs_dim_list SVS_GEN_DIM_COUNT) + +##### +##### Validate the ISA levels +##### + +if(NOT SVS_ISA_LEVELS) + message(FATAL_ERROR "SVS_ISA_LEVELS is empty in ${SVS_DISPATCH_SURFACE_FILE}.") +endif() + +set(svs_seen_levels) +set(svs_seen_infixes) +foreach(level_spec IN LISTS SVS_ISA_LEVELS) + string(REPLACE "|" ";" level_fields "${level_spec}") + list(LENGTH level_fields nfields) + if(NOT nfields EQUAL 3) + message(FATAL_ERROR + "Malformed SVS_ISA_LEVELS entry '${level_spec}': expected exactly " + "three '|'-separated fields ||." + ) + endif() + list(GET level_fields 0 level) + list(GET level_fields 1 arch) + list(GET level_fields 2 infix) + foreach(field level arch infix) + if(NOT ${field}) + message(FATAL_ERROR + "Malformed SVS_ISA_LEVELS entry '${level_spec}': ${field} is empty." + ) + endif() + endforeach() + if(level IN_LIST svs_seen_levels) + message(FATAL_ERROR "Duplicate ISA level '${level}' in SVS_ISA_LEVELS.") + endif() + if(infix IN_LIST svs_seen_infixes) + message(FATAL_ERROR + "Duplicate TU infix '${infix}' in SVS_ISA_LEVELS; infixes name " + "generated files and must be unique." + ) + endif() + list(APPEND svs_seen_levels ${level}) + list(APPEND svs_seen_infixes ${infix}) +endforeach() + +##### +##### Generate the header +##### + +# Line continuations are emitted with a trailing backslash; the generated macros +# are one logical line each. +set(SVS_GEN_DIM_LOOP "\\\n") +foreach(dim IN LISTS svs_dim_list) + string(APPEND SVS_GEN_DIM_LOOP " M(${dim}) \\\n") +endforeach() +string(APPEND SVS_GEN_DIM_LOOP " /* end */") + +set(SVS_GEN_TARGET_LOOP "\\\n") +set(SVS_GEN_LEVEL_LOOP "\\\n") +set(SVS_DISPATCH_TU_SPECS) +set(svs_x86_src_dir "${PROJECT_SOURCE_DIR}/include/svs/multi-arch/x86") +foreach(level_spec IN LISTS SVS_ISA_LEVELS) + string(REPLACE "|" ";" level_fields "${level_spec}") + list(GET level_fields 0 level) + list(GET level_fields 1 arch) + list(GET level_fields 2 infix) + + string(APPEND SVS_GEN_LEVEL_LOOP " M(${level}) \\\n") + foreach(dim IN LISTS svs_dim_list) + string(APPEND SVS_GEN_TARGET_LOOP " M(${dim}, ${level}) \\\n") + endforeach() + + # One translation unit per level, named after the level's infix. The file + # itself is short -- it loops over the generated extent list -- but it is + # committed rather than generated, because the private repository compiles + # these sources by path. + set(tu_src "${svs_x86_src_dir}/${infix}.cpp") + if(NOT EXISTS "${tu_src}") + message(FATAL_ERROR + "ISA level '${level}' has no translation unit: expected ${tu_src}. " + "Adding a level to SVS_ISA_LEVELS requires creating that file." + ) + endif() + list(APPEND SVS_DISPATCH_TU_SPECS "${tu_src}|${level}|${arch}|${infix}") + list(APPEND svs_level_report + "AVX_AVAILABILITY::${level} -march=${arch} ${infix}.cpp" + ) +endforeach() +string(APPEND SVS_GEN_TARGET_LOOP " /* end */") +string(APPEND SVS_GEN_LEVEL_LOOP " /* end */") + +##### +##### Emit the header +##### + +# The build always compiles against the build-tree copy, and it is placed ahead +# of the source include directory so that it wins. +set(SVS_GENERATED_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/generated/include") +set(SVS_GENERATED_DISPATCH_HEADER + "${SVS_GENERATED_INCLUDE_DIR}/svs/core/distance/dispatch_surface.h" +) +configure_file( + "${CMAKE_CURRENT_LIST_DIR}/templates/dispatch_surface.h.in" + "${SVS_GENERATED_DISPATCH_HEADER}" + @ONLY +) +target_include_directories( + ${SVS_LIB} BEFORE INTERFACE $ +) + +# Refresh the committed copy too, but only for the default declaration: the +# committed header exists so that a bare `-I include` compile works without +# CMake, and a one-off build with an overridden surface must not rewrite it. +# configure_file only touches the file when the content changes, so this neither +# dirties the tree nor forces rebuilds. +if(svs_surface_is_default) + configure_file( + "${CMAKE_CURRENT_LIST_DIR}/templates/dispatch_surface.h.in" + "${PROJECT_SOURCE_DIR}/include/svs/core/distance/dispatch_surface.h" + @ONLY + ) +endif() + +##### +##### Report the surface +##### + +list(LENGTH SVS_ISA_LEVELS svs_level_count) +string(REPLACE ";" " " svs_dims_display "${SVS_SUPPORTED_DIMS}") +message(STATUS + "Dispatch surface: ${SVS_GEN_DIM_COUNT} extents x ${svs_level_count} ISA levels" +) +message(STATUS " extents: ${svs_dims_display} svs::Dynamic") +foreach(entry IN LISTS svs_level_report) + message(STATUS " level: ${entry}") +endforeach() + +# Every enumerator without a translation unit is still reachable -- the entry +# points fall back to it -- so its kernels are built by each consumer instead. +set(svs_enum_header "${PROJECT_SOURCE_DIR}/include/svs/core/distance/distance_core.h") +if(EXISTS "${svs_enum_header}") + file(READ "${svs_enum_header}" svs_enum_text) + if(svs_enum_text MATCHES "enum class AVX_AVAILABILITY[ \t\r\n]*{([^}]*)}") + string(REPLACE "," ";" svs_enumerators "${CMAKE_MATCH_1}") + set(svs_undeclared) + foreach(enumerator IN LISTS svs_enumerators) + string(STRIP "${enumerator}" enumerator) + if(enumerator AND NOT enumerator IN_LIST svs_seen_levels) + list(APPEND svs_undeclared "${enumerator}") + endif() + endforeach() + if(svs_undeclared) + string(REPLACE ";" ", " svs_undeclared_display "${svs_undeclared}") + message(STATUS " not in the surface: ${svs_undeclared_display}") + message(STATUS + " dispatched to, but compiled by no translation unit, so " + "every consumer instantiates those kernels itself, at its own -march" + ) + endif() + endif() +endif() diff --git a/cmake/multi-arch.cmake b/cmake/multi-arch.cmake index aeb81e693..74be6c57f 100644 --- a/cmake/multi-arch.cmake +++ b/cmake/multi-arch.cmake @@ -12,25 +12,29 @@ # See the License for the specific language governing permissions and # limitations under the License. -set(SVS_X86_SRC_DIR "${PROJECT_SOURCE_DIR}/include/svs/multi-arch/x86") -set(SVS_X86 - "${SVS_X86_SRC_DIR}/avx2.cpp,avx2,haswell" - "${SVS_X86_SRC_DIR}/avx512.cpp,avx512,cascadelake" -) +# Writes the generated dispatch-surface header and populates +# SVS_DISPATCH_TU_SPECS -- "|||", one entry per ISA +# level. The extent list and the levels themselves are declared in +# cmake/dispatch-surface.cmake. +include("${CMAKE_CURRENT_LIST_DIR}/generate-dispatch-surface.cmake") set(SVS_X86_OBJECT_FILES) -foreach(x86_info IN LISTS SVS_X86) - string(REPLACE "," ";" x86_info "${x86_info}") - list(GET x86_info 0 src) - list(GET x86_info 1 avx) - list(GET x86_info 2 arch) - set(lib_name "svs_x86_${avx}") +foreach(tu_spec IN LISTS SVS_DISPATCH_TU_SPECS) + string(REPLACE "|" ";" tu_fields "${tu_spec}") + list(GET tu_fields 0 src) + list(GET tu_fields 2 arch) + list(GET tu_fields 3 infix) + + # Carries the instruction budget for this level, and nothing else. + set(lib_name "svs_x86_${infix}") add_library(${lib_name} INTERFACE) target_compile_options(${lib_name} INTERFACE -march=${arch} -mtune=${arch}) - set(obj_name ${arch}_obj) + set(obj_name ${arch}_obj) add_library(${obj_name} OBJECT ${src}) - target_link_libraries(${obj_name} PRIVATE ${SVS_LIB} svs::compile_options fmt::fmt ${lib_name}) + target_link_libraries( + ${obj_name} PRIVATE ${SVS_LIB} svs::compile_options fmt::fmt ${lib_name} + ) list(APPEND SVS_X86_OBJECT_FILES $) endforeach() diff --git a/cmake/templates/dispatch_surface.h.in b/cmake/templates/dispatch_surface.h.in new file mode 100644 index 000000000..64acd976d --- /dev/null +++ b/cmake/templates/dispatch_surface.h.in @@ -0,0 +1,40 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// GENERATED FILE -- DO NOT EDIT. Regenerated on every CMake configure from +// cmake/dispatch-surface.cmake; committed so a bare `-I include` compile works. + +#pragma once + +// clang-format off +// Reflowing would realign the escaped newlines and make the committed copy +// disagree with the one CMake writes -- an endless format/regenerate loop. + +// Number of extents with a fixed-extent kernel, including svs::Dynamic. +#define SVS_SUPPORTED_DIM_COUNT @SVS_GEN_DIM_COUNT@ + +// Invokes M(extent) once per extent. +#define SVS_FOR_EACH_SUPPORTED_DIM(M) @SVS_GEN_DIM_LOOP@ + +// Invokes M(extent, isa_level) once per (extent, ISA level) pair -- that is, +// once per kernel the library compiles ahead of time, modulo type pairs. +#define SVS_FOR_EACH_DISPATCH_TARGET(M) @SVS_GEN_TARGET_LOOP@ + +// Invokes M(isa_level) once per ISA level, weakest first. AVX_AVAILABILITY +// enumerators without a translation unit are absent: this is the surface. +#define SVS_FOR_EACH_ISA_LEVEL(M) @SVS_GEN_LEVEL_LOOP@ + +// clang-format on diff --git a/include/svs/core/distance/cosine.h b/include/svs/core/distance/cosine.h index 9f4924997..4cd8e3780 100644 --- a/include/svs/core/distance/cosine.h +++ b/include/svs/core/distance/cosine.h @@ -500,26 +500,11 @@ struct CosineSimilarityImpl { #if defined(__x86_64__) #include "svs/multi-arch/x86/preprocessor.h" -// TODO: connect with dim_supported_list -DISTANCE_CS_EXTERN_TEMPLATE(64, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_EXTERN_TEMPLATE(96, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_EXTERN_TEMPLATE(100, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_EXTERN_TEMPLATE(128, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_EXTERN_TEMPLATE(160, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_EXTERN_TEMPLATE(200, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_EXTERN_TEMPLATE(512, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_EXTERN_TEMPLATE(768, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_EXTERN_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX512); - -DISTANCE_CS_EXTERN_TEMPLATE(64, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_EXTERN_TEMPLATE(96, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_EXTERN_TEMPLATE(100, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_EXTERN_TEMPLATE(128, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_EXTERN_TEMPLATE(160, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_EXTERN_TEMPLATE(200, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_EXTERN_TEMPLATE(512, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_EXTERN_TEMPLATE(768, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_EXTERN_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX2); +// Declare every kernel the library compiles ahead of time. Missing an entry here +// makes a consumer instantiate it locally, at the consumer's own -march. +#define SVS_CS_EXTERN(DIM, LEVEL) SVS_INSTANTIATE_CS(extern template, DIM, LEVEL) +SVS_FOR_EACH_DISPATCH_TARGET(SVS_CS_EXTERN) +#undef SVS_CS_EXTERN #endif } // namespace svs::distance diff --git a/include/svs/core/distance/dispatch_surface.h b/include/svs/core/distance/dispatch_surface.h new file mode 100644 index 000000000..a357ac0a8 --- /dev/null +++ b/include/svs/core/distance/dispatch_surface.h @@ -0,0 +1,72 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// GENERATED FILE -- DO NOT EDIT. Regenerated on every CMake configure from +// cmake/dispatch-surface.cmake; committed so a bare `-I include` compile works. + +#pragma once + +// clang-format off +// Reflowing would realign the escaped newlines and make the committed copy +// disagree with the one CMake writes -- an endless format/regenerate loop. + +// Number of extents with a fixed-extent kernel, including svs::Dynamic. +#define SVS_SUPPORTED_DIM_COUNT 9 + +// Invokes M(extent) once per extent. +#define SVS_FOR_EACH_SUPPORTED_DIM(M) \ + M(64) \ + M(96) \ + M(100) \ + M(128) \ + M(160) \ + M(200) \ + M(512) \ + M(768) \ + M(svs::Dynamic) \ + /* end */ + +// Invokes M(extent, isa_level) once per (extent, ISA level) pair -- that is, +// once per kernel the library compiles ahead of time, modulo type pairs. +#define SVS_FOR_EACH_DISPATCH_TARGET(M) \ + M(64, AVX2) \ + M(96, AVX2) \ + M(100, AVX2) \ + M(128, AVX2) \ + M(160, AVX2) \ + M(200, AVX2) \ + M(512, AVX2) \ + M(768, AVX2) \ + M(svs::Dynamic, AVX2) \ + M(64, AVX512) \ + M(96, AVX512) \ + M(100, AVX512) \ + M(128, AVX512) \ + M(160, AVX512) \ + M(200, AVX512) \ + M(512, AVX512) \ + M(768, AVX512) \ + M(svs::Dynamic, AVX512) \ + /* end */ + +// Invokes M(isa_level) once per ISA level, weakest first. AVX_AVAILABILITY +// enumerators without a translation unit are absent: this is the surface. +#define SVS_FOR_EACH_ISA_LEVEL(M) \ + M(AVX2) \ + M(AVX512) \ + /* end */ + +// clang-format on diff --git a/include/svs/core/distance/distance_core.h b/include/svs/core/distance/distance_core.h index 4f59f9de0..5d50dee1f 100644 --- a/include/svs/core/distance/distance_core.h +++ b/include/svs/core/distance/distance_core.h @@ -21,6 +21,9 @@ #include "svs/lib/saveload.h" #include "svs/lib/type_traits.h" +// The extent list and the ISA levels, generated from cmake/dispatch-surface.cmake. +#include "svs/core/distance/dispatch_surface.h" + #include #include @@ -28,9 +31,16 @@ namespace svs::distance { enum class AVX_AVAILABILITY { NONE, AVX2, AVX512 }; -constexpr std::array supported_dim_list{ - 64, 96, 100, 128, 160, 200, 512, 768, svs::Dynamic}; +/// The extents that have a fixed-extent kernel, including svs::Dynamic. +#define SVS_DIM_LIST_ENTRY(N) N, +constexpr std::array supported_dim_list{ + SVS_FOR_EACH_SUPPORTED_DIM(SVS_DIM_LIST_ENTRY)}; +#undef SVS_DIM_LIST_ENTRY +/// Whether N has a fixed-extent kernel. +/// +/// Not a capability test: every dimensionality is supported. An extent answering +/// `false` dispatches to the svs::Dynamic kernel instead of a fully unrolled one. template constexpr bool is_dim_supported() { for (auto i : supported_dim_list) { if (i == N) { diff --git a/include/svs/core/distance/euclidean.h b/include/svs/core/distance/euclidean.h index 08aed5d25..a2fa68483 100644 --- a/include/svs/core/distance/euclidean.h +++ b/include/svs/core/distance/euclidean.h @@ -438,26 +438,11 @@ template struct L2Impl { #include "svs/multi-arch/x86/preprocessor.h" -// TODO: connect with dim_supported_list -DISTANCE_L2_EXTERN_TEMPLATE(64, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_EXTERN_TEMPLATE(96, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_EXTERN_TEMPLATE(100, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_EXTERN_TEMPLATE(128, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_EXTERN_TEMPLATE(160, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_EXTERN_TEMPLATE(200, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_EXTERN_TEMPLATE(512, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_EXTERN_TEMPLATE(768, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_EXTERN_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX512); - -DISTANCE_L2_EXTERN_TEMPLATE(64, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_EXTERN_TEMPLATE(96, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_EXTERN_TEMPLATE(100, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_EXTERN_TEMPLATE(128, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_EXTERN_TEMPLATE(160, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_EXTERN_TEMPLATE(200, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_EXTERN_TEMPLATE(512, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_EXTERN_TEMPLATE(768, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_EXTERN_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX2); +// Declare every kernel the library compiles ahead of time. Missing an entry here +// makes a consumer instantiate it locally, at the consumer's own -march. +#define SVS_L2_EXTERN(DIM, LEVEL) SVS_INSTANTIATE_L2(extern template, DIM, LEVEL) +SVS_FOR_EACH_DISPATCH_TARGET(SVS_L2_EXTERN) +#undef SVS_L2_EXTERN #endif } // namespace svs::distance diff --git a/include/svs/core/distance/inner_product.h b/include/svs/core/distance/inner_product.h index 0f7837a53..14293cb38 100644 --- a/include/svs/core/distance/inner_product.h +++ b/include/svs/core/distance/inner_product.h @@ -388,26 +388,12 @@ template struct IPImpl { #if defined(__x86_64__) #include "svs/multi-arch/x86/preprocessor.h" -// TODO: connect with dim_supported_list -DISTANCE_IP_EXTERN_TEMPLATE(64, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_EXTERN_TEMPLATE(96, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_EXTERN_TEMPLATE(100, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_EXTERN_TEMPLATE(128, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_EXTERN_TEMPLATE(160, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_EXTERN_TEMPLATE(200, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_EXTERN_TEMPLATE(512, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_EXTERN_TEMPLATE(768, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_EXTERN_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX512); - -DISTANCE_IP_EXTERN_TEMPLATE(64, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_EXTERN_TEMPLATE(96, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_EXTERN_TEMPLATE(100, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_EXTERN_TEMPLATE(128, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_EXTERN_TEMPLATE(160, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_EXTERN_TEMPLATE(200, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_EXTERN_TEMPLATE(512, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_EXTERN_TEMPLATE(768, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_EXTERN_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX2); + +// Declare every kernel the library compiles ahead of time. Missing an entry here +// makes a consumer instantiate it locally, at the consumer's own -march. +#define SVS_IP_EXTERN(DIM, LEVEL) SVS_INSTANTIATE_IP(extern template, DIM, LEVEL) +SVS_FOR_EACH_DISPATCH_TARGET(SVS_IP_EXTERN) +#undef SVS_IP_EXTERN #endif } // namespace svs::distance diff --git a/include/svs/multi-arch/x86/avx2.cpp b/include/svs/multi-arch/x86/avx2.cpp index bff53ae10..7ee912684 100644 --- a/include/svs/multi-arch/x86/avx2.cpp +++ b/include/svs/multi-arch/x86/avx2.cpp @@ -21,36 +21,11 @@ namespace svs::distance { -// TODO: connect with dim_supported_list -DISTANCE_L2_INSTANTIATE_TEMPLATE(64, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_INSTANTIATE_TEMPLATE(96, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_INSTANTIATE_TEMPLATE(100, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_INSTANTIATE_TEMPLATE(128, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_INSTANTIATE_TEMPLATE(160, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_INSTANTIATE_TEMPLATE(200, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_INSTANTIATE_TEMPLATE(512, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_INSTANTIATE_TEMPLATE(768, AVX_AVAILABILITY::AVX2); -DISTANCE_L2_INSTANTIATE_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX2); - -DISTANCE_IP_INSTANTIATE_TEMPLATE(64, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_INSTANTIATE_TEMPLATE(96, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_INSTANTIATE_TEMPLATE(100, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_INSTANTIATE_TEMPLATE(128, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_INSTANTIATE_TEMPLATE(160, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_INSTANTIATE_TEMPLATE(200, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_INSTANTIATE_TEMPLATE(512, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_INSTANTIATE_TEMPLATE(768, AVX_AVAILABILITY::AVX2); -DISTANCE_IP_INSTANTIATE_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX2); - -DISTANCE_CS_INSTANTIATE_TEMPLATE(64, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_INSTANTIATE_TEMPLATE(96, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_INSTANTIATE_TEMPLATE(100, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_INSTANTIATE_TEMPLATE(128, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_INSTANTIATE_TEMPLATE(160, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_INSTANTIATE_TEMPLATE(200, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_INSTANTIATE_TEMPLATE(512, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_INSTANTIATE_TEMPLATE(768, AVX_AVAILABILITY::AVX2); -DISTANCE_CS_INSTANTIATE_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX2); +// Define every kernel for this ISA level at every generated extent. Extents come +// from cmake/dispatch-surface.cmake, type pairs from multi-arch/x86/preprocessor.h. +#define SVS_DEFINE_FOR_DIM(DIM) SVS_INSTANTIATE_DISTANCES(template, DIM, AVX2) +SVS_FOR_EACH_SUPPORTED_DIM(SVS_DEFINE_FOR_DIM) +#undef SVS_DEFINE_FOR_DIM } // namespace svs::distance diff --git a/include/svs/multi-arch/x86/avx512.cpp b/include/svs/multi-arch/x86/avx512.cpp index bee150d75..c3cd44601 100644 --- a/include/svs/multi-arch/x86/avx512.cpp +++ b/include/svs/multi-arch/x86/avx512.cpp @@ -21,36 +21,11 @@ namespace svs::distance { -// TODO: connect with dim_supported_list -DISTANCE_L2_INSTANTIATE_TEMPLATE(64, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_INSTANTIATE_TEMPLATE(96, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_INSTANTIATE_TEMPLATE(100, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_INSTANTIATE_TEMPLATE(128, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_INSTANTIATE_TEMPLATE(160, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_INSTANTIATE_TEMPLATE(200, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_INSTANTIATE_TEMPLATE(512, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_INSTANTIATE_TEMPLATE(768, AVX_AVAILABILITY::AVX512); -DISTANCE_L2_INSTANTIATE_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX512); - -DISTANCE_IP_INSTANTIATE_TEMPLATE(64, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_INSTANTIATE_TEMPLATE(96, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_INSTANTIATE_TEMPLATE(100, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_INSTANTIATE_TEMPLATE(128, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_INSTANTIATE_TEMPLATE(160, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_INSTANTIATE_TEMPLATE(200, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_INSTANTIATE_TEMPLATE(512, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_INSTANTIATE_TEMPLATE(768, AVX_AVAILABILITY::AVX512); -DISTANCE_IP_INSTANTIATE_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX512); - -DISTANCE_CS_INSTANTIATE_TEMPLATE(64, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_INSTANTIATE_TEMPLATE(96, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_INSTANTIATE_TEMPLATE(100, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_INSTANTIATE_TEMPLATE(128, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_INSTANTIATE_TEMPLATE(160, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_INSTANTIATE_TEMPLATE(200, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_INSTANTIATE_TEMPLATE(512, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_INSTANTIATE_TEMPLATE(768, AVX_AVAILABILITY::AVX512); -DISTANCE_CS_INSTANTIATE_TEMPLATE(Dynamic, AVX_AVAILABILITY::AVX512); +// Define every kernel for this ISA level at every generated extent. Extents come +// from cmake/dispatch-surface.cmake, type pairs from multi-arch/x86/preprocessor.h. +#define SVS_DEFINE_FOR_DIM(DIM) SVS_INSTANTIATE_DISTANCES(template, DIM, AVX512) +SVS_FOR_EACH_SUPPORTED_DIM(SVS_DEFINE_FOR_DIM) +#undef SVS_DEFINE_FOR_DIM } // namespace svs::distance diff --git a/include/svs/multi-arch/x86/preprocessor.h b/include/svs/multi-arch/x86/preprocessor.h index 4e0cb941d..75941b70f 100644 --- a/include/svs/multi-arch/x86/preprocessor.h +++ b/include/svs/multi-arch/x86/preprocessor.h @@ -16,74 +16,72 @@ #pragma once -#define DISTANCE_L2_TEMPLATE_HELPER(SPEC, N, AVX) \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; \ - SPEC struct L2Impl; +#include "svs/core/distance/dispatch_surface.h" -#define DISTANCE_L2_INSTANTIATE_TEMPLATE(N, AVX) \ - DISTANCE_L2_TEMPLATE_HELPER(template, N, AVX); +///// +///// Element-type pairs. +///// +///// Every (query, dataset) combination of float, int8_t, uint8_t and Float16. +///// Each pair costs one instantiation per extent, ISA level and distance. +///// -#define DISTANCE_L2_EXTERN_TEMPLATE(N, AVX) \ - DISTANCE_L2_TEMPLATE_HELPER(extern template, N, AVX); +// Invokes M(query_type, dataset_type, ...) once per type pair. +#define SVS_FOR_EACH_TYPE_PAIR(M, ...) \ + M(float, float, __VA_ARGS__) \ + M(float, int8_t, __VA_ARGS__) \ + M(float, uint8_t, __VA_ARGS__) \ + M(float, svs::float16::Float16, __VA_ARGS__) \ + M(int8_t, float, __VA_ARGS__) \ + M(int8_t, int8_t, __VA_ARGS__) \ + M(int8_t, uint8_t, __VA_ARGS__) \ + M(int8_t, svs::float16::Float16, __VA_ARGS__) \ + M(uint8_t, float, __VA_ARGS__) \ + M(uint8_t, int8_t, __VA_ARGS__) \ + M(uint8_t, uint8_t, __VA_ARGS__) \ + M(uint8_t, svs::float16::Float16, __VA_ARGS__) \ + M(svs::float16::Float16, float, __VA_ARGS__) \ + M(svs::float16::Float16, int8_t, __VA_ARGS__) \ + M(svs::float16::Float16, uint8_t, __VA_ARGS__) \ + M(svs::float16::Float16, svs::float16::Float16, __VA_ARGS__) -#define DISTANCE_IP_TEMPLATE_HELPER(SPEC, N, AVX) \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; \ - SPEC struct IPImpl; +///// +///// Which type pairs each ISA level has kernels for. +///// +///// One line per level in SVS_ISA_LEVELS. A level with no entry here is a +///// compile error rather than a silently empty instantiation list. +///// -#define DISTANCE_IP_INSTANTIATE_TEMPLATE(N, AVX) \ - DISTANCE_IP_TEMPLATE_HELPER(template, N, AVX); +#define SVS_TYPE_PAIRS_NONE SVS_FOR_EACH_TYPE_PAIR +#define SVS_TYPE_PAIRS_AVX2 SVS_FOR_EACH_TYPE_PAIR +#define SVS_TYPE_PAIRS_AVX512 SVS_FOR_EACH_TYPE_PAIR -#define DISTANCE_IP_EXTERN_TEMPLATE(N, AVX) \ - DISTANCE_IP_TEMPLATE_HELPER(extern template, N, AVX); +///// +///// Instantiation. +///// -#define DISTANCE_CS_TEMPLATE_HELPER(SPEC, N, AVX) \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; \ - SPEC struct CosineSimilarityImpl; +// Resolve LEVEL to its type-pair list. The indirection is required so that +// LEVEL is expanded before being pasted. +#define SVS_TYPE_PAIRS_FOR_(LEVEL) SVS_TYPE_PAIRS_##LEVEL +#define SVS_TYPE_PAIRS_FOR(LEVEL, M, ...) SVS_TYPE_PAIRS_FOR_(LEVEL)(M, __VA_ARGS__) -#define DISTANCE_CS_INSTANTIATE_TEMPLATE(N, AVX) \ - DISTANCE_CS_TEMPLATE_HELPER(template, N, AVX); +#define SVS_DECLARE_ONE_L2(Ea, Eb, SPEC, N, LEVEL) \ + SPEC struct L2Impl; +#define SVS_DECLARE_ONE_IP(Ea, Eb, SPEC, N, LEVEL) \ + SPEC struct IPImpl; +#define SVS_DECLARE_ONE_CS(Ea, Eb, SPEC, N, LEVEL) \ + SPEC struct CosineSimilarityImpl; -#define DISTANCE_CS_EXTERN_TEMPLATE(N, AVX) \ - DISTANCE_CS_TEMPLATE_HELPER(extern template, N, AVX); +// SPEC is `template` for a definition or `extern template` for a declaration. +#define SVS_INSTANTIATE_L2(SPEC, N, LEVEL) \ + SVS_TYPE_PAIRS_FOR(LEVEL, SVS_DECLARE_ONE_L2, SPEC, N, LEVEL) +#define SVS_INSTANTIATE_IP(SPEC, N, LEVEL) \ + SVS_TYPE_PAIRS_FOR(LEVEL, SVS_DECLARE_ONE_IP, SPEC, N, LEVEL) +#define SVS_INSTANTIATE_CS(SPEC, N, LEVEL) \ + SVS_TYPE_PAIRS_FOR(LEVEL, SVS_DECLARE_ONE_CS, SPEC, N, LEVEL) + +// All three distances at once. Only usable where all three are declared, i.e. +// in a generated translation unit. +#define SVS_INSTANTIATE_DISTANCES(SPEC, N, LEVEL) \ + SVS_INSTANTIATE_L2(SPEC, N, LEVEL) \ + SVS_INSTANTIATE_IP(SPEC, N, LEVEL) \ + SVS_INSTANTIATE_CS(SPEC, N, LEVEL) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 8c812d35a..bb6d2a7d8 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -237,3 +237,9 @@ list(APPEND CMAKE_MODULE_PATH ${catch2_SOURCE_DIR}/extras) include(CTest) include(Catch) catch_discover_tests(tests ADD_TAGS_AS_LABELS SKIP_IS_FAILURE) + +# Checks the x86 dispatch surface. The target only exists where the multi-arch +# build ran, which is the same condition that makes the surface meaningful. +if(TARGET svs_x86_objects) + add_subdirectory(multi-arch) +endif() diff --git a/tests/multi-arch/CMakeLists.txt b/tests/multi-arch/CMakeLists.txt new file mode 100644 index 000000000..d52c1bc2e --- /dev/null +++ b/tests/multi-arch/CMakeLists.txt @@ -0,0 +1,140 @@ +# Copyright 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +##### +##### Tests of the x86 dispatch surface. +##### +##### Not Catch2 tests: what is being checked is a property of an object file or of +##### a process, so each probe has to be its own translation unit, compiled at a +##### known instruction budget and inspected from outside. +##### +##### Not PROJECT_SOURCE_DIR anywhere below: the downstream repository adds this +##### directory to its own project, where that points somewhere else entirely. +##### + +set(svs_dispatch_cmake_dir "${CMAKE_CURRENT_LIST_DIR}/../../cmake/dispatch-checks") + +# The object libraries exist so that each probe's single object file can be named +# with $, which is what the symbol-table checks read. +# +# svs::x86_options_base is -march=x86-64 -mtune=generic: a probe must know no more +# about the host than an arbitrary consumer of the headers does. +function(svs_add_dispatch_probe name source) + add_library(${name}_objects OBJECT ${source}) + target_link_libraries( + ${name}_objects PRIVATE svs::svs svs::compile_options svs::x86_options_base + ) + add_executable(${name}) + target_link_libraries( + ${name} + PRIVATE ${name}_objects svs::svs svs::compile_options svs::x86_options_base + ) +endfunction() + +# Names every kernel the surface declares and nothing else, so a declared-but- +# uninstantiated kernel is an undefined symbol here. +svs_add_dispatch_probe(dispatch_surface_probe x86/link_probe.cpp) + +# Reaches the kernels through the entry points instead, so it also sees whether the +# entry points route to the whole surface. +svs_add_dispatch_probe(dispatch_entry_probe x86/entry_probe.cpp) + +# Calls every kernel whose ISA level this host satisfies. Which kernels those are +# depends on the host, so this covers the whole surface only across the CI matrix. +add_test(NAME dispatch_surface_probe COMMAND dispatch_surface_probe) +add_test(NAME dispatch_entry_probe COMMAND dispatch_entry_probe) + +if(DEFINED CMAKE_NM AND CMAKE_NM) + set(svs_nm "${CMAKE_NM}") +else() + find_program(svs_nm NAMES nm llvm-nm) +endif() +if(DEFINED CMAKE_OBJDUMP AND CMAKE_OBJDUMP) + set(svs_objdump "${CMAKE_OBJDUMP}") +else() + find_program(svs_objdump NAMES objdump llvm-objdump) +endif() +find_program(svs_gdb NAMES gdb) + +if(svs_nm) + # Host-independent, unlike the runs above: it reads the symbol table rather + # than executing anything, so it sees every kernel in the surface everywhere. + add_test( + NAME dispatch_surface_linkage + COMMAND + "${CMAKE_COMMAND}" + "-DSVS_PROBE_OBJECT=$" + "-DSVS_ARCHIVE=$" + "-DSVS_NM=${svs_nm}" + -P "${svs_dispatch_cmake_dir}/check-dispatch-linkage.cmake" + ) + + # The same symbols, counted from the declaration instead of from a probe built + # out of the same generated header -- the one check a generator bug cannot pass + # by being wrong consistently. + add_test( + NAME dispatch_surface_declaration + COMMAND + "${CMAKE_COMMAND}" + "-DSVS_SURFACE_FILE=${SVS_DISPATCH_SURFACE_FILE}" + "-DSVS_TYPE_PAIR_HEADER=${CMAKE_CURRENT_LIST_DIR}/../../include/svs/multi-arch/x86/preprocessor.h" + "-DSVS_ENUM_HEADER=${CMAKE_CURRENT_LIST_DIR}/../../include/svs/core/distance/distance_core.h" + "-DSVS_ARCHIVE=$" + "-DSVS_CONSUMER_OBJECT=$" + "-DSVS_NM=${svs_nm}" + -P "${svs_dispatch_cmake_dir}/check-dispatch-declaration.cmake" + ) +else() + message(STATUS "nm not found; skipping the dispatch surface symbol-table tests") +endif() + +if(svs_objdump) + # One test per ISA level: a level whose object file carries instructions its + # runtime predicate does not guarantee faults on a host the dispatcher routes + # to it, which no symbol-table check can see. + foreach(tu_spec IN LISTS SVS_DISPATCH_TU_SPECS) + string(REPLACE "|" ";" tu_fields "${tu_spec}") + list(GET tu_fields 1 svs_level) + list(GET tu_fields 2 svs_arch) + list(GET tu_fields 3 svs_infix) + add_test( + NAME dispatch_instructions_${svs_infix} + COMMAND + "${CMAKE_COMMAND}" + "-DSVS_OBJECT=$" + "-DSVS_LEVEL=${svs_level}" + "-DSVS_ARCH=${svs_arch}" + "-DSVS_OBJDUMP=${svs_objdump}" + -P "${svs_dispatch_cmake_dir}/check-dispatch-instructions.cmake" + ) + endforeach() +else() + message(STATUS "objdump not found; skipping the dispatch surface instruction tests") +endif() + +if(svs_nm AND svs_gdb) + # The only check that observes a kernel run rather than exist: a specialization + # lost behind an `#if` still links and still counts. + add_test( + NAME dispatch_surface_execution + COMMAND + "${CMAKE_COMMAND}" + "-DSVS_PROBE=$" + "-DSVS_NM=${svs_nm}" + "-DSVS_GDB=${svs_gdb}" + -P "${svs_dispatch_cmake_dir}/check-dispatch-execution.cmake" + ) +else() + message(STATUS "gdb not found; skipping the dispatch surface execution test") +endif() diff --git a/tests/multi-arch/x86/entry_probe.cpp b/tests/multi-arch/x86/entry_probe.cpp new file mode 100644 index 000000000..567ae6784 --- /dev/null +++ b/tests/multi-arch/x86/entry_probe.cpp @@ -0,0 +1,112 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// A consumer that reaches the kernels the way real code does: through the +// distance entry points, which name every ISA level unconditionally and pick one +// at runtime. Complements link_probe.cpp, which names the kernels directly and so +// checks the surface without depending on the entry points routing to it. + +#include "svs/core/distance/cosine.h" +#include "svs/core/distance/euclidean.h" +#include "svs/core/distance/inner_product.h" +#include "svs/multi-arch/x86/preprocessor.h" + +#include "host_levels.h" + +#include +#include +#include +#include +#include + +namespace { + +// Deliberately not a multiple of any vector width, so the epilogue is exercised. +constexpr size_t entry_dynamic_dim = 97; + +constexpr size_t fixed_extent(bool longest) { + size_t result = longest ? 1 : svs::Dynamic; + for (auto dim : svs::distance::supported_dim_list) { + if (dim == svs::Dynamic) { + continue; + } + result = longest ? std::max(result, dim) : std::min(result, dim); + } + return result; +} + +// The extent the execution check breaks on: the surface's shortest fixed one, so +// the kernel it enters is a fully unrolled specialization rather than the epilogue. +constexpr size_t entry_report_dim = fixed_extent(false); + +constexpr size_t entry_buffer_dim = std::max(fixed_extent(true), entry_dynamic_dim); + +template const E* buffer() { + static const std::array values = []() { + std::array filled{}; + filled.fill(static_cast(1.0F)); + return filled; + }(); + return values.data(); +} + +// A template rather than a macro body: the svs::Dynamic entry points take a +// length, and `if constexpr` only discards the other branch inside a template. +template float entry_one() { + const Ea* a = buffer(); + const Eb* b = buffer(); + if constexpr (N == svs::Dynamic) { + return svs::distance::L2::compute(a, b, entry_dynamic_dim) + + svs::distance::IP::compute(a, b, entry_dynamic_dim) + + svs::distance::CosineSimilarity::compute(a, b, 1.0F, entry_dynamic_dim); + } else { + return svs::distance::L2::compute(a, b) + svs::distance::IP::compute(a, b) + + svs::distance::CosineSimilarity::compute(a, b, 1.0F); + } +} + +#define SVS_ENTRY_ONE(Ea, Eb, N) total += entry_one(); +#define SVS_ENTRY_DIM(N) SVS_FOR_EACH_TYPE_PAIR(SVS_ENTRY_ONE, N) + +float entry_all() { + float total = 0; + SVS_FOR_EACH_SUPPORTED_DIM(SVS_ENTRY_DIM) + return total; +} + +#undef SVS_ENTRY_DIM +#undef SVS_ENTRY_ONE + +} // namespace + +int main(int argc, char** argv) { + if (argc == 2 && std::strcmp(argv[1], "--report") == 0) { + // Read by cmake/dispatch-checks/check-dispatch-execution.cmake, which + // breaks on this extent's kernels to see which level the call enters. + std::printf("expect-level %d\n", svs_test::expected_level()); + std::printf("probe-extent %zu\n", entry_report_dim); + std::printf( + "one-call %f\n", + static_cast(entry_one()) + ); + return 0; + } + + // Printing keeps the calls from being optimized away, and makes the run a + // smoke test of every entry point over the whole surface. + std::printf("dispatch surface entry probe: %f\n", static_cast(entry_all())); + return 0; +} diff --git a/tests/multi-arch/x86/host_levels.h b/tests/multi-arch/x86/host_levels.h new file mode 100644 index 000000000..a94c42973 --- /dev/null +++ b/tests/multi-arch/x86/host_levels.h @@ -0,0 +1,56 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "svs/core/distance/distance_core.h" +#include "svs/lib/avx_detection.h" + +#include + +namespace svs_test { + +using svs::distance::AVX_AVAILABILITY; + +// The runtime predicate behind each ISA level, and the only place the dispatch +// tests state it: a level added to the surface without a specialization here is +// an undefined symbol, deliberately -- the predicate cannot be guessed. +template bool host_satisfies(); + +template <> inline bool host_satisfies() { + return svs::detail::avx_runtime_flags.is_avx2_supported(); +} + +template <> inline bool host_satisfies() { + return svs::detail::avx_runtime_flags.is_avx512f_supported(); +} + +/// The level the distance entry points must choose on this host. +/// +/// The entry points test the strongest level first, so their choice is the +/// highest-numbered satisfied level; AVX_AVAILABILITY::NONE is the fallback. +inline int expected_level() { + int highest = static_cast(AVX_AVAILABILITY::NONE); +#define SVS_HOST_LEVEL_ONE(LEVEL) \ + if (host_satisfies()) { \ + highest = std::max(highest, static_cast(AVX_AVAILABILITY::LEVEL)); \ + } + SVS_FOR_EACH_ISA_LEVEL(SVS_HOST_LEVEL_ONE) +#undef SVS_HOST_LEVEL_ONE + return highest; +} + +} // namespace svs_test diff --git a/tests/multi-arch/x86/link_probe.cpp b/tests/multi-arch/x86/link_probe.cpp new file mode 100644 index 000000000..443c87e62 --- /dev/null +++ b/tests/multi-arch/x86/link_probe.cpp @@ -0,0 +1,110 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// A consumer that names every kernel the dispatch surface declares and nothing +// else, so a declared-but-uninstantiated kernel is an undefined symbol here. + +#include "svs/core/distance/cosine.h" +#include "svs/core/distance/euclidean.h" +#include "svs/core/distance/inner_product.h" +#include "svs/lib/static.h" +#include "svs/multi-arch/x86/preprocessor.h" + +#include "host_levels.h" + +#include +#include +#include +#include + +namespace { + +using svs::distance::AVX_AVAILABILITY; +using svs_test::host_satisfies; + +// The longest fixed extent in the surface. +constexpr size_t probe_max_dim = []() { + size_t longest = 1; + for (auto dim : svs::distance::supported_dim_list) { + if (dim != svs::Dynamic && dim > longest) { + longest = dim; + } + } + return longest; +}(); + +// A length for the svs::Dynamic kernels. Deliberately not a multiple of any +// vector width, so the epilogue is exercised too. +constexpr size_t probe_dynamic_dim = 97; + +// One buffer serves every call, whichever extents the surface happens to declare. +constexpr size_t probe_buffer_dim = std::max(probe_max_dim, probe_dynamic_dim); + +template const E* buffer() { + static const std::array values = []() { + std::array filled{}; + filled.fill(static_cast(1.0F)); + return filled; + }(); + return values.data(); +} + +// svs::lib::MaybeStatic has no default constructor: a dynamic +// extent must be told its length. +template svs::lib::MaybeStatic probe_length() { + if constexpr (N == svs::Dynamic) { + return svs::lib::MaybeStatic(probe_dynamic_dim); + } else { + return svs::lib::MaybeStatic(); + } +} + +// Named directly rather than through L2::compute, which also reaches +// AVX_AVAILABILITY::NONE -- not in the surface, so every consumer instantiates it. +#define SVS_PROBE_ONE(Ea, Eb, N, LEVEL) \ + total += svs::distance::L2Impl::compute( \ + buffer(), buffer(), probe_length() \ + ); \ + total += svs::distance::IPImpl::compute( \ + buffer(), buffer(), probe_length() \ + ); \ + total += \ + svs::distance::CosineSimilarityImpl::compute( \ + buffer(), buffer(), 1.0F, probe_length() \ + ); + +#define SVS_PROBE_TARGET(N, LEVEL) \ + if (host_satisfies()) { \ + SVS_FOR_EACH_TYPE_PAIR(SVS_PROBE_ONE, N, LEVEL) \ + } + +float probe_all() { + float total = 0; + SVS_FOR_EACH_DISPATCH_TARGET(SVS_PROBE_TARGET) + return total; +} + +#undef SVS_PROBE_TARGET +#undef SVS_PROBE_ONE + +} // namespace + +int main() { + // Printing keeps the calls above from being optimized away, and makes the run + // a smoke test of every kernel this host can reach. + std::printf("dispatch surface probe: %f\n", static_cast(probe_all())); + return 0; +}