From 67a69bab30e7a0b00e229de22d86fc878caf924b Mon Sep 17 00:00:00 2001 From: Madan Mohan Manokar Date: Wed, 16 Sep 2026 14:01:53 +0000 Subject: [PATCH 1/2] Add compile-time AVX-512 16-wide ISA with VBMI table gathers. Introduce an astcenc-avx512 product binary (SIMD_WIDTH=16), implement gatherf and vtable lookups with VBMI permutes, and hoist float-table loads at the hot encode sites so the same table stays register-resident. --- CMakeLists.txt | 15 +- Source/CMakeLists.txt | 6 +- Source/UnitTest/CMakeLists.txt | 6 +- Source/UnitTest/cmake_core.cmake | 15 + Source/astcenc_averages_and_directions.cpp | 23 +- .../astcenc_ideal_endpoints_and_weights.cpp | 29 +- Source/astcenc_internal.h | 11 +- Source/astcenc_mathlib.h | 12 +- Source/astcenc_pick_best_endpoint_format.cpp | 13 +- Source/astcenc_vecmathlib.h | 58 +- Source/astcenc_vecmathlib_avx512_16.h | 835 ++++++++++++++++++ Source/astcenccli_entry.cpp | 80 +- Source/astcenccli_toplevel_help.cpp | 4 +- Source/cmake_core.cmake | 35 + 14 files changed, 1107 insertions(+), 35 deletions(-) create mode 100644 Source/astcenc_vecmathlib_avx512_16.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 06de8f33d..da68ff3f6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,6 +35,7 @@ set(CMAKE_XCODE_ATTRIBUTE_VALID_ARCHS "x86_64 x86_64h arm64") include(CTest) +option(ASTCENC_ISA_AVX512 "Enable astcenc builds for AVX-512 SIMD") option(ASTCENC_ISA_AVX2 "Enable astcenc builds for AVX2 SIMD") option(ASTCENC_ISA_SSE41 "Enable astcenc builds for SSE4.1 SIMD") option(ASTCENC_ISA_SSE2 "Enable astcenc builds for SSE2 SIMD") @@ -75,14 +76,25 @@ if("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin") if(${ASTCENC_ISA_NATIVE}) message(FATAL_ERROR "ISA_NATIVE cannot be used in a universal build") endif() + + if(${ASTCENC_ISA_AVX512}) + message(FATAL_ERROR "ISA_AVX512 cannot be used in a universal build") + endif() endif() else() set(ASTCENC_UNIVERSAL_BUILD OFF) endif() +# MSVC /arch:AVX512 does not enable AVX-512VBMI; Clang-CL and GNU do. +if(${ASTCENC_ISA_AVX512} AND CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + message(FATAL_ERROR + "ASTCENC_ISA_AVX512 requires Clang or GCC (AVX-512VBMI). " + "MSVC /arch:AVX512 is not sufficient; configure with -T ClangCL.") +endif() + # Count options which MUST be x64 set(ASTCENC_X64_ISA_COUNT 0) -set(ASTCENC_CONFIGS ${ASTCENC_ISA_AVX2} ${ASTCENC_ISA_SSE41} ${ASTCENC_ISA_SSE2}) +set(ASTCENC_CONFIGS ${ASTCENC_ISA_AVX512} ${ASTCENC_ISA_AVX2} ${ASTCENC_ISA_SSE41} ${ASTCENC_ISA_SSE2}) foreach(ASTCENC_CONFIG ${ASTCENC_CONFIGS}) if(${ASTCENC_CONFIG}) math(EXPR ASTCENC_X64_ISA_COUNT "${ASTCENC_X64_ISA_COUNT} + 1") @@ -127,6 +139,7 @@ printopt("SVE 256b backend " ${ASTCENC_ISA_SVE_256}) printopt("SVE 128b backend " ${ASTCENC_ISA_SVE_128}) printopt("NEON backend " ${ASTCENC_ISA_NEON}) message(STATUS "x86-64 backend options") +printopt("AVX-512 backend " ${ASTCENC_ISA_AVX512}) printopt("AVX2 backend " ${ASTCENC_ISA_AVX2}) printopt("SSE4.1 backend " ${ASTCENC_ISA_SSE41}) printopt("SSE2 backend " ${ASTCENC_ISA_SSE2}) diff --git a/Source/CMakeLists.txt b/Source/CMakeLists.txt index 89a1d0056..0636d6a3a 100644 --- a/Source/CMakeLists.txt +++ b/Source/CMakeLists.txt @@ -31,8 +31,8 @@ else() set(ASTCENC_CODEC enc) endif() -set(ASTCENC_ARTIFACTS native none sve_256 sve_128 neon avx2 sse4.1 sse2) -set(ASTCENC_CONFIGS ${ASTCENC_ISA_NATIVE} ${ASTCENC_ISA_NONE} ${ASTCENC_ISA_SVE_256} ${ASTCENC_ISA_SVE_128} ${ASTCENC_ISA_NEON} ${ASTCENC_ISA_AVX2} ${ASTCENC_ISA_SSE41} ${ASTCENC_ISA_SSE2}) +set(ASTCENC_ARTIFACTS native none sve_256 sve_128 neon avx512 avx2 sse4.1 sse2) +set(ASTCENC_CONFIGS ${ASTCENC_ISA_NATIVE} ${ASTCENC_ISA_NONE} ${ASTCENC_ISA_SVE_256} ${ASTCENC_ISA_SVE_128} ${ASTCENC_ISA_NEON} ${ASTCENC_ISA_AVX512} ${ASTCENC_ISA_AVX2} ${ASTCENC_ISA_SSE41} ${ASTCENC_ISA_SSE2}) list(LENGTH ASTCENC_ARTIFACTS ASTCENC_ARTIFACTS_LEN) math(EXPR ASTCENC_ARTIFACTS_LEN "${ASTCENC_ARTIFACTS_LEN} - 1") @@ -52,6 +52,8 @@ foreach(INDEX RANGE ${ASTCENC_ARTIFACTS_LEN}) set(CMAKE_OSX_ARCHITECTURES x86_64) elseif(${ASTCENC_ISA_SIMD} MATCHES "sse4.1") set(CMAKE_OSX_ARCHITECTURES x86_64) + elseif(${ASTCENC_ISA_SIMD} MATCHES "avx512") + set(CMAKE_OSX_ARCHITECTURES x86_64h) elseif(${ASTCENC_ISA_SIMD} MATCHES "avx2") set(CMAKE_OSX_ARCHITECTURES x86_64h) elseif(${ASTCENC_ISA_SIMD} MATCHES "none") diff --git a/Source/UnitTest/CMakeLists.txt b/Source/UnitTest/CMakeLists.txt index 9e4135675..e5a33d6e8 100644 --- a/Source/UnitTest/CMakeLists.txt +++ b/Source/UnitTest/CMakeLists.txt @@ -15,8 +15,8 @@ # under the License. # ---------------------------------------------------------------------------- -set(ASTCENC_ARTIFACTS native none sve_256 sve_128 neon avx2 sse4.1 sse2) -set(ASTCENC_CONFIGS ${ASTCENC_ISA_NATIVE} ${ASTCENC_ISA_NONE} ${ASTCENC_ISA_SVE_256} ${ASTCENC_ISA_SVE_128} ${ASTCENC_ISA_NEON} ${ASTCENC_ISA_AVX2} ${ASTCENC_ISA_SSE41} ${ASTCENC_ISA_SSE2}) +set(ASTCENC_ARTIFACTS native none sve_256 sve_128 neon avx512 avx2 sse4.1 sse2) +set(ASTCENC_CONFIGS ${ASTCENC_ISA_NATIVE} ${ASTCENC_ISA_NONE} ${ASTCENC_ISA_SVE_256} ${ASTCENC_ISA_SVE_128} ${ASTCENC_ISA_NEON} ${ASTCENC_ISA_AVX512} ${ASTCENC_ISA_AVX2} ${ASTCENC_ISA_SSE41} ${ASTCENC_ISA_SSE2}) list(LENGTH ASTCENC_ARTIFACTS ASTCENC_ARTIFACTS_LEN) math(EXPR ASTCENC_ARTIFACTS_LEN "${ASTCENC_ARTIFACTS_LEN} - 1") @@ -36,6 +36,8 @@ foreach(INDEX RANGE ${ASTCENC_ARTIFACTS_LEN}) set(CMAKE_OSX_ARCHITECTURES x86_64) elseif(${ASTCENC_ISA_SIMD} MATCHES "sse4.1") set(CMAKE_OSX_ARCHITECTURES x86_64) + elseif(${ASTCENC_ISA_SIMD} MATCHES "avx512") + set(CMAKE_OSX_ARCHITECTURES x86_64h) elseif(${ASTCENC_ISA_SIMD} MATCHES "avx2") set(CMAKE_OSX_ARCHITECTURES x86_64h) elseif(${ASTCENC_ISA_SIMD} MATCHES "none") diff --git a/Source/UnitTest/cmake_core.cmake b/Source/UnitTest/cmake_core.cmake index 7ccc51d44..8c19a500f 100644 --- a/Source/UnitTest/cmake_core.cmake +++ b/Source/UnitTest/cmake_core.cmake @@ -174,6 +174,21 @@ elseif(${ASTCENC_ISA_SIMD} MATCHES "sse4.1") PRIVATE $<$>:-msse4.1 -mpopcnt>) +elseif(${ASTCENC_ISA_SIMD} MATCHES "avx512") + target_compile_definitions(${ASTCENC_TEST} + PRIVATE + ASTCENC_NEON=0 + ASTCENC_SVE=0 + ASTCENC_SSE=41 + ASTCENC_AVX=3 + ASTCENC_POPCNT=1 + ASTCENC_F16C=1) + + target_compile_options(${ASTCENC_TEST} + PRIVATE + $<$>:-mavx512f -mavx512bw -mavx512dq -mavx512vl -mavx512vbmi -mpopcnt -mf16c> + $<$:/arch:AVX512>) + elseif(${ASTCENC_ISA_SIMD} MATCHES "avx2") target_compile_definitions(${ASTCENC_TEST} PRIVATE diff --git a/Source/astcenc_averages_and_directions.cpp b/Source/astcenc_averages_and_directions.cpp index d1dea24ec..141eec385 100644 --- a/Source/astcenc_averages_and_directions.cpp +++ b/Source/astcenc_averages_and_directions.cpp @@ -735,6 +735,11 @@ void compute_error_squared_rgba( vfloatacc uncor_errorsumv = vfloatacc::zero(); vfloatacc samec_errorsumv = vfloatacc::zero(); + vgatherf_table tr = vgatherf_load(blk.data_r, blk.texel_count); + vgatherf_table tg = vgatherf_load(blk.data_g, blk.texel_count); + vgatherf_table tb = vgatherf_load(blk.data_b, blk.texel_count); + vgatherf_table ta = vgatherf_load(blk.data_a, blk.texel_count); + for (size_t partition = 0; partition < partition_count; partition++) { const uint8_t *texel_indexes = pi.texels_of_partition[partition]; @@ -780,10 +785,10 @@ void compute_error_squared_rgba( vmask mask = lane_ids < vint_from_size(texel_count); const uint8_t* texel_idxs = texel_indexes + i; - vfloat data_r = gatherf_byte_inds(blk.data_r, texel_idxs); - vfloat data_g = gatherf_byte_inds(blk.data_g, texel_idxs); - vfloat data_b = gatherf_byte_inds(blk.data_b, texel_idxs); - vfloat data_a = gatherf_byte_inds(blk.data_a, texel_idxs); + vfloat data_r = gatherf(tr, texel_idxs); + vfloat data_g = gatherf(tg, texel_idxs); + vfloat data_b = gatherf(tb, texel_idxs); + vfloat data_a = gatherf(ta, texel_idxs); vfloat uncor_param = (data_r * l_uncor_bs0) + (data_g * l_uncor_bs1) @@ -853,6 +858,10 @@ void compute_error_squared_rgb( vfloatacc uncor_errorsumv = vfloatacc::zero(); vfloatacc samec_errorsumv = vfloatacc::zero(); + vgatherf_table tr = vgatherf_load(blk.data_r, blk.texel_count); + vgatherf_table tg = vgatherf_load(blk.data_g, blk.texel_count); + vgatherf_table tb = vgatherf_load(blk.data_b, blk.texel_count); + for (size_t partition = 0; partition < partition_count; partition++) { partition_lines3& pl = plines[partition]; @@ -894,9 +903,9 @@ void compute_error_squared_rgb( vmask mask = lane_ids < vint_from_size(texel_count); const uint8_t* texel_idxs = texel_indexes + i; - vfloat data_r = gatherf_byte_inds(blk.data_r, texel_idxs); - vfloat data_g = gatherf_byte_inds(blk.data_g, texel_idxs); - vfloat data_b = gatherf_byte_inds(blk.data_b, texel_idxs); + vfloat data_r = gatherf(tr, texel_idxs); + vfloat data_g = gatherf(tg, texel_idxs); + vfloat data_b = gatherf(tb, texel_idxs); vfloat uncor_param = (data_r * l_uncor_bs0) + (data_g * l_uncor_bs1) diff --git a/Source/astcenc_ideal_endpoints_and_weights.cpp b/Source/astcenc_ideal_endpoints_and_weights.cpp index bd2e4ba28..07baab954 100644 --- a/Source/astcenc_ideal_endpoints_and_weights.cpp +++ b/Source/astcenc_ideal_endpoints_and_weights.cpp @@ -47,10 +47,11 @@ static vfloat bilinear_infill_vla( const uint8_t* weight_idx3 = di.texel_weights_tr[3] + index; // Load the bilinear filter weights from the decimated grid - vfloat weight_val0 = gatherf_byte_inds(weights, weight_idx0); - vfloat weight_val1 = gatherf_byte_inds(weights, weight_idx1); - vfloat weight_val2 = gatherf_byte_inds(weights, weight_idx2); - vfloat weight_val3 = gatherf_byte_inds(weights, weight_idx3); + vgatherf_table wt = vgatherf_load(weights, di.weight_count); + vfloat weight_val0 = gatherf(wt, weight_idx0); + vfloat weight_val1 = gatherf(wt, weight_idx1); + vfloat weight_val2 = gatherf(wt, weight_idx2); + vfloat weight_val3 = gatherf(wt, weight_idx3); // Load the weight contribution factors for each decimated weight vfloat tex_weight_float0 = loada(di.texel_weight_contribs_float_tr[0] + index); @@ -85,8 +86,9 @@ static vfloat bilinear_infill_vla_2( const uint8_t* weight_idx1 = di.texel_weights_tr[1] + index; // Load the bilinear filter weights from the decimated grid - vfloat weight_val0 = gatherf_byte_inds(weights, weight_idx0); - vfloat weight_val1 = gatherf_byte_inds(weights, weight_idx1); + vgatherf_table wt = vgatherf_load(weights, di.weight_count); + vfloat weight_val0 = gatherf(wt, weight_idx0); + vfloat weight_val1 = gatherf(wt, weight_idx1); // Load the weight contribution factors for each decimated weight vfloat tex_weight_float0 = loada(di.texel_weight_contribs_float_tr[0] + index); @@ -871,6 +873,9 @@ void compute_ideal_weights_for_decimation( // Compute an initial average for each decimated weight bool constant_wes = ei.is_constant_weight_error_scale; vfloat weight_error_scale(ei.weight_error_scale[0]); + vgatherf_table wtab = vgatherf_load(ei.weights, texel_count); + vgatherf_table etab = constant_wes ? wtab + : vgatherf_load(ei.weight_error_scale, texel_count); // This overshoots - this is OK as we initialize the array tails in the // decimation table structures to safe values ... @@ -892,13 +897,13 @@ void compute_ideal_weights_for_decimation( if (!constant_wes) { - weight_error_scale = gatherf_byte_inds(ei.weight_error_scale, texel); + weight_error_scale = gatherf(etab, texel); } vfloat contrib_weight = weight * weight_error_scale; weight_weight += contrib_weight; - initial_weight += gatherf_byte_inds(ei.weights, texel) * contrib_weight; + initial_weight += gatherf(wtab, texel) * contrib_weight; } storea(initial_weight / weight_weight, dec_weight_ideal_value + i); @@ -930,6 +935,8 @@ void compute_ideal_weights_for_decimation( constexpr float stepsize = 0.25f; constexpr float chd_scale = -WEIGHTS_TEXEL_SUM; + vgatherf_table itab = vgatherf_load(infilled_weights, texel_count); + for (unsigned int i = 0; i < weight_count; i += ASTCENC_SIMD_WIDTH) { vfloat weight_val = loada(dec_weight_ideal_value + i); @@ -951,12 +958,12 @@ void compute_ideal_weights_for_decimation( if (!constant_wes) { - weight_error_scale = gatherf_byte_inds(ei.weight_error_scale, texel); + weight_error_scale = gatherf(etab, texel); } vfloat scale = weight_error_scale * contrib_weight; - vfloat old_weight = gatherf_byte_inds(infilled_weights, texel); - vfloat ideal_weight = gatherf_byte_inds(ei.weights, texel); + vfloat old_weight = gatherf(itab, texel); + vfloat ideal_weight = gatherf(wtab, texel); error_change0 += contrib_weight * scale; error_change1 += (old_weight - ideal_weight) * scale; diff --git a/Source/astcenc_internal.h b/Source/astcenc_internal.h index 46942171b..1298dafd6 100644 --- a/Source/astcenc_internal.h +++ b/Source/astcenc_internal.h @@ -66,7 +66,12 @@ Constants ============================================================================ */ #if !defined(ASTCENC_BLOCK_MAX_TEXELS) - #define ASTCENC_BLOCK_MAX_TEXELS 216 // A 3D 6x6x6 block + // 6x6x6 = 216 live texels. SIMD_WIDTH 16 needs a multiple of 16, so pad to 224. + #if ASTCENC_AVX >= 3 + #define ASTCENC_BLOCK_MAX_TEXELS 224 + #else + #define ASTCENC_BLOCK_MAX_TEXELS 216 + #endif #endif /** @brief The maximum number of texels a block can support (6x6x6 block). */ @@ -157,8 +162,8 @@ static constexpr unsigned int TUNE_MAX_ANGULAR_QUANT { 7 }; /* QUANT_12 */ static_assert((BLOCK_MAX_TEXELS % ASTCENC_SIMD_WIDTH) == 0, "BLOCK_MAX_TEXELS must be multiple of ASTCENC_SIMD_WIDTH"); -static_assert(BLOCK_MAX_TEXELS <= 216, - "BLOCK_MAX_TEXELS must not be greater than 216"); +static_assert(BLOCK_MAX_TEXELS <= 224, + "BLOCK_MAX_TEXELS must not be greater than 224"); static_assert((BLOCK_MAX_WEIGHTS % ASTCENC_SIMD_WIDTH) == 0, "BLOCK_MAX_WEIGHTS must be multiple of ASTCENC_SIMD_WIDTH"); diff --git a/Source/astcenc_mathlib.h b/Source/astcenc_mathlib.h index 44f153d29..a87cc1db8 100644 --- a/Source/astcenc_mathlib.h +++ b/Source/astcenc_mathlib.h @@ -57,7 +57,13 @@ #endif #ifndef ASTCENC_AVX - #if defined(__AVX2__) + // AVX-512 backend needs VBMI (vpermb / vpermt2b). AVX-512F-only CPUs + // (Skylake-X, Cascade Lake) must stay on the AVX2 8-wide path. + // F+VBMI selects this 16-wide backend. + #if defined(__AVX512F__) && defined(__AVX512VBMI__) + #define ASTCENC_AVX 3 + #define ASTCENC_X86_GATHERS 1 + #elif defined(__AVX2__) #define ASTCENC_AVX 2 #define ASTCENC_X86_GATHERS 1 #elif defined(__AVX__) @@ -92,7 +98,9 @@ #endif // Force vector-sized SIMD alignment -#if ASTCENC_AVX || ASTCENC_SVE == 8 +#if ASTCENC_AVX >= 3 + #define ASTCENC_VECALIGN 64 +#elif ASTCENC_AVX || ASTCENC_SVE == 8 #define ASTCENC_VECALIGN 32 #elif ASTCENC_SSE || ASTCENC_NEON || ASTCENC_SVE == 4 #define ASTCENC_VECALIGN 16 diff --git a/Source/astcenc_pick_best_endpoint_format.cpp b/Source/astcenc_pick_best_endpoint_format.cpp index 485785def..eb2d468c3 100644 --- a/Source/astcenc_pick_best_endpoint_format.cpp +++ b/Source/astcenc_pick_best_endpoint_format.cpp @@ -120,6 +120,11 @@ static void compute_error_squared_rgb_single_partition( vfloat l_bs1(l_pline.bs.lane<1>()); vfloat l_bs2(l_pline.bs.lane<2>()); + vgatherf_table tr = vgatherf_load(blk.data_r, blk.texel_count); + vgatherf_table tg = vgatherf_load(blk.data_g, blk.texel_count); + vgatherf_table tb = vgatherf_load(blk.data_b, blk.texel_count); + vgatherf_table ta = vgatherf_load(blk.data_a, blk.texel_count); + vint lane_ids = vint::lane_id(); for (unsigned int i = 0; i < texel_count; i += ASTCENC_SIMD_WIDTH) { @@ -129,15 +134,15 @@ static void compute_error_squared_rgb_single_partition( lane_ids += vint(ASTCENC_SIMD_WIDTH); // Compute the error that arises from just ditching alpha - vfloat data_a = gatherf_byte_inds(blk.data_a, tix); + vfloat data_a = gatherf(ta, tix); vfloat alpha_diff = data_a - default_a; alpha_diff = alpha_diff * alpha_diff; haccumulate(a_drop_errv, alpha_diff, mask); - vfloat data_r = gatherf_byte_inds(blk.data_r, tix); - vfloat data_g = gatherf_byte_inds(blk.data_g, tix); - vfloat data_b = gatherf_byte_inds(blk.data_b, tix); + vfloat data_r = gatherf(tr, tix); + vfloat data_g = gatherf(tg, tix); + vfloat data_b = gatherf(tb, tix); // Compute uncorrelated error vfloat param = data_r * uncor_bs0 diff --git a/Source/astcenc_vecmathlib.h b/Source/astcenc_vecmathlib.h index 5b95647d1..3cf62d320 100644 --- a/Source/astcenc_vecmathlib.h +++ b/Source/astcenc_vecmathlib.h @@ -49,6 +49,7 @@ * * 4-wide for x86-64 SSE4.1 * * 8-wide for Armv8-A SVE * * 8-wide for x86-64 AVX2 + * * 16-wide for x86-64 AVX-512 */ #ifndef ASTC_VECMATHLIB_H_INCLUDED @@ -80,7 +81,34 @@ template T gatherf_byte_inds(const float* base, const uint8_t* indices); -#if ASTCENC_AVX >= 2 +#if ASTCENC_AVX >= 3 + // Compile-time AVX-512: expose 16-wide VLA. + #include "astcenc_vecmathlib_sse_4.h" + #include "astcenc_vecmathlib_common_4.h" + #include "astcenc_vecmathlib_avx512_16.h" + + #define ASTCENC_SIMD_WIDTH 16 + + using vfloat = vfloat16; + + #if defined(ASTCENC_NO_INVARIANCE) + using vfloatacc = vfloat16; + #else + using vfloatacc = vfloat4; + #endif + + using vint = vint16; + using vmask = vmask16; + + using vtable_16x8 = vtable16_16x8; + using vtable_32x8 = vtable16_32x8; + using vtable_64x8 = vtable16_64x8; + + constexpr auto loada = vfloat16::loada; + constexpr auto load1 = vfloat16::load1; + constexpr auto vint_from_size = vint16_from_size; + +#elif ASTCENC_AVX >= 2 // If we have AVX2 expose 8-wide VLA. #include "astcenc_vecmathlib_sse_4.h" #include "astcenc_vecmathlib_common_4.h" @@ -240,6 +268,34 @@ template T gatherf_byte_inds(const float* base, const uint8_t* indic constexpr auto vint_from_size = vint4_from_size; #endif +#ifndef ASTCENC_HAS_VGATHERF_TABLE +/** + * @brief Portable float-table gather: keep the pointer, gather from memory. + * + * AVX-512 replaces this with a register-resident VBMI table in + * astcenc_vecmathlib_avx512_16.h. + */ +struct vgatherf_table { + const float* base; +}; + +ASTCENC_SIMD_INLINE vgatherf_table vgatherf_load(const float* base, unsigned int count) +{ + (void)count; + return vgatherf_table{base}; +} + +ASTCENC_SIMD_INLINE vfloat gatherf(const vgatherf_table& t, vint indices) +{ + return gatherf(t.base, indices); +} + +ASTCENC_SIMD_INLINE vfloat gatherf(const vgatherf_table& t, const uint8_t* indices) +{ + return gatherf_byte_inds(t.base, indices); +} +#endif + /** * @brief Round a count down to the largest multiple of the SIMD width. * diff --git a/Source/astcenc_vecmathlib_avx512_16.h b/Source/astcenc_vecmathlib_avx512_16.h new file mode 100644 index 000000000..4cf6e4b93 --- /dev/null +++ b/Source/astcenc_vecmathlib_avx512_16.h @@ -0,0 +1,835 @@ +// SPDX-License-Identifier: Apache-2.0 +// ---------------------------------------------------------------------------- +// Copyright 2019-2026 Arm Limited +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy +// of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. +// ---------------------------------------------------------------------------- + +/** + * @brief 16x32-bit vectors, implemented using AVX-512F. + * + * Experimental compile-time AVX-512 ISA: ASTCENC_SIMD_WIDTH is 16. This is a + * VLA backend sibling of astcenc_vecmathlib_avx2_8.h, not a runtime overlay. + * gatherf is implemented with AVX-512 VBMI (vpermb / vpermt2b) for tables that + * fit in 1–4 zmm, and a scalar-load fallback for larger index ranges. It does + * not use AVX-512F vgatherdps. vgatherf_load hoists those masked zmm loads so + * callers can permute many times from one table. vtable_lookup_32bit uses + * vpermb over packed byte tables instead of AVX2-style lane-local pshufb. + */ + +#ifndef ASTC_VECMATHLIB_AVX512_16_H_INCLUDED +#define ASTC_VECMATHLIB_AVX512_16_H_INCLUDED + +#ifndef ASTCENC_SIMD_INLINE + #error "Include astcenc_vecmathlib.h, do not include directly" +#endif + +#include +#include +#include + +// ============================================================================ +// vfloat16 data type +// ============================================================================ + +struct vfloat16 +{ + ASTCENC_SIMD_INLINE vfloat16() = default; + + ASTCENC_SIMD_INLINE explicit vfloat16(const float *p) + { + m = _mm512_loadu_ps(p); + } + + ASTCENC_SIMD_INLINE explicit vfloat16(float a) + { + m = _mm512_set1_ps(a); + } + + ASTCENC_SIMD_INLINE explicit vfloat16(__m512 a) + { + m = a; + } + + static ASTCENC_SIMD_INLINE vfloat16 zero() + { + return vfloat16(_mm512_setzero_ps()); + } + + static ASTCENC_SIMD_INLINE vfloat16 load1(const float* p) + { + return vfloat16(_mm512_set1_ps(*p)); + } + + static ASTCENC_SIMD_INLINE vfloat16 loada(const float* p) + { + return vfloat16(_mm512_load_ps(p)); + } + + __m512 m; +}; + +// ============================================================================ +// vint16 data type +// ============================================================================ + +struct vint16 +{ + ASTCENC_SIMD_INLINE vint16() = default; + + ASTCENC_SIMD_INLINE explicit vint16(const int *p) + { + m = _mm512_loadu_si512(p); + } + + ASTCENC_SIMD_INLINE explicit vint16(const uint8_t *p) + { + __m128i bytes; + std::memcpy(&bytes, p, sizeof(bytes)); + m = _mm512_cvtepu8_epi32(bytes); + } + + ASTCENC_SIMD_INLINE explicit vint16(int a) + { + m = _mm512_set1_epi32(a); + } + + ASTCENC_SIMD_INLINE explicit vint16(__m512i a) + { + m = a; + } + + static ASTCENC_SIMD_INLINE vint16 zero() + { + return vint16(_mm512_setzero_si512()); + } + + static ASTCENC_SIMD_INLINE vint16 load1(const int* p) + { + return vint16(_mm512_set1_epi32(*p)); + } + + static ASTCENC_SIMD_INLINE vint16 load(const uint8_t* p) + { + return vint16(_mm512_loadu_si512(p)); + } + + static ASTCENC_SIMD_INLINE vint16 loada(const int* p) + { + return vint16(_mm512_load_si512(p)); + } + + static ASTCENC_SIMD_INLINE vint16 lane_id() + { + return vint16(_mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15)); + } + + __m512i m; +}; + +// ============================================================================ +// vmask16 data type (AVX-512 k-mask) +// ============================================================================ + +struct vmask16 +{ + ASTCENC_SIMD_INLINE explicit vmask16(__mmask16 a) + { + m = a; + } + + ASTCENC_SIMD_INLINE explicit vmask16(bool a) + { + m = a ? static_cast<__mmask16>(0xFFFF) : static_cast<__mmask16>(0); + } + + __mmask16 m; +}; + +// ============================================================================ +// vmask16 operators and functions +// ============================================================================ + +ASTCENC_SIMD_INLINE vmask16 operator|(vmask16 a, vmask16 b) +{ + return vmask16(static_cast<__mmask16>(a.m | b.m)); +} + +ASTCENC_SIMD_INLINE vmask16 operator&(vmask16 a, vmask16 b) +{ + return vmask16(static_cast<__mmask16>(a.m & b.m)); +} + +ASTCENC_SIMD_INLINE vmask16 operator^(vmask16 a, vmask16 b) +{ + return vmask16(static_cast<__mmask16>(a.m ^ b.m)); +} + +ASTCENC_SIMD_INLINE vmask16 operator~(vmask16 a) +{ + return vmask16(static_cast<__mmask16>(a.m ^ 0xFFFF)); +} + +ASTCENC_SIMD_INLINE unsigned int mask(vmask16 a) +{ + return static_cast(a.m); +} + +ASTCENC_SIMD_INLINE bool any(vmask16 a) +{ + return a.m != 0; +} + +ASTCENC_SIMD_INLINE bool all(vmask16 a) +{ + return a.m == 0xFFFF; +} + +// ============================================================================ +// vint16 operators and functions +// ============================================================================ + +ASTCENC_SIMD_INLINE vint16 operator+(vint16 a, vint16 b) +{ + return vint16(_mm512_add_epi32(a.m, b.m)); +} + +ASTCENC_SIMD_INLINE vint16& operator+=(vint16& a, const vint16& b) +{ + a = a + b; + return a; +} + +ASTCENC_SIMD_INLINE vint16 operator-(vint16 a, vint16 b) +{ + return vint16(_mm512_sub_epi32(a.m, b.m)); +} + +ASTCENC_SIMD_INLINE vint16 operator*(vint16 a, vint16 b) +{ + return vint16(_mm512_mullo_epi32(a.m, b.m)); +} + +ASTCENC_SIMD_INLINE vint16 operator~(vint16 a) +{ + return vint16(_mm512_xor_si512(a.m, _mm512_set1_epi32(-1))); +} + +ASTCENC_SIMD_INLINE vint16 operator|(vint16 a, vint16 b) +{ + return vint16(_mm512_or_si512(a.m, b.m)); +} + +ASTCENC_SIMD_INLINE vint16 operator&(vint16 a, vint16 b) +{ + return vint16(_mm512_and_si512(a.m, b.m)); +} + +ASTCENC_SIMD_INLINE vint16 operator^(vint16 a, vint16 b) +{ + return vint16(_mm512_xor_si512(a.m, b.m)); +} + +ASTCENC_SIMD_INLINE vmask16 operator==(vint16 a, vint16 b) +{ + return vmask16(_mm512_cmpeq_epi32_mask(a.m, b.m)); +} + +ASTCENC_SIMD_INLINE vmask16 operator!=(vint16 a, vint16 b) +{ + return vmask16(_mm512_cmpneq_epi32_mask(a.m, b.m)); +} + +ASTCENC_SIMD_INLINE vmask16 operator<(vint16 a, vint16 b) +{ + return vmask16(_mm512_cmplt_epi32_mask(a.m, b.m)); +} + +ASTCENC_SIMD_INLINE vmask16 operator>(vint16 a, vint16 b) +{ + return vmask16(_mm512_cmpgt_epi32_mask(a.m, b.m)); +} + +template ASTCENC_SIMD_INLINE vint16 lsl(vint16 a) +{ + return vint16(_mm512_slli_epi32(a.m, s)); +} + +template ASTCENC_SIMD_INLINE vint16 asr(vint16 a) +{ + return vint16(_mm512_srai_epi32(a.m, s)); +} + +template ASTCENC_SIMD_INLINE vint16 lsr(vint16 a) +{ + return vint16(_mm512_srli_epi32(a.m, s)); +} + +ASTCENC_SIMD_INLINE vint16 min(vint16 a, vint16 b) +{ + return vint16(_mm512_min_epi32(a.m, b.m)); +} + +ASTCENC_SIMD_INLINE vint16 max(vint16 a, vint16 b) +{ + return vint16(_mm512_max_epi32(a.m, b.m)); +} + +ASTCENC_SIMD_INLINE int hmin_s(vint16 a) +{ + return _mm512_reduce_min_epi32(a.m); +} + +ASTCENC_SIMD_INLINE vint16 hmin(vint16 a) +{ + return vint16(hmin_s(a)); +} + +ASTCENC_SIMD_INLINE int hmax_s(vint16 a) +{ + return _mm512_reduce_max_epi32(a.m); +} + +ASTCENC_SIMD_INLINE vint16 hmax(vint16 a) +{ + return vint16(hmax_s(a)); +} + +ASTCENC_SIMD_INLINE vint16 vint16_from_size(size_t a) +{ + assert(a <= std::numeric_limits::max()); + return vint16(static_cast(a)); +} + +ASTCENC_SIMD_INLINE void storea(vint16 a, int* p) +{ + _mm512_store_si512(p, a.m); +} + +ASTCENC_SIMD_INLINE void store(vint16 a, int* p) +{ + _mm512_storeu_si512(p, a.m); +} + +ASTCENC_SIMD_INLINE void store_nbytes(vint16 a, uint8_t* p) +{ + _mm_storeu_si128(reinterpret_cast<__m128i*>(p), + _mm512_extracti32x4_epi32(a.m, 0)); +} + +ASTCENC_SIMD_INLINE void pack_and_store_low_bytes(vint16 v, uint8_t* p) +{ + _mm_storeu_si128(reinterpret_cast<__m128i*>(p), _mm512_cvtepi32_epi8(v.m)); +} + +ASTCENC_SIMD_INLINE vint16 select(vint16 a, vint16 b, vmask16 cond) +{ + return vint16(_mm512_mask_blend_epi32(cond.m, a.m, b.m)); +} + +// ============================================================================ +// vfloat16 operators and functions +// ============================================================================ + +ASTCENC_SIMD_INLINE vfloat16 operator+(vfloat16 a, vfloat16 b) +{ + return vfloat16(_mm512_add_ps(a.m, b.m)); +} + +ASTCENC_SIMD_INLINE vfloat16& operator+=(vfloat16& a, const vfloat16& b) +{ + a = a + b; + return a; +} + +ASTCENC_SIMD_INLINE vfloat16 operator-(vfloat16 a, vfloat16 b) +{ + return vfloat16(_mm512_sub_ps(a.m, b.m)); +} + +ASTCENC_SIMD_INLINE vfloat16 operator*(vfloat16 a, vfloat16 b) +{ + return vfloat16(_mm512_mul_ps(a.m, b.m)); +} + +ASTCENC_SIMD_INLINE vfloat16 operator*(vfloat16 a, float b) +{ + return vfloat16(_mm512_mul_ps(a.m, _mm512_set1_ps(b))); +} + +ASTCENC_SIMD_INLINE vfloat16 operator*(float a, vfloat16 b) +{ + return vfloat16(_mm512_mul_ps(_mm512_set1_ps(a), b.m)); +} + +ASTCENC_SIMD_INLINE vfloat16 operator/(vfloat16 a, vfloat16 b) +{ + return vfloat16(_mm512_div_ps(a.m, b.m)); +} + +ASTCENC_SIMD_INLINE vfloat16 operator/(vfloat16 a, float b) +{ + return vfloat16(_mm512_div_ps(a.m, _mm512_set1_ps(b))); +} + +ASTCENC_SIMD_INLINE vfloat16 operator/(float a, vfloat16 b) +{ + return vfloat16(_mm512_div_ps(_mm512_set1_ps(a), b.m)); +} + +ASTCENC_SIMD_INLINE vmask16 operator==(vfloat16 a, vfloat16 b) +{ + return vmask16(_mm512_cmp_ps_mask(a.m, b.m, _CMP_EQ_OQ)); +} + +ASTCENC_SIMD_INLINE vmask16 operator!=(vfloat16 a, vfloat16 b) +{ + return vmask16(_mm512_cmp_ps_mask(a.m, b.m, _CMP_NEQ_UQ)); +} + +ASTCENC_SIMD_INLINE vmask16 operator<(vfloat16 a, vfloat16 b) +{ + return vmask16(_mm512_cmp_ps_mask(a.m, b.m, _CMP_LT_OQ)); +} + +ASTCENC_SIMD_INLINE vmask16 operator>(vfloat16 a, vfloat16 b) +{ + return vmask16(_mm512_cmp_ps_mask(a.m, b.m, _CMP_GT_OQ)); +} + +ASTCENC_SIMD_INLINE vmask16 operator<=(vfloat16 a, vfloat16 b) +{ + return vmask16(_mm512_cmp_ps_mask(a.m, b.m, _CMP_LE_OQ)); +} + +ASTCENC_SIMD_INLINE vmask16 operator>=(vfloat16 a, vfloat16 b) +{ + return vmask16(_mm512_cmp_ps_mask(a.m, b.m, _CMP_GE_OQ)); +} + +ASTCENC_SIMD_INLINE vfloat16 min(vfloat16 a, vfloat16 b) +{ + return vfloat16(_mm512_min_ps(a.m, b.m)); +} + +ASTCENC_SIMD_INLINE vfloat16 min(vfloat16 a, float b) +{ + return min(a, vfloat16(b)); +} + +ASTCENC_SIMD_INLINE vfloat16 max(vfloat16 a, vfloat16 b) +{ + return vfloat16(_mm512_max_ps(a.m, b.m)); +} + +ASTCENC_SIMD_INLINE vfloat16 max(vfloat16 a, float b) +{ + return max(a, vfloat16(b)); +} + +ASTCENC_SIMD_INLINE vfloat16 clamp(float minv, float maxv, vfloat16 a) +{ + a.m = _mm512_max_ps(a.m, _mm512_set1_ps(minv)); + a.m = _mm512_min_ps(a.m, _mm512_set1_ps(maxv)); + return a; +} + +ASTCENC_SIMD_INLINE vfloat16 clampzo(vfloat16 a) +{ + a.m = _mm512_max_ps(a.m, _mm512_setzero_ps()); + a.m = _mm512_min_ps(a.m, _mm512_set1_ps(1.0f)); + return a; +} + +ASTCENC_SIMD_INLINE vfloat16 abs(vfloat16 a) +{ + return vfloat16(_mm512_and_ps(a.m, _mm512_castsi512_ps(_mm512_set1_epi32(0x7fffffff)))); +} + +ASTCENC_SIMD_INLINE vfloat16 round(vfloat16 a) +{ + return vfloat16(_mm512_roundscale_ps(a.m, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)); +} + +ASTCENC_SIMD_INLINE float hmin_s(vfloat16 a) +{ + return _mm512_reduce_min_ps(a.m); +} + +ASTCENC_SIMD_INLINE vfloat16 hmin(vfloat16 a) +{ + return vfloat16(hmin_s(a)); +} + +ASTCENC_SIMD_INLINE float hmax_s(vfloat16 a) +{ + return _mm512_reduce_max_ps(a.m); +} + +ASTCENC_SIMD_INLINE vfloat16 hmax(vfloat16 a) +{ + return vfloat16(hmax_s(a)); +} + +ASTCENC_SIMD_INLINE float hadd_s(vfloat16 a) +{ + // Four sequential 4-wide adds: invariant with 4-wide accumulation order + // of pairing 128-bit lanes, matching AVX2's two 4-wide adds extended. + vfloat4 q0(_mm512_extractf32x4_ps(a.m, 0)); + vfloat4 q1(_mm512_extractf32x4_ps(a.m, 1)); + vfloat4 q2(_mm512_extractf32x4_ps(a.m, 2)); + vfloat4 q3(_mm512_extractf32x4_ps(a.m, 3)); + return (hadd_s(q0) + hadd_s(q1)) + (hadd_s(q2) + hadd_s(q3)); +} + +ASTCENC_SIMD_INLINE vfloat16 select(vfloat16 a, vfloat16 b, vmask16 cond) +{ + return vfloat16(_mm512_mask_blend_ps(cond.m, a.m, b.m)); +} + +ASTCENC_SIMD_INLINE void haccumulate(vfloat4& accum, vfloat16 a) +{ + haccumulate(accum, vfloat4(_mm512_extractf32x4_ps(a.m, 0))); + haccumulate(accum, vfloat4(_mm512_extractf32x4_ps(a.m, 1))); + haccumulate(accum, vfloat4(_mm512_extractf32x4_ps(a.m, 2))); + haccumulate(accum, vfloat4(_mm512_extractf32x4_ps(a.m, 3))); +} + +ASTCENC_SIMD_INLINE void haccumulate(vfloat16& accum, vfloat16 a) +{ + accum += a; +} + +ASTCENC_SIMD_INLINE void haccumulate(vfloat4& accum, vfloat16 a, vmask16 m) +{ + a = select(vfloat16::zero(), a, m); + haccumulate(accum, a); +} + +ASTCENC_SIMD_INLINE void haccumulate(vfloat16& accum, vfloat16 a, vmask16 m) +{ + a = select(vfloat16::zero(), a, m); + haccumulate(accum, a); +} + +ASTCENC_SIMD_INLINE vfloat16 sqrt(vfloat16 a) +{ + return vfloat16(_mm512_sqrt_ps(a.m)); +} + +/** + * @brief Expand 16 dword indices into VBMI byte indices for a float table. + * + * Lane i with float index k becomes bytes {4k, 4k+1, 4k+2, 4k+3}. + */ +ASTCENC_SIMD_INLINE __m512i vbmi_float_byte_idx(__m512i idx32) +{ + return _mm512_add_epi32( + _mm512_mullo_epi32(idx32, _mm512_set1_epi32(0x04040404)), + _mm512_set1_epi32(0x03020100)); +} + +ASTCENC_SIMD_INLINE __mmask16 vbmi_load_mask(unsigned int count, unsigned int offset) +{ + if (count <= offset) + { + return 0; + } + unsigned int n = count - offset; + return n >= 16 ? static_cast<__mmask16>(0xFFFF) + : static_cast<__mmask16>((1u << n) - 1u); +} + +ASTCENC_SIMD_INLINE vfloat16 gatherf_scalar(const float* base, vint16 indices) +{ + alignas(64) int idx[16]; + _mm512_store_si512(idx, indices.m); + return vfloat16(_mm512_setr_ps( + base[idx[0]], base[idx[1]], base[idx[2]], base[idx[3]], + base[idx[4]], base[idx[5]], base[idx[6]], base[idx[7]], + base[idx[8]], base[idx[9]], base[idx[10]], base[idx[11]], + base[idx[12]], base[idx[13]], base[idx[14]], base[idx[15]])); +} + +/** + * @brief Register-resident float table for VBMI gatherf (up to 64 entries). + * + * Load once with vgatherf_load; permute many times with gatherf. Tables larger + * than 64 keep the pointer and use the scalar-load gather. + */ +struct vgatherf_table { + __m512 t0; + __m512 t1; + __m512 t2; + __m512 t3; + const float* base; + unsigned int count; +}; + +#define ASTCENC_HAS_VGATHERF_TABLE 1 + +ASTCENC_SIMD_INLINE vgatherf_table vgatherf_load(const float* base, unsigned int count) +{ + vgatherf_table t; + t.t0 = _mm512_setzero_ps(); + t.t1 = _mm512_setzero_ps(); + t.t2 = _mm512_setzero_ps(); + t.t3 = _mm512_setzero_ps(); + t.base = base; + t.count = count; + + if (count > 0 && count <= 64) + { + t.t0 = _mm512_maskz_loadu_ps(vbmi_load_mask(count, 0), base); + if (count > 16) + { + t.t1 = _mm512_maskz_loadu_ps(vbmi_load_mask(count, 16), base + 16); + } + if (count > 32) + { + t.t2 = _mm512_maskz_loadu_ps(vbmi_load_mask(count, 32), base + 32); + } + if (count > 48) + { + t.t3 = _mm512_maskz_loadu_ps(vbmi_load_mask(count, 48), base + 48); + } + } + + return t; +} + +/** + * @brief Gather 16 floats from a preloaded VBMI table, else scalar loads. + * + * Replaces AVX-512F vgatherdps: convert dword indices to byte indices and + * vpermb (16-entry table) or vpermt2b (32-entry, or two vpermt2b for 64). + */ +ASTCENC_SIMD_INLINE vfloat16 gatherf(const vgatherf_table& t, vint16 indices) +{ + if (t.count == 0 || t.count > 64) + { + return gatherf_scalar(t.base, indices); + } + + if (t.count <= 16) + { + __m512i r = _mm512_permutexvar_epi8( + vbmi_float_byte_idx(indices.m), _mm512_castps_si512(t.t0)); + return vfloat16(_mm512_castsi512_ps(r)); + } + + if (t.count <= 32) + { + __m512i r = _mm512_permutex2var_epi8( + _mm512_castps_si512(t.t0), + vbmi_float_byte_idx(indices.m), + _mm512_castps_si512(t.t1)); + return vfloat16(_mm512_castsi512_ps(r)); + } + + __m512i bidx_lo = vbmi_float_byte_idx(indices.m); + __m512i bidx_hi = vbmi_float_byte_idx( + _mm512_sub_epi32(indices.m, _mm512_set1_epi32(32))); + __m512i lo = _mm512_permutex2var_epi8( + _mm512_castps_si512(t.t0), bidx_lo, _mm512_castps_si512(t.t1)); + __m512i hi = _mm512_permutex2var_epi8( + _mm512_castps_si512(t.t2), bidx_hi, _mm512_castps_si512(t.t3)); + __mmask16 ge32 = _mm512_cmpge_epi32_mask(indices.m, _mm512_set1_epi32(32)); + return vfloat16(_mm512_mask_blend_ps(ge32, _mm512_castsi512_ps(lo), + _mm512_castsi512_ps(hi))); +} + +ASTCENC_SIMD_INLINE vfloat16 gatherf(const vgatherf_table& t, const uint8_t* indices) +{ + return gatherf(t, vint16(indices)); +} + +/** + * @brief Gather 16 floats with AVX-512 VBMI permutes, else scalar loads. + * + * Convenience wrapper: infers the live table span from the index vector + * (hmax + 1) then permutes. Prefer vgatherf_load when the same table is + * gathered many times. Masked table loads use only [0, max_index] so + * mid-array callers stay in bounds. + */ +ASTCENC_SIMD_INLINE vfloat16 gatherf(const float* base, vint16 indices) +{ + int mx = hmax_s(indices); + if (mx < 0) + { + return gatherf_scalar(base, indices); + } + return gatherf(vgatherf_load(base, static_cast(mx) + 1u), indices); +} + +template<> +ASTCENC_SIMD_INLINE vfloat16 gatherf_byte_inds(const float* base, const uint8_t* indices) +{ + return gatherf(base, vint16(indices)); +} + +ASTCENC_SIMD_INLINE void store(vfloat16 a, float* p) +{ + _mm512_storeu_ps(p, a.m); +} + +ASTCENC_SIMD_INLINE void storea(vfloat16 a, float* p) +{ + _mm512_store_ps(p, a.m); +} + +ASTCENC_SIMD_INLINE vint16 float_to_int(vfloat16 a) +{ + return vint16(_mm512_cvttps_epi32(a.m)); +} + +ASTCENC_SIMD_INLINE vint16 float_to_int_rtn(vfloat16 a) +{ + a = a + vfloat16(0.5f); + return vint16(_mm512_cvttps_epi32(a.m)); +} + +ASTCENC_SIMD_INLINE vfloat16 int_to_float(vint16 a) +{ + return vfloat16(_mm512_cvtepi32_ps(a.m)); +} + +ASTCENC_SIMD_INLINE vint16 float_as_int(vfloat16 a) +{ + return vint16(_mm512_castps_si512(a.m)); +} + +ASTCENC_SIMD_INLINE vfloat16 int_as_float(vint16 a) +{ + return vfloat16(_mm512_castsi512_ps(a.m)); +} + +/* + * Packed 8-bit tables in a zmm. VBMI vpermb indexes any of 64 bytes in one + * instruction, so 16/32/64-entry tables are a single register (no AVX2-style + * 16-byte broadcast or XOR-chain pshufb). + */ +struct vtable16_16x8 { + vint16 t0; +}; + +struct vtable16_32x8 { + vint16 t0; +}; + +struct vtable16_64x8 { + vint16 t0; +}; + +ASTCENC_SIMD_INLINE vint16 vtable_lookup_32bit_vbmi(vint16 table, vint16 idx) +{ + __m512i r = _mm512_permutexvar_epi8(idx.m, table.m); + return vint16(_mm512_and_si512(r, _mm512_set1_epi32(0xFF))); +} + +ASTCENC_SIMD_INLINE void vtable_prepare( + vtable16_16x8& table, + const uint8_t* data +) { + table.t0 = vint16(_mm512_zextsi128_si512(_mm_loadu_si128( + reinterpret_cast(data)))); +} + +ASTCENC_SIMD_INLINE void vtable_prepare( + vtable16_32x8& table, + const uint8_t* data +) { + table.t0 = vint16(_mm512_zextsi256_si512(_mm256_loadu_si256( + reinterpret_cast(data)))); +} + +ASTCENC_SIMD_INLINE void vtable_prepare( + vtable16_64x8& table, + const uint8_t* data +) { + table.t0 = vint16(_mm512_loadu_si512(data)); +} + +ASTCENC_SIMD_INLINE vint16 vtable_lookup_32bit( + const vtable16_16x8& tbl, + vint16 idx +) { + return vtable_lookup_32bit_vbmi(tbl.t0, idx); +} + +ASTCENC_SIMD_INLINE vint16 vtable_lookup_32bit( + const vtable16_32x8& tbl, + vint16 idx +) { + return vtable_lookup_32bit_vbmi(tbl.t0, idx); +} + +ASTCENC_SIMD_INLINE vint16 vtable_lookup_32bit( + const vtable16_64x8& tbl, + vint16 idx +) { + return vtable_lookup_32bit_vbmi(tbl.t0, idx); +} + +ASTCENC_SIMD_INLINE vint16 interleave_rgba8(vint16 r, vint16 g, vint16 b, vint16 a) +{ + return r + lsl<8>(g) + lsl<16>(b) + lsl<24>(a); +} + +ASTCENC_SIMD_INLINE void store_lanes_masked(uint8_t* base, vint16 data, vmask16 mask) +{ + _mm512_mask_storeu_epi32(base, mask.m, data.m); +} + +ASTCENC_SIMD_INLINE void print(vint16 a) +{ + alignas(64) int v[16]; + storea(a, v); + printf("v16_i32:\n %8d %8d %8d %8d %8d %8d %8d %8d\n %8d %8d %8d %8d %8d %8d %8d %8d\n", + v[0], v[1], v[2], v[3], v[4], v[5], v[6], v[7], + v[8], v[9], v[10], v[11], v[12], v[13], v[14], v[15]); +} + +ASTCENC_SIMD_INLINE void printx(vint16 a) +{ + alignas(64) int v[16]; + storea(a, v); + unsigned int uv[16]; + std::memcpy(uv, v, sizeof(uv)); + printf("v16_i32:\n %08x %08x %08x %08x %08x %08x %08x %08x\n %08x %08x %08x %08x %08x %08x %08x %08x\n", + uv[0], uv[1], uv[2], uv[3], uv[4], uv[5], uv[6], uv[7], + uv[8], uv[9], uv[10], uv[11], uv[12], uv[13], uv[14], uv[15]); +} + +ASTCENC_SIMD_INLINE void print(vfloat16 a) +{ + alignas(64) float v[16]; + storea(a, v); + printf("v16_f32:\n"); + for (int i = 0; i < 16; i++) + { + printf(" %0.4f", static_cast(v[i])); + if ((i & 7) == 7) + { + printf("\n"); + } + } +} + +ASTCENC_SIMD_INLINE void print(vmask16 a) +{ + print(select(vint16(0), vint16(1), a)); +} + +#endif // ASTC_VECMATHLIB_AVX512_16_H_INCLUDED diff --git a/Source/astcenccli_entry.cpp b/Source/astcenccli_entry.cpp index c5ea17340..fe3f83cff 100644 --- a/Source/astcenccli_entry.cpp +++ b/Source/astcenccli_entry.cpp @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // ---------------------------------------------------------------------------- -// Copyright 2020-2024 Arm Limited +// Copyright 2020-2026 Arm Limited // // Licensed under the Apache License, Version 2.0 (the "License"); you may not // use this file except in compliance with the License. You may obtain a copy @@ -50,6 +50,12 @@ static bool g_cpu_has_sse41 { false }; /** Does this CPU support AVX2? Set to -1 if not yet initialized. */ static bool g_cpu_has_avx2 { false }; +/** Does this CPU support AVX-512F? Set to -1 if not yet initialized. */ +static bool g_cpu_has_avx512f { false }; + +/** Does this CPU support AVX-512 VBMI? Set to -1 if not yet initialized. */ +static bool g_cpu_has_avx512vbmi { false }; + /** Does this CPU support POPCNT? Set to -1 if not yet initialized. */ static bool g_cpu_has_popcnt { false }; @@ -90,6 +96,23 @@ static void detect_cpu_isa() __cpuidex(data, 7, 0); // AVX2 = Bank 7, EBX, bit 5 g_cpu_has_avx2 = data[1] & (1 << 5) ? true : false; + // AVX-512F = Bank 7, EBX, bit 16; also require OS XCR0 zmm state + bool avx512f = data[1] & (1 << 16) ? true : false; + g_cpu_has_avx512f = false; + g_cpu_has_avx512vbmi = false; + if (avx512f && g_cpu_has_sse41) + { + __cpuidex(data, 1, 0); + bool osxsave = data[2] & (1 << 27) ? true : false; + if (osxsave) + { + unsigned long long xcr = _xgetbv(0); + g_cpu_has_avx512f = (xcr & 0xE6) == 0xE6; + } + } + __cpuidex(data, 7, 0); + // AVX-512 VBMI = Bank 7, ECX, bit 1 + g_cpu_has_avx512vbmi = g_cpu_has_avx512f && (data[2] & (1 << 1)); } // Ensure state bits are updated before init flag is updated @@ -121,10 +144,25 @@ static void detect_cpu_isa() } g_cpu_has_avx2 = 0; + g_cpu_has_avx512f = 0; + g_cpu_has_avx512vbmi = 0; if (__get_cpuid_count(7, 0, &data[0], &data[1], &data[2], &data[3])) { // AVX2 = Bank 7, EBX, bit 5 g_cpu_has_avx2 = data[1] & (1 << 5) ? true : false; + bool avx512f = data[1] & (1 << 16) ? true : false; + bool avx512vbmi = data[2] & (1u << 1) ? true : false; + if (avx512f) + { + unsigned int eax, edx; + if (__get_cpuid_count(1, 0, &data[0], &data[1], &data[2], &data[3]) && + (data[2] & (1u << 27))) + { + __asm__ volatile ("xgetbv" : "=a"(eax), "=d"(edx) : "c"(0)); + g_cpu_has_avx512f = (eax & 0xE6) == 0xE6; + } + } + g_cpu_has_avx512vbmi = g_cpu_has_avx512f && avx512vbmi; } // Ensure state bits are updated before init flag is updated @@ -184,6 +222,33 @@ static bool cpu_supports_sse41() } #endif +#if ASTCENC_AVX >= 3 +/** + * @brief Run-time detection if the host CPU supports AVX-512F. + * + * @return @c true if supported, @c false if not. + */ +static bool cpu_supports_avx512f() +{ + if (!g_init) + { + detect_cpu_isa(); + } + + return g_cpu_has_avx512f; +} + +static bool cpu_supports_avx512vbmi() +{ + if (!g_init) + { + detect_cpu_isa(); + } + + return g_cpu_has_avx512vbmi; +} +#endif + #if ASTCENC_AVX >= 2 /** * @brief Run-time detection if the host CPU supports AVX 2 extension. @@ -221,6 +286,19 @@ static inline void print_error( */ static bool validate_cpu_isa() { + #if ASTCENC_AVX >= 3 + if (!cpu_supports_avx512f()) + { + print_error("ERROR: Host does not support AVX-512F ISA extension\n"); + return false; + } + if (!cpu_supports_avx512vbmi()) + { + print_error("ERROR: Host does not support AVX-512 VBMI ISA extension\n"); + return false; + } + #endif + #if ASTCENC_AVX >= 2 if (!cpu_supports_avx2()) { diff --git a/Source/astcenccli_toplevel_help.cpp b/Source/astcenccli_toplevel_help.cpp index e46a97332..20b407b7c 100644 --- a/Source/astcenccli_toplevel_help.cpp +++ b/Source/astcenccli_toplevel_help.cpp @@ -561,7 +561,9 @@ QUICK REFERENCE /* See header for documentation. */ void astcenc_print_header() { -#if (ASTCENC_AVX == 2) +#if (ASTCENC_AVX >= 3) + const char* simdtype = "avx512+vbmi"; +#elif (ASTCENC_AVX == 2) const char* simdtype = "avx2"; #elif (ASTCENC_SSE == 41) const char* simdtype = "sse4.1"; diff --git a/Source/cmake_core.cmake b/Source/cmake_core.cmake index daf06f486..b453338b2 100644 --- a/Source/cmake_core.cmake +++ b/Source/cmake_core.cmake @@ -396,6 +396,41 @@ macro(astcenc_set_properties ASTCENC_TARGET_NAME ASTCENC_VENEER_TYPE) $<${is_gnu_fe}:-Wno-unused-command-line-argument>) endif() + elseif(${ASTCENC_ISA_SIMD} MATCHES "avx512") + target_compile_definitions(${ASTCENC_TARGET_NAME} + PRIVATE + ASTCENC_NEON=0 + ASTCENC_SVE=0 + ASTCENC_SSE=41 + ASTCENC_AVX=3 + ASTCENC_X86_GATHERS=$ + ASTCENC_POPCNT=1 + ASTCENC_F16C=1) + + if (${ASTCENC_VENEER_TYPE} GREATER 0) + target_compile_options(${ASTCENC_TARGET_NAME} + PRIVATE + $<${is_gnu_fe}:-msse2> + $<${is_gnu_fe}:-mno-sse4.1> + $<${is_gnu_fe}:-Wno-unused-command-line-argument>) + else() + target_compile_options(${ASTCENC_TARGET_NAME} + PRIVATE + $<${is_msvc_fe}:/arch:AVX512> + $<${is_clangcl}:-mavx512f -mavx512bw -mavx512dq -mavx512vl -mavx512vbmi -mpopcnt -mf16c> + $<${is_gnu_fe}:-mavx512f -mavx512bw -mavx512dq -mavx512vl -mavx512vbmi -mpopcnt -mf16c> + $<${is_gnu_fe}:-Wno-unused-command-line-argument>) + endif() + + # Non-invariant builds enable FMA (included in AVX-512F). This reduces + # image quality by up to 0.2 dB (normally much less) in exchange for + # higher encode throughput. + if((NOT ${ASTCENC_INVARIANCE}) AND (NOT ${ASTCENC_VENEER_TYPE})) + target_compile_options(${ASTCENC_TARGET_NAME} + PRIVATE + $<${is_gnu_fe}:-mfma>) + endif() + elseif(${ASTCENC_ISA_SIMD} MATCHES "avx2") # Gathers are quite slow on many x86 microarchitectures, to the point where # it can be significantly faster to just avoid them use scalar loads. From b06609461d0eb25461c400fc380e878c2da7a993 Mon Sep 17 00:00:00 2001 From: Madan Mohan Manokar Date: Wed, 16 Sep 2026 14:12:43 +0000 Subject: [PATCH 2/2] Document the AVX-512 ISA and add 16-wide SIMD unit tests. Describe astcenc-avx512 in the README, changelog, and build guide, cover gatherf/vtable in test_simd (including VLA round tests that are multiples of SIMD_WIDTH), and accept --encoder avx512 in the Python CLI and image test runners. --- .github/workflows/post_weekly_release.yaml | 23 +- Docs/Building.md | 16 +- Docs/ChangeLog-5x.md | 12 + README.md | 11 +- Source/UnitTest/test_simd.cpp | 298 ++++++++++++++++++++- Source/astcenc.h | 8 +- Source/astcenc_vecmathlib_avx512_16.h | 4 +- Test/astc_test_functional.py | 2 +- Test/astc_test_image.py | 4 +- Utils/Example/CMakeLists.txt | 1 + 10 files changed, 354 insertions(+), 25 deletions(-) diff --git a/.github/workflows/post_weekly_release.yaml b/.github/workflows/post_weekly_release.yaml index 2500447ea..542d0d97c 100644 --- a/.github/workflows/post_weekly_release.yaml +++ b/.github/workflows/post_weekly_release.yaml @@ -80,7 +80,7 @@ jobs: export CXX=clang++ mkdir build_rel cd build_rel - cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON -DASTCENC_PACKAGE=x64 .. + cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_ISA_AVX512=ON -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON -DASTCENC_PACKAGE=x64 .. make install package -j4 - name: Upload binaries @@ -101,10 +101,21 @@ jobs: python -m pip install --upgrade pip pip install numpy Pillow + # astcenc-avx512 is always packaged. GitHub-hosted runners may lack + # AVX-512F+VBMI (or OS XCR0 state); probe the binary and run it only + # when the veneer accepts the host. - name: Run system tests run: | python ./Test/astc_test_functional.py - python ./Test/astc_test_image.py --encoder all-x86 --test-set Small + python ./Test/astc_test_image.py --encoder sse2 --test-set Small + python ./Test/astc_test_image.py --encoder sse4.1 --test-set Small + python ./Test/astc_test_image.py --encoder avx2 --test-set Small + if ./bin/astcenc-avx512 -help >/dev/null 2>&1; then + python ./Test/astc_test_functional.py --encoder avx512 + python ./Test/astc_test_image.py --encoder avx512 --test-set Small + else + echo "Skipping avx512 execute tests: host has no AVX-512F+VBMI" + fi build-macos-universal: name: macOS universal @@ -160,7 +171,7 @@ jobs: run: | mkdir build_rel cd build_rel - cmake -G "Visual Studio 17 2022" -T ClangCL -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON -DASTCENC_PACKAGE=x64 .. + cmake -G "Visual Studio 17 2022" -T ClangCL -DCMAKE_INSTALL_PREFIX=../ -DASTCENC_ISA_AVX512=ON -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON -DASTCENC_PACKAGE=x64 .. msbuild astcencoder.sln -property:Configuration=Release msbuild PACKAGE.vcxproj -property:Configuration=Release msbuild INSTALL.vcxproj -property:Configuration=Release @@ -205,6 +216,12 @@ jobs: - name: Run system tests run: | python ./Test/astc_test_image.py --test-set Small + bin\astcenc-avx512.exe -help >nul 2>&1 + if %ERRORLEVEL%==0 ( + python ./Test/astc_test_image.py --encoder avx512 --test-set Small + ) else ( + echo Skipping avx512 execute tests: host has no AVX-512F+VBMI + ) shell: cmd sign-binaries: diff --git a/Docs/Building.md b/Docs/Building.md index e1622f280..03d0a5410 100644 --- a/Docs/Building.md +++ b/Docs/Building.md @@ -27,11 +27,11 @@ cd build # x86-64 using a Visual Studio solution cmake -G "Visual Studio 16 2019" -T ClangCL -DCMAKE_INSTALL_PREFIX=..\ ^ - -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON .. + -DASTCENC_ISA_AVX512=ON -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON .. # x86-64 using NMake cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=..\ ^ - -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON .. + -DASTCENC_ISA_AVX512=ON -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON .. ``` A single CMake configure can build multiple binaries for a single target CPU @@ -83,7 +83,7 @@ cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ # x86-64 cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ \ - -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON .. + -DASTCENC_ISA_AVX512=ON -DASTCENC_ISA_AVX2=ON -DASTCENC_ISA_SSE41=ON -DASTCENC_ISA_SSE2=ON .. # macOS universal binary build cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../ .. @@ -207,6 +207,16 @@ To enable this binary variant add `-DASTCENC_ISA_NONE=ON` to the CMake command line when configuring. It is NOT recommended to use this for production; it is significantly slower than the vectorized SIMD builds. +### AVX-512 builds + +An optional x86-64 AVX-512 backend is enabled with `-DASTCENC_ISA_AVX512=ON`. +This produces `astcenc-avx512`, a 16-wide VLA codec. It requires AVX-512F and +AVX-512VBMI in the compiler flags, and will refuse to run unless the CPU and +OS expose both (including OS XCR0 state). + +It is not part of macOS universal builds. Windows builds need Clang or +Clang-CL (`-T ClangCL`); MSVC `/arch:AVX512` does not enable VBMI. + ### No x86 gather instruction builds On many x86 microarchitectures the native AVX gather instructions are slower diff --git a/Docs/ChangeLog-5x.md b/Docs/ChangeLog-5x.md index 4d03b6895..221a45508 100644 --- a/Docs/ChangeLog-5x.md +++ b/Docs/ChangeLog-5x.md @@ -6,6 +6,18 @@ release of the 5.x series. All performance data on this page is measured on an Intel Core i5-9600K clocked at 4.2 GHz, running `astcenc` using AVX2 and 6 threads. + +## 5.8.0 + +**Status:** In development. + +* **Codec library updates:** + * **Optimization:** Added a compile-time AVX-512 SIMD backend + (`ASTCENC_SIMD_WIDTH` 16). +* **Command line tool updates:** + * **Feature:** New `astcenc-avx512` binary, enabled with + `-DASTCENC_ISA_AVX512=ON`. Requires AVX-512F and AVX-512VBMI. + ## 5.7.0 diff --git a/README.md b/README.md index acf4f98d4..dd4ba81b0 100644 --- a/README.md +++ b/README.md @@ -67,15 +67,20 @@ Binaries are provided for 64-bit builds on Windows, macOS, and Linux. For Windows and Linux, the builds of astcenc are provided as multiple binaries, each tuned for a specific SIMD instruction set. -For x86-64 we provide, in order of increasing performance: +For x86-64 we provide these SIMD builds, from the baseline ISA to the +highest ISA we ship: * `astcenc-sse2` - uses SSE2 * `astcenc-sse4.1` - uses SSE4.1 and POPCNT * `astcenc-avx2` - uses AVX2, SSE4.2, POPCNT, and F16C +* `astcenc-avx512` - uses AVX-512F, AVX-512BW, AVX-512DQ, AVX-512VL, + AVX-512VBMI, POPCNT, and F16C The x86-64 SSE2 builds will work on all x86-64 machines, but it is the slowest -of the three. The other two require extended CPU instruction set support which -is not universally available, but each step gains ~15% more performance. +of the four. The other builds require extended CPU instruction set support +which is not universally available. AVX2 is typically ~15% faster than SSE4.1. +The AVX-512 build is a 16-wide VLA backend and needs both AVX-512F and +AVX-512VBMI in hardware and enabled in the OS (XCR0). For Arm we provide, in order of increasing performance: diff --git a/Source/UnitTest/test_simd.cpp b/Source/UnitTest/test_simd.cpp index b72563d19..9a028b925 100644 --- a/Source/UnitTest/test_simd.cpp +++ b/Source/UnitTest/test_simd.cpp @@ -71,6 +71,45 @@ static vint8 vint8_lit( #endif +#if ASTCENC_SIMD_WIDTH == 16 +/** + * @brief Construct from 16 scalar values. + * + * The value of @c a is stored to lane 0 (LSB) in the SIMD register. + */ +static vfloat16 vfloat16_lit( + float a, float b, float c, float d, + float e, float f, float g, float h, + float i, float j, float k, float l, + float m, float n, float o, float p +) { + alignas(64) float data[16] { + a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p + }; + + return vfloat16(data); +} + +/** + * @brief Construct from 16 scalar values. + * + * The value of @c a is stored to lane 0 (LSB) in the SIMD register. + */ +static vint16 vint16_lit( + int a, int b, int c, int d, + int e, int f, int g, int h, + int i, int j, int k, int l, + int m, int n, int o, int p +) { + alignas(64) int data[16] { + a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p + }; + + return vint16(data); +} + +#endif + static unsigned int round_down(unsigned int x) { unsigned int remainder = x % ASTCENC_SIMD_WIDTH; @@ -91,10 +130,12 @@ static unsigned int round_up(unsigned int x) /** @brief Test VLA loop limit round down. */ TEST(SuiteMisc, RoundDownVLA) { - // Static ones which are valid for all VLA widths + // Multiples of the compile-time VLA width EXPECT_EQ(round_down_to_simd_multiple_vla(0), 0u); - EXPECT_EQ(round_down_to_simd_multiple_vla(8), 8u); - EXPECT_EQ(round_down_to_simd_multiple_vla(16), 16u); + EXPECT_EQ(round_down_to_simd_multiple_vla(ASTCENC_SIMD_WIDTH), + static_cast(ASTCENC_SIMD_WIDTH)); + EXPECT_EQ(round_down_to_simd_multiple_vla(ASTCENC_SIMD_WIDTH * 2), + static_cast(ASTCENC_SIMD_WIDTH * 2)); // Variable ones which depend on VLA width EXPECT_EQ(round_down_to_simd_multiple_vla(3), round_down(3)); @@ -106,10 +147,12 @@ TEST(SuiteMisc, RoundDownVLA) /** @brief Test VLA loop limit round up. */ TEST(SuiteMisc, RoundUpVLA) { - // Static ones which are valid for all VLA widths + // Multiples of the compile-time VLA width EXPECT_EQ(round_up_to_simd_multiple_vla(0), 0u); - EXPECT_EQ(round_up_to_simd_multiple_vla(8), 8u); - EXPECT_EQ(round_up_to_simd_multiple_vla(16), 16u); + EXPECT_EQ(round_up_to_simd_multiple_vla(ASTCENC_SIMD_WIDTH), + static_cast(ASTCENC_SIMD_WIDTH)); + EXPECT_EQ(round_up_to_simd_multiple_vla(ASTCENC_SIMD_WIDTH * 2), + static_cast(ASTCENC_SIMD_WIDTH * 2)); // Variable ones which depend on VLA width EXPECT_EQ(round_up_to_simd_multiple_vla(3), round_up(3)); @@ -2058,7 +2101,7 @@ TEST(SuiteVint4, interleave_rgba8) #endif } -# if ASTCENC_SIMD_WIDTH == 8 +#if ASTCENC_SIMD_WIDTH == 8 // VFLOAT8 tests - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -3831,4 +3874,245 @@ TEST(SuiteVint8, vtable8_64x8) #endif +#if ASTCENC_SIMD_WIDTH == 16 + +alignas(64) static const float f32_data16[17] { + 0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, + 8.0f, 9.0f, 10.0f, 11.0f, 12.0f, 13.0f, 14.0f, 15.0f, 16.0f +}; + +/** @brief Test unaligned vfloat16 data load. */ +TEST(SuiteVfloat16, UnalignedLoad) +{ + vfloat16 a(&(f32_data16[1])); + + alignas(64) float ra[16]; + storea(a, ra); + + for (int i = 0; i < 16; i++) + { + EXPECT_EQ(ra[i], static_cast(i + 1)); + } +} + +/** @brief Test aligned vfloat16 data load. */ +TEST(SuiteVfloat16, AlignedLoad) +{ + vfloat16 a = vfloat16::loada(f32_data16); + + alignas(64) float ra[16]; + storea(a, ra); + + for (int i = 0; i < 16; i++) + { + EXPECT_EQ(ra[i], static_cast(i)); + } +} + +/** @brief Test vfloat16 add. */ +TEST(SuiteVfloat16, Add) +{ + vfloat16 a = vfloat16_lit( + 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, + 9.0f, 10.0f, 11.0f, 12.0f, 13.0f, 14.0f, 15.0f, 16.0f); + vfloat16 b = vfloat16_lit( + 0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f, 0.7f, 0.8f, + 0.9f, 1.0f, 1.1f, 1.2f, 1.3f, 1.4f, 1.5f, 1.6f); + vfloat16 r = a + b; + + alignas(64) float ra[16]; + storea(r, ra); + + EXPECT_EQ(ra[0], 1.0f + 0.1f); + EXPECT_EQ(ra[7], 8.0f + 0.8f); + EXPECT_EQ(ra[15], 16.0f + 1.6f); +} + +/** @brief Test vfloat16 gatherf. */ +TEST(SuiteVfloat16, gatherf) +{ + vint16 indices = vint16_lit(0, 4, 3, 2, 7, 15, 8, 1, 9, 10, 11, 12, 13, 14, 6, 5); + vfloat16 r = gatherf(f32_data16, indices); + + alignas(64) float ra[16]; + storea(r, ra); + + EXPECT_EQ(ra[0], 0.0f); + EXPECT_EQ(ra[1], 4.0f); + EXPECT_EQ(ra[2], 3.0f); + EXPECT_EQ(ra[3], 2.0f); + EXPECT_EQ(ra[4], 7.0f); + EXPECT_EQ(ra[5], 15.0f); + EXPECT_EQ(ra[6], 8.0f); + EXPECT_EQ(ra[7], 1.0f); + EXPECT_EQ(ra[15], 5.0f); +} + +/** @brief Test vfloat16 vgatherf_table for a 16-entry table. */ +TEST(SuiteVfloat16, gatherf_table16) +{ + vgatherf_table table = vgatherf_load(f32_data16, 16); + vint16 indices = vint16_lit(0, 4, 3, 2, 7, 15, 8, 1, 9, 10, 11, 12, 13, 14, 6, 5); + vfloat16 r = gatherf(table, indices); + + alignas(64) float ra[16]; + storea(r, ra); + + EXPECT_EQ(ra[0], 0.0f); + EXPECT_EQ(ra[5], 15.0f); + EXPECT_EQ(ra[15], 5.0f); +} + +/** @brief Test vfloat16 vgatherf_table for a 32-entry table. */ +TEST(SuiteVfloat16, gatherf_table32) +{ + alignas(64) float data[32]; + for (int i = 0; i < 32; i++) + { + data[i] = static_cast(i); + } + + vgatherf_table table = vgatherf_load(data, 32); + vint16 indices = vint16_lit(0, 16, 31, 17, 8, 24, 7, 15, 1, 30, 18, 19, 20, 21, 22, 23); + vfloat16 r = gatherf(table, indices); + + alignas(64) float ra[16]; + storea(r, ra); + + EXPECT_EQ(ra[0], 0.0f); + EXPECT_EQ(ra[1], 16.0f); + EXPECT_EQ(ra[2], 31.0f); + EXPECT_EQ(ra[3], 17.0f); + EXPECT_EQ(ra[5], 24.0f); +} + +/** @brief Test vfloat16 vgatherf_table for a 64-entry table. */ +TEST(SuiteVfloat16, gatherf_table64) +{ + alignas(64) float data[64]; + for (int i = 0; i < 64; i++) + { + data[i] = static_cast(i); + } + + vgatherf_table table = vgatherf_load(data, 64); + vint16 indices = vint16_lit(0, 32, 63, 48, 16, 31, 33, 47, 1, 62, 17, 18, 19, 20, 21, 22); + vfloat16 r = gatherf(table, indices); + + alignas(64) float ra[16]; + storea(r, ra); + + EXPECT_EQ(ra[0], 0.0f); + EXPECT_EQ(ra[1], 32.0f); + EXPECT_EQ(ra[2], 63.0f); + EXPECT_EQ(ra[3], 48.0f); + EXPECT_EQ(ra[6], 33.0f); +} + +/** @brief Test vfloat16 vgatherf_table scalar fallback for tables larger than 64. */ +TEST(SuiteVfloat16, gatherf_table144) +{ + alignas(64) float data[144]; + for (int i = 0; i < 144; i++) + { + data[i] = static_cast(i); + } + + vgatherf_table table = vgatherf_load(data, 144); + vint16 indices = vint16_lit(0, 64, 143, 80, 16, 96, 32, 127, 1, 65, 142, 81, 48, 112, 63, 128); + vfloat16 r = gatherf(table, indices); + + alignas(64) float ra[16]; + storea(r, ra); + + EXPECT_EQ(ra[0], 0.0f); + EXPECT_EQ(ra[1], 64.0f); + EXPECT_EQ(ra[2], 143.0f); + EXPECT_EQ(ra[3], 80.0f); + EXPECT_EQ(ra[5], 96.0f); + EXPECT_EQ(ra[7], 127.0f); + EXPECT_EQ(ra[10], 142.0f); + EXPECT_EQ(ra[15], 128.0f); +} + +/** @brief Test vfloat16 select. */ +TEST(SuiteVfloat16, select) +{ + vfloat16 m1 = vfloat16(1.0f); + vfloat16 m2 = vfloat16_lit( + 1.0f, 2.0f, 1.0f, 2.0f, 1.0f, 2.0f, 1.0f, 2.0f, + 1.0f, 2.0f, 1.0f, 2.0f, 1.0f, 2.0f, 1.0f, 2.0f); + vmask16 cond = m1 == m2; + + vfloat16 a = vfloat16_lit( + 1.0f, 3.0f, 3.0f, 1.0f, 1.0f, 3.0f, 3.0f, 1.0f, + 1.0f, 3.0f, 3.0f, 1.0f, 1.0f, 3.0f, 3.0f, 1.0f); + vfloat16 b = vfloat16_lit( + 4.0f, 2.0f, 2.0f, 4.0f, 4.0f, 2.0f, 2.0f, 4.0f, + 4.0f, 2.0f, 2.0f, 4.0f, 4.0f, 2.0f, 2.0f, 4.0f); + + vfloat16 r1 = select(a, b, cond); + + alignas(64) float ra[16]; + storea(r1, ra); + + EXPECT_EQ(ra[0], 4.0f); + EXPECT_EQ(ra[1], 3.0f); + EXPECT_EQ(ra[2], 2.0f); + EXPECT_EQ(ra[3], 1.0f); +} + +/** @brief Test vint16 table permute. */ +TEST(SuiteVint16, vtable16_16x8) +{ + uint8_t data[16] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f + }; + + vtable16_16x8 table; + vtable_prepare(table, data); + + vint16 index = vint16_lit(0, 7, 4, 15, 1, 2, 14, 4, 3, 5, 6, 8, 9, 10, 11, 12); + + vint16 result = vtable_lookup_32bit(table, index); + + alignas(64) int ra[16]; + store(result, ra); + + EXPECT_EQ(ra[0], 0); + EXPECT_EQ(ra[1], 7); + EXPECT_EQ(ra[2], 4); + EXPECT_EQ(ra[3], 15); + EXPECT_EQ(ra[6], 14); +} + +/** @brief Test vint16 64-entry table permute. */ +TEST(SuiteVint16, vtable16_64x8) +{ + uint8_t data[64]; + for (int i = 0; i < 64; i++) + { + data[i] = static_cast(i); + } + + vtable16_64x8 table; + vtable_prepare(table, data); + + vint16 index = vint16_lit(0, 7, 4, 15, 16, 20, 38, 63, 32, 48, 1, 62, 17, 18, 19, 31); + + vint16 result = vtable_lookup_32bit(table, index); + + alignas(64) int ra[16]; + store(result, ra); + + EXPECT_EQ(ra[0], 0); + EXPECT_EQ(ra[4], 16); + EXPECT_EQ(ra[6], 38); + EXPECT_EQ(ra[7], 63); + EXPECT_EQ(ra[8], 32); +} + +#endif + } diff --git a/Source/astcenc.h b/Source/astcenc.h index 3f638d887..946a243a5 100644 --- a/Source/astcenc.h +++ b/Source/astcenc.h @@ -48,10 +48,10 @@ * Extended instruction set support * ================================ * - * This library supports use of extended instruction sets, such as SSE4.1 and AVX2. These are - * enabled at compile time when building the library. There is no runtime checking in the core - * library that the instruction sets used are actually available. Checking compatibility is the - * responsibility of the calling code. + * This library supports use of extended instruction sets, such as SSE4.1, AVX2, and AVX-512. + * These are enabled at compile time when building the library. There is no runtime checking in + * the core library that the instruction sets used are actually available. Checking compatibility + * is the responsibility of the calling code. * * Threading * ========= diff --git a/Source/astcenc_vecmathlib_avx512_16.h b/Source/astcenc_vecmathlib_avx512_16.h index 4cf6e4b93..287313629 100644 --- a/Source/astcenc_vecmathlib_avx512_16.h +++ b/Source/astcenc_vecmathlib_avx512_16.h @@ -18,8 +18,8 @@ /** * @brief 16x32-bit vectors, implemented using AVX-512F. * - * Experimental compile-time AVX-512 ISA: ASTCENC_SIMD_WIDTH is 16. This is a - * VLA backend sibling of astcenc_vecmathlib_avx2_8.h, not a runtime overlay. + * Compile-time AVX-512 ISA: ASTCENC_SIMD_WIDTH is 16. This is a VLA backend + * sibling of astcenc_vecmathlib_avx2_8.h, not a runtime overlay. * gatherf is implemented with AVX-512 VBMI (vpermb / vpermt2b) for tables that * fit in 1–4 zmm, and a scalar-load fallback for larger index ranges. It does * not use AVX-512F vgatherdps. vgatherf_load hoists those masked zmm loads so diff --git a/Test/astc_test_functional.py b/Test/astc_test_functional.py index 063e01606..4b18d0c5f 100644 --- a/Test/astc_test_functional.py +++ b/Test/astc_test_functional.py @@ -2269,7 +2269,7 @@ def main() -> int: ''' parser = argparse.ArgumentParser() - coders = ['none', 'neon', 'sve_128', 'sve_256', 'sse2', 'sse4.1', 'avx2'] + coders = ['none', 'neon', 'sve_128', 'sve_256', 'sse2', 'sse4.1', 'avx2', 'avx512'] parser.add_argument('--encoder', dest='encoder', default='avx2', choices=coders, help='test encoder variant') args = parser.parse_known_args() diff --git a/Test/astc_test_image.py b/Test/astc_test_image.py index 37cbedd51..3b770a584 100644 --- a/Test/astc_test_image.py +++ b/Test/astc_test_image.py @@ -357,7 +357,7 @@ def parse_command_line(): test_encoders = [ 'none', 'native', 'universal', 'neon', 'sve_256', 'sve_128', - 'sse2', 'sse4.1', 'avx2' + 'sse2', 'sse4.1', 'avx2', 'avx512' ] test_encoders_arm64 = [ @@ -365,7 +365,7 @@ def parse_command_line(): ] test_encoders_x86 = [ - 'sse2', 'sse4.1', 'avx2' + 'sse2', 'sse4.1', 'avx2', 'avx512' ] encoder_choices = reference_encoders + test_encoders diff --git a/Utils/Example/CMakeLists.txt b/Utils/Example/CMakeLists.txt index 2a80f154d..a64461165 100644 --- a/Utils/Example/CMakeLists.txt +++ b/Utils/Example/CMakeLists.txt @@ -34,6 +34,7 @@ set(ASTCENC_VERSION "main" CACHE STRING "ASTCEncoder git version to import") # * Add "-DASTCENC_ISA_SSE2:String=ON" and link against "astcenc-sse2-static" # * Add "-DASTCENC_ISA_SSE41:String=ON" and link against "astcenc-sse4.1-static" # * Add "-DASTCENC_ISA_AVX2:String=ON" and link against "astcenc-avx2-static" +# * Add "-DASTCENC_ISA_AVX512:String=ON" and link against "astcenc-avx512-static" # * Add "-DASTCENC_ISA_NEON:String=ON" and link against "astcenc-neon-static" # # The DOWNLOAD_COMMAND here is slightly convoluted so that we can point the