| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // | ||
| 2 | // SPDX-FileCopyrightText: Copyright 2025 Arm Limited and/or its affiliates <open-source-office@arm.com> | ||
| 3 | // | ||
| 4 | // SPDX-License-Identifier: Apache-2.0 | ||
| 5 | // | ||
| 6 | |||
| 7 | #include <gtest/gtest.h> | ||
| 8 | |||
| 9 | #include <array> | ||
| 10 | #include <cstddef> | ||
| 11 | #include <cstdint> | ||
| 12 | #include <cstdlib> | ||
| 13 | #include <sstream> | ||
| 14 | #include <string> | ||
| 15 | #include <tuple> | ||
| 16 | |||
| 17 | #include "kai/ukernels/matmul/matmul_clamp_f16_qai8dxp_qsi4cxp/kai_matmul_clamp_f16_qai8dxp1x4_qsi4cxp4x4_1x4_neon_dotprod.h" | ||
| 18 | #include "kai/ukernels/matmul/matmul_clamp_f16_qai8dxp_qsi4cxp/kai_matmul_clamp_f16_qai8dxp1x8_qsi4cxp4x8_1x4_neon_dotprod.h" | ||
| 19 | #include "kai/ukernels/matmul/matmul_clamp_f16_qai8dxp_qsi4cxp/kai_matmul_clamp_f16_qai8dxp4x4_qsi4cxp4x4_16x4_neon_dotprod.h" | ||
| 20 | #include "kai/ukernels/matmul/matmul_clamp_f16_qai8dxp_qsi4cxp/kai_matmul_clamp_f16_qai8dxp4x8_qsi4cxp4x8_16x4_neon_i8mm.h" | ||
| 21 | #include "kai/ukernels/matmul/matmul_clamp_f16_qai8dxp_qsi4cxp/kai_matmul_clamp_f16_qai8dxp_qsi4cxp_interface.h" | ||
| 22 | #include "kai/ukernels/matmul/pack/kai_lhs_quant_pack_qai8dxp_f16_neon.h" | ||
| 23 | #include "kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qsi4cxp_qs4cxs1s0.h" | ||
| 24 | #include "test/common/buffer.hpp" | ||
| 25 | #include "test/common/compare.hpp" | ||
| 26 | #include "test/common/cpu_info.hpp" | ||
| 27 | #include "test/common/data_format.hpp" | ||
| 28 | #include "test/common/float16.hpp" | ||
| 29 | #include "test/common/int4.hpp" | ||
| 30 | #include "test/common/matmul_test_common.hpp" | ||
| 31 | #include "test/common/matrix_portion.hpp" | ||
| 32 | #include "test/common/memory.hpp" | ||
| 33 | #include "test/common/round.hpp" | ||
| 34 | #include "test/common/test_suite.hpp" | ||
| 35 | #include "test/reference/cast.hpp" | ||
| 36 | #include "test/reference/clamp.hpp" | ||
| 37 | #include "test/reference/fill.hpp" | ||
| 38 | #include "test/reference/matmul.hpp" | ||
| 39 | #include "test/reference/pack.hpp" | ||
| 40 | #include "test/reference/pad.hpp" | ||
| 41 | #include "test/reference/quantize.hpp" | ||
| 42 | |||
| 43 | namespace kai::test { | ||
| 44 | |||
| 45 | static const std::array<UkernelVariant<kai_matmul_clamp_f16_qai8dxp_qsi4cxp_ukernel>, 4> | ||
| 46 | variants_kai_matmul_clamp_f16_qai8dxp_qsi4cxp = {{ | ||
| 47 | {UKERNEL_MATMUL_VARIANT(clamp_f16_qai8dxp1x4_qsi4cxp4x4_1x4_neon_dotprod), | ||
| 48 | "kai_matmul_clamp_f16_qai8dxp1x4_qsi4cxp4x4_1x4_neon_dotprod", cpu_has_dotprod_and_fp16}, | ||
| 49 | {UKERNEL_MATMUL_VARIANT(clamp_f16_qai8dxp4x4_qsi4cxp4x4_16x4_neon_dotprod), | ||
| 50 | "kai_matmul_clamp_f16_qai8dxp4x4_qsi4cxp4x4_16x4_neon_dotprod", cpu_has_dotprod_and_fp16}, | ||
| 51 | {UKERNEL_MATMUL_VARIANT(clamp_f16_qai8dxp1x8_qsi4cxp4x8_1x4_neon_dotprod), | ||
| 52 | "kai_matmul_clamp_f16_qai8dxp1x8_qsi4cxp4x8_1x4_neon_dotprod", cpu_has_dotprod_and_fp16}, | ||
| 53 | {UKERNEL_MATMUL_VARIANT(clamp_f16_qai8dxp4x8_qsi4cxp4x8_16x4_neon_i8mm), | ||
| 54 | "kai_matmul_clamp_f16_qai8dxp4x8_qsi4cxp4x8_16x4_neon_i8mm", cpu_has_i8mm_and_fp16}, | ||
| 55 | }}; | ||
| 56 | |||
| 57 | class MatMulTest_f16_qai8dxp_qsi4cxp : public ::testing::TestWithParam<MatMulTestPortionedParamsWithBias> {}; | ||
| 58 | |||
| 59 |
7/14✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
✓ Branch 4 taken 1 time.
✗ Branch 5 not taken.
✓ Branch 6 taken 1 time.
✗ Branch 7 not taken.
✓ Branch 8 taken 1 time.
✗ Branch 9 not taken.
✓ Branch 10 taken 1 time.
✗ Branch 11 not taken.
✓ Branch 12 taken 1 time.
✗ Branch 13 not taken.
|
2690 | TEST_P(MatMulTest_f16_qai8dxp_qsi4cxp, EndToEnd) { |
| 60 | 6776 | const auto& [variant_index, matmul_shape, portion, has_bias] = GetParam(); | |
| 61 | 1792 | const auto& ukernel_variant = variants_kai_matmul_clamp_f16_qai8dxp_qsi4cxp.at(variant_index); | |
| 62 | |||
| 63 |
2/4✓ Branch 0 taken 896 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 896 times.
✗ Branch 3 not taken.
|
896 | if (ukernel_variant.fn_is_supported && !ukernel_variant.fn_is_supported()) { |
| 64 | ✗ | GTEST_SKIP() << "Unsupported CPU feature"; | |
| 65 | } | ||
| 66 | |||
| 67 | 896 | const std::uint32_t seed = 0; | |
| 68 | |||
| 69 | 1792 | const size_t M = matmul_shape.m; | |
| 70 | 1792 | const size_t N = matmul_shape.n; | |
| 71 | 1792 | const size_t K = matmul_shape.k; | |
| 72 | |||
| 73 | 896 | const auto mr = ukernel_variant.interface.get_mr(); | |
| 74 | 896 | const auto nr = ukernel_variant.interface.get_nr(); | |
| 75 | 896 | const auto kr = ukernel_variant.interface.get_kr(); | |
| 76 | 896 | const auto sr = ukernel_variant.interface.get_sr(); | |
| 77 | |||
| 78 |
4/4✓ Branch 0 taken 448 times.
✓ Branch 1 taken 448 times.
✓ Branch 2 taken 168 times.
✓ Branch 3 taken 280 times.
|
896 | if (mr == 1 && M > 1) { |
| 79 |
3/6✓ Branch 0 taken 280 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 280 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 280 times.
✗ Branch 5 not taken.
|
280 | GTEST_SKIP() << "Kernel does not support M != 1"; |
| 80 | } | ||
| 81 | |||
| 82 | 616 | auto m_step = ukernel_variant.interface.get_m_step(); | |
| 83 |
3/14✓ Branch 0 taken 616 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 616 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✓ Branch 13 taken 616 times.
|
616 | ASSERT_TRUE(m_step % mr == 0); |
| 84 | |||
| 85 | 616 | auto n_step = ukernel_variant.interface.get_n_step(); | |
| 86 |
3/14✓ Branch 0 taken 616 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 616 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✓ Branch 13 taken 616 times.
|
616 | ASSERT_TRUE(n_step % nr == 0); |
| 87 | |||
| 88 | 1232 | const auto rect = portion.compute_portion(M, N, m_step, n_step); | |
| 89 |
4/4✓ Branch 0 taken 568 times.
✓ Branch 1 taken 48 times.
✓ Branch 2 taken 8 times.
✓ Branch 3 taken 560 times.
|
616 | if (rect.height() == 0 || rect.width() == 0) { |
| 90 |
9/18✓ Branch 0 taken 56 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 56 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 56 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 56 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 56 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 56 times.
✗ Branch 11 not taken.
✓ Branch 12 taken 56 times.
✗ Branch 13 not taken.
✓ Branch 14 taken 56 times.
✗ Branch 15 not taken.
✓ Branch 16 taken 56 times.
✗ Branch 17 not taken.
|
56 | GTEST_SKIP() << "Empty dimension of matrix(" << rect.width() << "," << rect.height() << ")"; |
| 91 | } | ||
| 92 | |||
| 93 | // Generates input data. | ||
| 94 | 560 | const auto ref_lhs_f16 = fill_random<Float16>(M * K, seed + 0); | |
| 95 |
1/2✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
|
560 | const auto ref_rhs = fill_random<float>(N * K, seed + 1); |
| 96 | 560 | Buffer ref_biases; | |
| 97 | |||
| 98 |
2/2✓ Branch 0 taken 280 times.
✓ Branch 1 taken 280 times.
|
560 | if (has_bias) { |
| 99 |
1/2✓ Branch 0 taken 280 times.
✗ Branch 1 not taken.
|
280 | ref_biases = fill_random<float>(N, seed + 2); |
| 100 | 280 | } | |
| 101 | // For reference implementation, Casting FP16 input to FP32 type and FP32 output back to FP16 because the matmul | ||
| 102 | // implementation works with FP32 accumulation and casts the result to FP16 | ||
| 103 |
3/6✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 560 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 560 times.
✗ Branch 5 not taken.
|
560 | const auto ref_lhs = cast<float, Float16>(ref_lhs_f16.data(), ref_lhs_f16.size() * 8 / size_in_bits<Float16>); |
| 104 | |||
| 105 | // Runs the reference implementation. | ||
| 106 | // * Quantizes the LHS matrix using 8-bit symmetric quantization. | ||
| 107 | // * Quantizes the RHS matrix using 8-bit asymmetric quantization. | ||
| 108 | // * Performs GEMM. | ||
| 109 | 1680 | const auto [ref_lhs_qvalues, ref_lhs_scales, ref_lhs_zero_points] = | |
| 110 |
2/4✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 560 times.
✗ Branch 3 not taken.
|
560 | quantize_asymmetric_per_block_dynamic<float, int8_t, float, int32_t>(ref_lhs.data(), M, K, K); |
| 111 | 2800 | const auto [ref_rhs_qsi4, ref_rhs_scales] = | |
| 112 |
2/4✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 560 times.
✗ Branch 3 not taken.
|
560 | quantize_symmetric_per_block_dynamic<float, Int4, float>(ref_rhs.data(), N, K, K); |
| 113 | |||
| 114 | 560 | const auto ref_dst_no_clamp = | |
| 115 |
1/2✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
|
560 | matmul_nt_t_quantized<int8_t, float, int32_t, Int4, float, int32_t, float, float, int32_t, float>( |
| 116 |
4/8✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 560 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 560 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 560 times.
✗ Branch 7 not taken.
|
1120 | M, N, K, ref_lhs_qvalues.data(), ref_lhs_scales.data(), ref_lhs_zero_points.data(), 1, K, |
| 117 |
7/10✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 560 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 280 times.
✓ Branch 5 taken 280 times.
✓ Branch 6 taken 280 times.
✓ Branch 7 taken 280 times.
✓ Branch 8 taken 280 times.
✗ Branch 9 not taken.
|
560 | ref_rhs_qsi4.data(), ref_rhs_scales.data(), nullptr, 1, K, has_bias ? ref_biases.data() : nullptr, nullptr, |
| 118 | nullptr, 1); | ||
| 119 | |||
| 120 | // Clamps the reference output. | ||
| 121 | 560 | const auto clamp_ratio = 0.8F; | |
| 122 |
2/4✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 560 times.
✗ Branch 3 not taken.
|
1680 | const auto [clamp_min, clamp_max] = find_clamp_range<float>(ref_dst_no_clamp.data(), M * N, clamp_ratio); |
| 123 |
4/8✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 560 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 560 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 560 times.
✗ Branch 7 not taken.
|
560 | const auto ref_dst_float = clamp<float>(ref_dst_no_clamp.data(), M * N, clamp_min, clamp_max); |
| 124 | |||
| 125 | // Cast the reference output to F16 | ||
| 126 |
3/6✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 560 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 560 times.
✗ Branch 5 not taken.
|
560 | auto ref_dst = cast<Float16, float>(ref_dst_float.data(), ref_dst_float.size() * 8 / size_in_bits<float>); |
| 127 | |||
| 128 | // Runs the LHS packing micro-kernel. | ||
| 129 |
1/2✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
|
560 | const auto lhs_start_row = rect.start_row(); |
| 130 |
1/2✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
|
560 | const auto imp_packed_lhs_size = kai_get_lhs_packed_size_lhs_quant_pack_qai8dxp_f16_neon(M, K, mr, kr, sr); |
| 131 |
1/2✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
|
560 | Buffer imp_packed_lhs(imp_packed_lhs_size); |
| 132 | |||
| 133 | 560 | auto lhs_stride = K * sizeof(uint16_t); | |
| 134 |
1/2✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
|
560 | auto lhs_offset = kai_get_lhs_offset_lhs_quant_pack_qai8dxp_f16_neon(lhs_start_row, lhs_stride); |
| 135 |
1/2✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
|
560 | auto lhs_packed_offset = kai_get_lhs_packed_offset_lhs_quant_pack_qai8dxp_f16_neon(lhs_start_row, K, mr, kr, sr); |
| 136 |
1/2✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
|
560 | auto lhs_matmul_offset = ukernel_variant.interface.get_lhs_packed_offset(lhs_start_row, K); |
| 137 | |||
| 138 |
4/16✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 560 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 560 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
✓ Branch 14 taken 560 times.
✗ Branch 15 not taken.
|
560 | ASSERT_EQ(lhs_packed_offset, lhs_matmul_offset); |
| 139 | |||
| 140 |
1/2✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
|
560 | kai_run_lhs_quant_pack_qai8dxp_f16_neon( |
| 141 |
2/4✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 560 times.
✗ Branch 3 not taken.
|
560 | rect.height() /* m */, K, mr, kr, sr, 0, ref_lhs_f16.data() + lhs_offset, lhs_stride, |
| 142 |
1/2✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
|
560 | imp_packed_lhs.data() + lhs_packed_offset); |
| 143 | |||
| 144 |
1/2✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
|
1120 | const auto ref_rhs_qsi4_padded = pad_row<Int4>( |
| 145 |
4/8✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 560 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 560 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 560 times.
✗ Branch 7 not taken.
|
560 | ref_rhs_qsi4.data(), N, K, K, round_up_multiple(K, 2), round_up_division(N * round_up_multiple(K, 2), 2)); |
| 146 | |||
| 147 |
1/2✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
|
560 | const auto imp_packed_rhs_size = kai_get_rhs_packed_size_rhs_pack_nxk_qsi4cxp_qs4cxs1s0(N, K, nr, kr, sr); |
| 148 |
1/2✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
|
560 | Buffer imp_packed_rhs(imp_packed_rhs_size); |
| 149 |
1/2✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
|
560 | const auto rhs_start_row = rect.start_col(); |
| 150 |
1/2✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
|
560 | auto rhs_packed_offset = kai_get_rhs_packed_offset_rhs_pack_nxk_qsi4cxp_qs4cxs1s0(rhs_start_row, K, nr, kr, sr); |
| 151 |
1/2✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
|
560 | auto rhs_matmul_offset = ukernel_variant.interface.get_rhs_packed_offset(rhs_start_row, K); |
| 152 |
4/16✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 560 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 560 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✓ Branch 15 taken 560 times.
|
560 | ASSERT_EQ(rhs_packed_offset, rhs_matmul_offset); |
| 153 | |||
| 154 | // Runs the RHS packing micro-kernel. | ||
| 155 | 560 | kai_rhs_pack_nxk_qsi4cxp_qs4cxs1s0_params params{}; | |
| 156 | 560 | params.lhs_zero_point = 1; | |
| 157 | 560 | params.rhs_zero_point = 0; | |
| 158 | |||
| 159 |
1/2✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
|
560 | kai_run_rhs_pack_nxk_qsi4cxp_qs4cxs1s0( |
| 160 |
1/2✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
|
560 | 1, N, K, nr, kr, sr, reinterpret_cast<const uint8_t*>(ref_rhs_qsi4_padded.data()), |
| 161 |
3/4✓ Branch 0 taken 280 times.
✓ Branch 1 taken 280 times.
✓ Branch 2 taken 280 times.
✗ Branch 3 not taken.
|
560 | has_bias ? reinterpret_cast<const float*>(ref_biases.data()) : nullptr, |
| 162 |
2/4✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 560 times.
✗ Branch 3 not taken.
|
560 | reinterpret_cast<const float*>(ref_rhs_scales.data()), imp_packed_rhs.data(), 0, ¶ms); |
| 163 | |||
| 164 | 560 | const auto dst_stride_row = N * sizeof(uint16_t); | |
| 165 | 560 | const auto dst_stride_col = sizeof(uint16_t); | |
| 166 | 1120 | const auto dst_offset = | |
| 167 |
3/6✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 560 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 560 times.
✗ Branch 5 not taken.
|
560 | ukernel_variant.interface.get_dst_offset(rect.start_row(), rect.start_col(), dst_stride_row); |
| 168 |
2/4✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 560 times.
✗ Branch 3 not taken.
|
560 | const auto ref_dst_offset = rect.start_row() * dst_stride_row + rect.start_col() * dst_stride_col; |
| 169 |
4/16✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 560 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 560 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✓ Branch 15 taken 560 times.
|
560 | ASSERT_EQ(dst_offset, ref_dst_offset); |
| 170 | |||
| 171 | // Runs the GEMM micro-kernel. | ||
| 172 |
1/2✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
|
560 | const auto imp_dst_size = ukernel_variant.interface.get_dst_size(M, N); |
| 173 |
5/18✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 560 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 560 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 560 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
✗ Branch 16 not taken.
✓ Branch 17 taken 560 times.
|
560 | ASSERT_EQ(imp_dst_size, ref_dst.size()); |
| 174 |
1/2✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
|
560 | Buffer imp_dst(imp_dst_size); |
| 175 |
1/2✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
|
1120 | ukernel_variant.interface.run_matmul( |
| 176 |
3/6✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 560 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 560 times.
✗ Branch 5 not taken.
|
560 | rect.height(), rect.width(), K, imp_packed_lhs.data() + lhs_matmul_offset, |
| 177 |
2/4✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 560 times.
✗ Branch 3 not taken.
|
560 | imp_packed_rhs.data() + rhs_matmul_offset, imp_dst.data() + dst_offset, dst_stride_row, dst_stride_col, |
| 178 | 1120 | clamp_min, clamp_max); | |
| 179 | |||
| 180 | // Compares the output of the micro-kernels against the output of the reference implementation for the portion | ||
| 181 | // tested. | ||
| 182 |
1/2✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
|
560 | DefaultMismatchHandler handler(0, 0.02, 0, 0.05); |
| 183 | 560 | DataFormat dst_format = DataFormat(DataType::FP16); | |
| 184 |
3/6✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 560 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 560 times.
✗ Branch 5 not taken.
|
560 | const auto success = compare(imp_dst.data(), ref_dst.data(), dst_format, M, N, rect, handler); |
| 185 |
4/16✓ Branch 0 taken 560 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 560 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 560 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✓ Branch 15 taken 560 times.
|
560 | ASSERT_TRUE(success); |
| 186 | 896 | } | |
| 187 |
19/62✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
✓ Branch 4 taken 1 time.
✗ Branch 5 not taken.
✓ Branch 6 taken 1 time.
✗ Branch 7 not taken.
✓ Branch 8 taken 1 time.
✗ Branch 9 not taken.
✓ Branch 10 taken 1 time.
✗ Branch 11 not taken.
✓ Branch 12 taken 1 time.
✗ Branch 13 not taken.
✓ Branch 14 taken 1 time.
✗ Branch 15 not taken.
✓ Branch 16 taken 1 time.
✗ Branch 17 not taken.
✓ Branch 18 taken 1 time.
✗ Branch 19 not taken.
✓ Branch 20 taken 1 time.
✗ Branch 21 not taken.
✓ Branch 22 taken 1 time.
✗ Branch 23 not taken.
✓ Branch 24 taken 1 time.
✗ Branch 25 not taken.
✓ Branch 26 taken 1 time.
✗ Branch 27 not taken.
✓ Branch 28 taken 1 time.
✗ Branch 29 not taken.
✓ Branch 30 taken 1 time.
✗ Branch 31 not taken.
✓ Branch 32 taken 1 time.
✗ Branch 33 not taken.
✓ Branch 34 taken 896 times.
✗ Branch 35 not taken.
✗ Branch 36 not taken.
✗ Branch 37 not taken.
✗ Branch 38 not taken.
✗ Branch 39 not taken.
✗ Branch 40 not taken.
✗ Branch 41 not taken.
✗ Branch 42 not taken.
✗ Branch 43 not taken.
✗ Branch 44 not taken.
✗ Branch 45 not taken.
✗ Branch 46 not taken.
✗ Branch 47 not taken.
✗ Branch 48 not taken.
✗ Branch 49 not taken.
✗ Branch 50 not taken.
✗ Branch 51 not taken.
✗ Branch 52 not taken.
✗ Branch 53 not taken.
✗ Branch 54 not taken.
✗ Branch 55 not taken.
✗ Branch 56 not taken.
✗ Branch 57 not taken.
✗ Branch 58 not taken.
✗ Branch 59 not taken.
✓ Branch 60 taken 896 times.
✗ Branch 61 not taken.
|
1794 | INSTANTIATE_TEST_SUITE_P( |
| 188 | MatMul, MatMulTest_f16_qai8dxp_qsi4cxp, | ||
| 189 | testing::Combine( | ||
| 190 | testing::Range<size_t>(0, variants_kai_matmul_clamp_f16_qai8dxp_qsi4cxp.size()), | ||
| 191 | testing::Values( | ||
| 192 | MatMulShape{1, 2, 32}, // | ||
| 193 | MatMulShape{1, 3, 32}, // | ||
| 194 | MatMulShape{1, 4, 32}, // | ||
| 195 | MatMulShape{1, 5, 31}, // | ||
| 196 | MatMulShape{3, 3, 32}, // | ||
| 197 | MatMulShape{4, 4, 32}, // | ||
| 198 | MatMulShape{5, 5, 31}, // | ||
| 199 | MatMulShape{16, 32, 64}, // | ||
| 200 | MatMulShape{16, 32, 36}, // | ||
| 201 | MatMulShape{15, 35, 65}, // | ||
| 202 | MatMulShape{8, 32, 64}, // | ||
| 203 | MatMulShape{15, 31, 45}, // | ||
| 204 | MatMulShape{1, 35, 65}, // | ||
| 205 | MatMulShape{1, 128, 32}, // | ||
| 206 | MatMulShape{64, 128, 32}, // | ||
| 207 | MatMulShape{77, 99, 64}), | ||
| 208 | testing::Values( | ||
| 209 | MatrixPortion(0, 0, 1, 1), // Full matrix. | ||
| 210 | MatrixPortion(0, 0, 1, 0.25), // Leftmost portion. | ||
| 211 | MatrixPortion(0, 0.75, 1, 1), // Rightmost portion. | ||
| 212 | MatrixPortion(0, 0.5, 1, 0.8), // Somewhere Middle | ||
| 213 | MatrixPortion(0.75, 0.75, 1, 1), // Bottom-right corner. | ||
| 214 | MatrixPortion(0.75, 0, 1, 1), // Partial rows | ||
| 215 | MatrixPortion(0.4, 0.5, 0.6, 0.8) // Somewhere Middle | ||
| 216 | ), | ||
| 217 | testing::Bool()), | ||
| 218 | [](const auto& info) { | ||
| 219 | const auto variant_idx = std::get<0>(info.param); | ||
| 220 | const std::string name{variants_kai_matmul_clamp_f16_qai8dxp_qsi4cxp.at(variant_idx).name}; | ||
| 221 | const auto shape = std::get<MatMulShape>(info.param); | ||
| 222 | const auto portion = std::get<2>(info.param); | ||
| 223 | const auto has_bias = std::get<3>(info.param); | ||
| 224 | |||
| 225 | return test_description(name, shape, portion, has_bias); | ||
| 226 | }); | ||
| 227 | |||
| 228 | } // namespace kai::test | ||
| 229 |